vault backup: 2026-04-22 10:10:19

This commit is contained in:
2026-04-22 10:10:19 +08:00
parent 51f07228d3
commit 388d4439de
15 changed files with 1656 additions and 355 deletions
+62 -74
View File
@@ -244,89 +244,77 @@ graph LR
## 参考实现
### 伪代码
### 核心实现(伪代码)
```python
class HeavyKeeper:
def __init__(self, m, L, threshold, decay_factor):
self.m = m # 每层桶数
self.L = L # 哈希层数
self.threshold = threshold
self.decay_factor = decay_factor # 衰减因子
# 初始化多层 sketch
self.buckets = [[KeeperBucket() for _ in range(m)]
for _ in range(L)]
# 初始化哈希函数
self.hash_funcs = [get_hash_func(i) for i in range(L)]
def insert(self, item, timestamp):
def __init__(self, m, L, decay_factor):
self.m, self.L = m, L
self.decay_factor = decay_factor
self.buckets = [[Bucket() for _ in range(m)] for _ in range(L)]
def insert(self, item):
for layer in range(self.L):
idx = self.hash_funcs[layer](item) % self.m
bucket = self.buckets[layer][idx]
bucket = self.buckets[layer][hash(item) % self.m]
if bucket.item == item:
# 守护项匹配,直接计数(大象流强化)
bucket.count += 1
bucket.last_seen = timestamp
bucket.count += 1 # 守护项匹配,直接计数
else:
# 计算替换概率
estimated_freq = self._estimate_freq(item)
replace_prob = min(1, estimated_freq / bucket.count)
if random.random() < replace_prob:
# 替换为新项(大象流夺权)
bucket.item = item
bucket.count = 1
bucket.error = bucket.count
bucket.last_seen = timestamp
# 空桶直接占领;否则按概率争夺
prob = 1 if bucket.count == 0 else min(1, self._estimate(item) / bucket.count)
if random() < prob:
bucket.item, bucket.count, bucket.error = item, 1, 0 # 替换为新项
else:
# 不替换,仅增加误差(老鼠流被阻拦)
bucket.error += 1
def apply_decay(self, current_time):
"""应用衰减机制"""
for layer in range(self.L):
for bucket in self.buckets[layer]:
elapsed = current_time - bucket.last_seen
if elapsed > DECAY_INTERVAL:
bucket.count *= self.decay_factor
bucket.error *= self.decay_factor
# 归零清理
if bucket.count < 1:
bucket.item = None
bucket.count = 0
bucket.error = 0
def query(self, item):
"""查询Item的频率估计"""
min_count = float('inf')
for layer in range(self.L):
idx = self.hash_funcs[layer](item) % self.m
bucket = self.buckets[layer][idx]
if bucket.item == item:
min_count = min(min_count, bucket.count)
return min_count if min_count != float('inf') else 0
def get_top_k(self, k):
"""获取Top K大象流"""
candidates = {}
for layer in range(self.L):
for bucket in self.buckets[layer]:
if bucket.count >= self.threshold and bucket.item:
item = bucket.item
candidates[item] = max(candidates.get(item, 0),
bucket.count)
# 返回 Top-k
return sorted(candidates.items(),
key=lambda x: x[1],
reverse=True)[:k]
bucket.error += 1 # 不替换,仅增加误差
```
### Go 参考实现
```go
package heavykeeper
type Bucket struct {
Item []byte
Count uint32
Error uint32
}
type HeavyKeeper struct {
m, L uint32
decayFactor float64
buckets [][]Bucket
hashFuncs []func([]byte) uint64
}
func (hk *HeavyKeeper) Insert(item []byte) {
for layer := uint32(0); layer < hk.L; layer++ {
idx := hk.hashFuncs[layer](item) % hk.m
bucket := &hk.buckets[layer][idx]
if bytes.Equal(bucket.Item, item) {
bucket.Count++
continue
}
var prob float64
if bucket.Count == 0 {
prob = 1 // 空桶直接占领
} else {
prob = float64(hk.estimate(item)) / float64(bucket.Count)
if prob > 1 {
prob = 1
}
}
if rand.Float64() < prob {
bucket.Item, bucket.Count, bucket.Error = item, 1, 0
} else {
bucket.Error++
}
}
}
```
> **说明**:`insert` 是核心逻辑——对每层哈希找桶,匹配守护项则计数,否则按概率争夺。伪代码省略了衰减、查询、Top-K 等辅助方法,实际部署时按需补充。
## 相关算法对比
| 算法 | 空间复杂度 | 大象流准确性 | 老鼠流过滤 | 衰减支持 | 适用场景 |