vault backup: 2026-04-22 10:10:19
This commit is contained in:
+62
-74
@@ -244,89 +244,77 @@ graph LR
|
||||
|
||||
## 参考实现
|
||||
|
||||
### 伪代码
|
||||
### 核心实现(伪代码)
|
||||
|
||||
```python
|
||||
class HeavyKeeper:
|
||||
def __init__(self, m, L, threshold, decay_factor):
|
||||
self.m = m # 每层桶数
|
||||
self.L = L # 哈希层数
|
||||
self.threshold = threshold
|
||||
self.decay_factor = decay_factor # 衰减因子
|
||||
|
||||
# 初始化多层 sketch
|
||||
self.buckets = [[KeeperBucket() for _ in range(m)]
|
||||
for _ in range(L)]
|
||||
|
||||
# 初始化哈希函数
|
||||
self.hash_funcs = [get_hash_func(i) for i in range(L)]
|
||||
|
||||
def insert(self, item, timestamp):
|
||||
def __init__(self, m, L, decay_factor):
|
||||
self.m, self.L = m, L
|
||||
self.decay_factor = decay_factor
|
||||
self.buckets = [[Bucket() for _ in range(m)] for _ in range(L)]
|
||||
|
||||
def insert(self, item):
|
||||
for layer in range(self.L):
|
||||
idx = self.hash_funcs[layer](item) % self.m
|
||||
bucket = self.buckets[layer][idx]
|
||||
|
||||
bucket = self.buckets[layer][hash(item) % self.m]
|
||||
if bucket.item == item:
|
||||
# 守护项匹配,直接计数(大象流强化)
|
||||
bucket.count += 1
|
||||
bucket.last_seen = timestamp
|
||||
bucket.count += 1 # 守护项匹配,直接计数
|
||||
else:
|
||||
# 计算替换概率
|
||||
estimated_freq = self._estimate_freq(item)
|
||||
replace_prob = min(1, estimated_freq / bucket.count)
|
||||
|
||||
if random.random() < replace_prob:
|
||||
# 替换为新项(大象流夺权)
|
||||
bucket.item = item
|
||||
bucket.count = 1
|
||||
bucket.error = bucket.count
|
||||
bucket.last_seen = timestamp
|
||||
# 空桶直接占领;否则按概率争夺
|
||||
prob = 1 if bucket.count == 0 else min(1, self._estimate(item) / bucket.count)
|
||||
if random() < prob:
|
||||
bucket.item, bucket.count, bucket.error = item, 1, 0 # 替换为新项
|
||||
else:
|
||||
# 不替换,仅增加误差(老鼠流被阻拦)
|
||||
bucket.error += 1
|
||||
|
||||
def apply_decay(self, current_time):
|
||||
"""应用衰减机制"""
|
||||
for layer in range(self.L):
|
||||
for bucket in self.buckets[layer]:
|
||||
elapsed = current_time - bucket.last_seen
|
||||
if elapsed > DECAY_INTERVAL:
|
||||
bucket.count *= self.decay_factor
|
||||
bucket.error *= self.decay_factor
|
||||
|
||||
# 归零清理
|
||||
if bucket.count < 1:
|
||||
bucket.item = None
|
||||
bucket.count = 0
|
||||
bucket.error = 0
|
||||
|
||||
def query(self, item):
|
||||
"""查询Item的频率估计"""
|
||||
min_count = float('inf')
|
||||
for layer in range(self.L):
|
||||
idx = self.hash_funcs[layer](item) % self.m
|
||||
bucket = self.buckets[layer][idx]
|
||||
if bucket.item == item:
|
||||
min_count = min(min_count, bucket.count)
|
||||
|
||||
return min_count if min_count != float('inf') else 0
|
||||
|
||||
def get_top_k(self, k):
|
||||
"""获取Top K大象流"""
|
||||
candidates = {}
|
||||
for layer in range(self.L):
|
||||
for bucket in self.buckets[layer]:
|
||||
if bucket.count >= self.threshold and bucket.item:
|
||||
item = bucket.item
|
||||
candidates[item] = max(candidates.get(item, 0),
|
||||
bucket.count)
|
||||
|
||||
# 返回 Top-k
|
||||
return sorted(candidates.items(),
|
||||
key=lambda x: x[1],
|
||||
reverse=True)[:k]
|
||||
bucket.error += 1 # 不替换,仅增加误差
|
||||
```
|
||||
|
||||
### Go 参考实现
|
||||
|
||||
```go
|
||||
package heavykeeper
|
||||
|
||||
type Bucket struct {
|
||||
Item []byte
|
||||
Count uint32
|
||||
Error uint32
|
||||
}
|
||||
|
||||
type HeavyKeeper struct {
|
||||
m, L uint32
|
||||
decayFactor float64
|
||||
buckets [][]Bucket
|
||||
hashFuncs []func([]byte) uint64
|
||||
}
|
||||
|
||||
func (hk *HeavyKeeper) Insert(item []byte) {
|
||||
for layer := uint32(0); layer < hk.L; layer++ {
|
||||
idx := hk.hashFuncs[layer](item) % hk.m
|
||||
bucket := &hk.buckets[layer][idx]
|
||||
|
||||
if bytes.Equal(bucket.Item, item) {
|
||||
bucket.Count++
|
||||
continue
|
||||
}
|
||||
|
||||
var prob float64
|
||||
if bucket.Count == 0 {
|
||||
prob = 1 // 空桶直接占领
|
||||
} else {
|
||||
prob = float64(hk.estimate(item)) / float64(bucket.Count)
|
||||
if prob > 1 {
|
||||
prob = 1
|
||||
}
|
||||
}
|
||||
if rand.Float64() < prob {
|
||||
bucket.Item, bucket.Count, bucket.Error = item, 1, 0
|
||||
} else {
|
||||
bucket.Error++
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
> **说明**:`insert` 是核心逻辑——对每层哈希找桶,匹配守护项则计数,否则按概率争夺。伪代码省略了衰减、查询、Top-K 等辅助方法,实际部署时按需补充。
|
||||
|
||||
## 相关算法对比
|
||||
|
||||
| 算法 | 空间复杂度 | 大象流准确性 | 老鼠流过滤 | 衰减支持 | 适用场景 |
|
||||
|
||||
Reference in New Issue
Block a user