This commit is contained in:
@@ -20,30 +20,28 @@
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph 插入元素x["插入元素 x"]
|
||||
X["x"] --> H1["h₁(x) = 3"]
|
||||
X --> H2["h₂(x) = 7"]
|
||||
X --> H3["h₃(x) = 11"]
|
||||
H1 --> B3["bit[3] = 1"]
|
||||
H2 --> B7["bit[7] = 1"]
|
||||
H3 --> B11["bit[11] = 1"]
|
||||
end
|
||||
X["元素 x"] --> H1["h1(x) = 3"]
|
||||
X --> H2["h2(x) = 7"]
|
||||
X --> H3["h3(x) = 11"]
|
||||
H1 --> B3["bit[3] = 1"]
|
||||
H2 --> B7["bit[7] = 1"]
|
||||
H3 --> B11["bit[11] = 1"]
|
||||
```
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph 查询y["查询元素 y — 一定不存在"]
|
||||
Y["y"] --> YH1["h₁(y) = 3 → bit[3]=1 ✓"]
|
||||
Y --> YH2["h₂(y) = 5 → bit[5]=0 ✗"]
|
||||
YH1 --> YR["结论:一定不存在"]
|
||||
subgraph qy["查询元素 y - 一定不存在"]
|
||||
Y["y"] --> YH1["h1(y)=3 bit[3]=1"]
|
||||
Y --> YH2["h2(y)=5 bit[5]=0"]
|
||||
YH1 --> YR["结论: 一定不存在"]
|
||||
YH2 --> YR
|
||||
end
|
||||
|
||||
subgraph 查询z["查询元素 z — 假阳性"]
|
||||
Z["z"] --> ZH1["h₁(z)=3 → bit[3]=1 ✓"]
|
||||
Z --> ZH2["h₂(z)=7 → bit[7]=1 ✓"]
|
||||
Z --> ZH3["h₃(z)=11 → bit[11]=1 ✓"]
|
||||
ZH1 --> ZR["结论:可能存在<br/>⚠️ 实际未插入(假阳性)"]
|
||||
subgraph qz["查询元素 z - 假阳性"]
|
||||
Z["z"] --> ZH1["h1(z)=3 bit[3]=1"]
|
||||
Z --> ZH2["h2(z)=7 bit[7]=1"]
|
||||
Z --> ZH3["h3(z)=11 bit[11]=1"]
|
||||
ZH1 --> ZR["结论: 可能存在 / 实际未插入"]
|
||||
ZH2 --> ZR
|
||||
ZH3 --> ZR
|
||||
end
|
||||
@@ -51,15 +49,8 @@ graph LR
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph 位数组["位数组 (m=16)"]
|
||||
direction LR
|
||||
b0["0"] --- b1["0"] --- b2["0"] --- b3["1"] --- b4["0"] --- b5["0"] --- b6["0"] --- b7["1"]
|
||||
b7 --- b8["0"] --- b9["0"] --- b10["0"] --- b11["1"] --- b12["0"] --- b13["0"] --- b14["0"] --- b15["0"]
|
||||
end
|
||||
|
||||
插入元素x -.->|h₁ h₂ h₃| b3
|
||||
插入元素x -.-> b7
|
||||
插入元素x -.-> b11
|
||||
b0["0"] --- b1["0"] --- b2["0"] --- b3["1"] --- b4["0"] --- b5["0"] --- b6["0"] --- b7["1"]
|
||||
b7 --- b8["0"] --- b9["0"] --- b10["0"] --- b11["1"] --- b12["0"] --- b13["0"] --- b14["0"] --- b15["0"]
|
||||
```
|
||||
|
||||
### 假阳性率
|
||||
@@ -81,32 +72,32 @@ $$p \approx \left(1 - e^{-kn/m}\right)^k$$
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph BF["布隆过滤器 — 极致省空间"]
|
||||
subgraph BF["布隆过滤器 - 极致省空间"]
|
||||
B1["位数组 m bit"] --> B2["k 个哈希函数"]
|
||||
B2 --> B3["插入: O(k)"]
|
||||
B2 --> B4["查询: O(k)"]
|
||||
B2 --> B5["删除: ❌"]
|
||||
B2 --> B5["删除: 不支持"]
|
||||
B2 --> B6["假阳性: 有"]
|
||||
end
|
||||
|
||||
subgraph HS["HashSet — 精确但占空间"]
|
||||
subgraph HS["HashSet - 精确但占空间"]
|
||||
HS1["存完整元素"] --> HS2["哈希表"]
|
||||
HS2 --> HS3["插入: O(1)"]
|
||||
HS2 --> HS4["查询: O(1)"]
|
||||
HS2 --> HS5["删除: ✅"]
|
||||
HS2 --> HS5["删除: 支持"]
|
||||
HS2 --> HS6["假阳性: 无"]
|
||||
end
|
||||
|
||||
subgraph CBF["Counting Bloom — 可删除"]
|
||||
subgraph CBF["Counting Bloom - 可删除"]
|
||||
C1["计数器数组 4m bit"] --> C2["k 个哈希函数"]
|
||||
C2 --> C3["插入: O(k)"]
|
||||
C2 --> C4["查询: O(k)"]
|
||||
C2 --> C5["删除: ✅"]
|
||||
C2 --> C5["删除: 支持"]
|
||||
C2 --> C6["假阳性: 有"]
|
||||
end
|
||||
|
||||
BF -.->|"约 4x 空间差"| CBF
|
||||
HS -.->|"约 4~10x 空间差"| BF
|
||||
HS -.->|"约 4-10x 空间差"| BF
|
||||
```
|
||||
|
||||
| | 布隆过滤器 | HashSet | Counting Bloom Filter |
|
||||
@@ -207,16 +198,16 @@ sequenceDiagram
|
||||
participant DB as DB
|
||||
|
||||
C->>BF: 查询 user:50
|
||||
BF-->>C: 一定不存在 ✗
|
||||
C-->>C: 直接返回,不查缓存/DB
|
||||
BF-->>C: 一定不存在
|
||||
C-->>C: 直接返回 不查缓存和DB
|
||||
|
||||
C->>BF: 查询 user:100
|
||||
BF-->>C: 可能存在 ✓
|
||||
BF-->>C: 可能存在
|
||||
C->>R: GET user:100
|
||||
alt 缓存命中
|
||||
R-->>C: 返回数据 ✅
|
||||
else 缓存 miss
|
||||
C->>DB: SELECT * FROM users WHERE id=100
|
||||
R-->>C: 返回数据
|
||||
else 缓存miss
|
||||
C->>DB: SELECT
|
||||
DB-->>C: 返回数据
|
||||
C->>R: SET user:100 + TTL
|
||||
end
|
||||
|
||||
@@ -19,24 +19,14 @@
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph 数据流["高速数据流"]
|
||||
F1["流 A: 10000次 🐘"]
|
||||
F2["流 B: 8000次 🐘"]
|
||||
F3["流 C: 1次 🐭"]
|
||||
F4["流 D: 2次 🐭"]
|
||||
F5["流 E: 1次 🐭"]
|
||||
F6["流 ...: 1次 🐭"]
|
||||
end
|
||||
F1["流A: 10000次"] --> HK["HeavyKeeper"]
|
||||
F2["流B: 8000次"] --> HK
|
||||
F3["流C: 1次"] --> HK
|
||||
F4["流D: 2次"] --> HK
|
||||
F5["流E: 1次"] --> HK
|
||||
|
||||
F1 --> HK["HeavyKeeper"]
|
||||
F2 --> HK
|
||||
F3 --> HK
|
||||
F4 --> HK
|
||||
F5 --> HK
|
||||
F6 --> HK
|
||||
|
||||
HK --> TOP["Top-K 结果:<br/>流 A: ~10000<br/>流 B: ~8000"]
|
||||
HK -.->|老鼠流被衰减淘汰| GONE["流 C~E: 归零 🗑️"]
|
||||
HK --> TOP["Top-K: 流A~10000 流B~8000"]
|
||||
HK -.->|"老鼠流衰减淘汰"| GONE["流C~E: 归零"]
|
||||
```
|
||||
|
||||
### 背景:为什么需要 HeavyKeeper
|
||||
@@ -53,26 +43,21 @@ graph LR
|
||||
```mermaid
|
||||
graph TB
|
||||
Input["插入流 x"] --> Loop["遍历 d 行"]
|
||||
|
||||
Loop --> Hash["pos = hᵢ(x) mod w"]
|
||||
Loop --> Hash["pos = hi(x) mod w"]
|
||||
Hash --> Check{"bucket[pos]?"}
|
||||
|
||||
Check -->|"flowID == x"| Hit["count++"]
|
||||
Check -->|"flowID == 空"| Empty["写入 (x, 1)"]
|
||||
Check -->|"flowID ≠ x"| Conflict{"random() < p?"}
|
||||
|
||||
Check -->|"flowID != x"| Conflict{"random() < p?"}
|
||||
Conflict -->|"否"| Skip["跳过"]
|
||||
Conflict -->|"是"| Decay["count--"]
|
||||
|
||||
Decay --> Zero{"count == 0?"}
|
||||
Zero -->|"是"| Replace["替换为 (x, 1)"]
|
||||
Zero -->|"否"| Skip2["保留原流"]
|
||||
|
||||
Zero -->|"否"| Keep["保留原流"]
|
||||
Hit --> NextRow["下一行"]
|
||||
Empty --> NextRow
|
||||
Skip --> NextRow
|
||||
Replace --> NextRow
|
||||
Skip2 --> NextRow
|
||||
Keep --> NextRow
|
||||
NextRow --> Loop
|
||||
|
||||
style Hit fill:#4caf50,color:#fff
|
||||
@@ -83,17 +68,6 @@ graph TB
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph 数据结构["HeavyKeeper 结构 (d=3, w=8)"]
|
||||
direction TB
|
||||
Row0["行 0: h₀(x)"]
|
||||
Row1["行 1: h₁(x)"]
|
||||
Row2["行 2: h₂(x)"]
|
||||
|
||||
Row0 --- B00["流C:3"] --- B01["流A:10000"] --- B02[" "] --- B03["流D:1"] --- B04["流A:9500"] --- B05[" "] --- B06["流B:8000"] --- B07["流A:9800"]
|
||||
Row1 --- B10["流B:7500"] --- B11[" "] --- B12["流A:10200"] --- B13["流E:1"] --- B14[" "] --- B15["流B:8200"] --- B16[" "] --- B17["流A:9000"]
|
||||
Row2 --- B20[" "] --- B21["流A:10100"] --- B22["流C:2"] --- B23["流B:7800"] --- B24[" "] --- B25["流A:9900"] --- B26[" "] --- B27["流D:0🗑️"]
|
||||
end
|
||||
|
||||
Query["查询流 A"] --> Max["取各行最大 count = 10200"]
|
||||
```
|
||||
|
||||
@@ -126,20 +100,20 @@ Top-k 查询:
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph p0["p = 0 — 无衰减"]
|
||||
P0E["老鼠流永不清除<br/>噪声累积,Top-K 精度低"]
|
||||
subgraph p0["p=0 无衰减"]
|
||||
P0E["老鼠流永不清除 / 噪声累积 / Top-K精度低"]
|
||||
end
|
||||
|
||||
subgraph pok["p 适中 (0.01~0.1) — 最佳"]
|
||||
POKE["老鼠流: count=1 → 一次衰减→0 🗑️<br/>大象流: count=10000 → -1 无感 ✅<br/>Top-K 精度高"]
|
||||
subgraph pok["p适中 0.01-0.1 最佳"]
|
||||
POKE["老鼠流count=1 一次衰减归零 / 大象流count=10000 -1无感 / Top-K精度高"]
|
||||
end
|
||||
|
||||
subgraph pbig["p 过大 (>0.3)"]
|
||||
PBIGE["大象流也被严重衰减<br/>估计偏低,Top-K 不准"]
|
||||
subgraph pbig["p过大 大于0.3"]
|
||||
PBIGE["大象流也被严重衰减 / 估计偏低 / Top-K不准"]
|
||||
end
|
||||
|
||||
p0 -->|"增大 p"| pok
|
||||
pok -->|"继续增大 p"| pbig
|
||||
p0 -->|"增大p"| pok
|
||||
pok -->|"继续增大p"| pbig
|
||||
```
|
||||
|
||||
| 衰减概率 p | 效果 |
|
||||
@@ -304,14 +278,9 @@ func sortDesc(items []Item) {
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
Log["Nginx Access Log<br/>stdin 流式读取"] --> Parse["提取 Client IP"]
|
||||
Parse --> HK["HeavyKeeper<br/>4行 × 65536桶 × 5%衰减<br/>≈ 2MB"]
|
||||
HK --> TopK["Top-10 热门 IP"]
|
||||
|
||||
subgraph 流量模型
|
||||
Elephant["192.168.1.100 🐘<br/>10.0.0.5 🐘"] -->|"高频"| HK
|
||||
Mouse["172.16.x.x 🐭<br/>大量低频 IP"] -->|"衰减淘汰"| HK
|
||||
end
|
||||
Log["Nginx Access Log"] --> Parse["提取 Client IP"]
|
||||
Parse --> HK["HeavyKeeper / 4行x65536桶 / 5%衰减 / 约2MB"]
|
||||
HK --> TopK["Top-10 热门IP"]
|
||||
```
|
||||
|
||||
```go
|
||||
|
||||
@@ -17,22 +17,22 @@
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph 击穿["🔓 缓存击穿 — 单热点 key 过期"]
|
||||
subgraph bd["缓存击穿 - 单热点key过期"]
|
||||
B1["热 key 过期"] --> B2["大量并发同时 miss"]
|
||||
B2 --> B3["同时回源 DB"]
|
||||
B3 --> B4["DB 瞬时尖峰 💥"]
|
||||
B3 --> B4["DB 瞬时尖峰"]
|
||||
end
|
||||
|
||||
subgraph 雪崩["❄️ 缓存雪崩 — 大面积 key 集中失效"]
|
||||
A1["大量 key 同时刻过期<br/>或缓存节点宕机"] --> A2["请求大面积 miss"]
|
||||
subgraph av["缓存雪崩 - 大面积key集中失效"]
|
||||
A1["大量 key 同时刻过期 或 缓存节点宕机"] --> A2["请求大面积 miss"]
|
||||
A2 --> A3["全部涌向 DB"]
|
||||
A3 --> A4["DB 持续高压 💥"]
|
||||
A3 --> A4["DB 持续高压"]
|
||||
end
|
||||
|
||||
subgraph 穿透["🕳️ 缓存穿透 — 查不存在的数据"]
|
||||
subgraph pn["缓存穿透 - 查不存在的数据"]
|
||||
C1["请求查询不存在的 key"] --> C2["缓存永远 miss"]
|
||||
C2 --> C3["每次直达 DB"]
|
||||
C3 --> C4["DB 持续高压 💥"]
|
||||
C3 --> C4["DB 持续高压"]
|
||||
end
|
||||
```
|
||||
|
||||
@@ -51,18 +51,18 @@ graph TB
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph 击穿["击穿"]
|
||||
subgraph bd["击穿"]
|
||||
A1["A1 永不过期"]
|
||||
A2["A2 加锁排队<br/>singleflight"]
|
||||
A2["A2 加锁排队 / singleflight"]
|
||||
end
|
||||
|
||||
subgraph 雪崩["雪崩"]
|
||||
subgraph av["雪崩"]
|
||||
B1["B1 加锁/限流"]
|
||||
B2["B2 随机失效"]
|
||||
B3["B3 Redis 高可用<br/>多级缓存"]
|
||||
B3["B3 Redis高可用 / 多级缓存"]
|
||||
end
|
||||
|
||||
subgraph 穿透["穿透"]
|
||||
subgraph pn["穿透"]
|
||||
C1["C1 参数校验"]
|
||||
C2["C2 缓存空对象"]
|
||||
C3["C3 布隆过滤器"]
|
||||
@@ -245,23 +245,23 @@ func (s *Service) GetWithRateLimit(ctx context.Context, key string) (any, error)
|
||||
|
||||
```mermaid
|
||||
graph LR
|
||||
subgraph 无 Jitter["❌ 无随机偏移"]
|
||||
subgraph no_jitter["无随机偏移"]
|
||||
T1["key_a TTL=30min"] --> E1["同时过期"]
|
||||
T2["key_b TTL=30min"] --> E1
|
||||
T3["key_c TTL=30min"] --> E1
|
||||
E1 --> CRASH["DB 雪崩 💥"]
|
||||
E1 --> CRASH["DB 雪崩"]
|
||||
end
|
||||
|
||||
subgraph 有 Jitter["✅ 有随机偏移"]
|
||||
subgraph has_jitter["有随机偏移"]
|
||||
J1["key_a TTL=30m+2m"] --> D1["32min 过期"]
|
||||
J2["key_b TTL=30m+7m"] --> D2["37min 过期"]
|
||||
J3["key_c TTL=30m+4m"] --> D3["34min 过期"]
|
||||
D1 --> SAFE["DB 压力分散 ✅"]
|
||||
D1 --> SAFE["DB 压力分散"]
|
||||
D2 --> SAFE
|
||||
D3 --> SAFE
|
||||
end
|
||||
|
||||
无 Jitter -.->|加 jitter| 有 Jitter
|
||||
no_jitter -.->|"加 jitter"| has_jitter
|
||||
```
|
||||
|
||||
```go
|
||||
@@ -284,20 +284,12 @@ for _, item := range items {
|
||||
|
||||
```mermaid
|
||||
graph TB
|
||||
subgraph 多级缓存架构
|
||||
Client["客户端请求"]
|
||||
L1["L1 本地缓存<br/>bigcache / ristretto"]
|
||||
L2["L2 Redis<br/>Sentinel / Cluster"]
|
||||
L3["L3 DB"]
|
||||
|
||||
Client --> L1
|
||||
L1 -->|miss| L2
|
||||
L2 -->|miss| L3
|
||||
L3 --> L2
|
||||
L2 --> L1
|
||||
|
||||
L1 -.->|Redis 不可用时<br/>降级到本地| Client
|
||||
end
|
||||
Client["客户端请求"] --> L1["L1 本地缓存 / bigcache / ristretto"]
|
||||
L1 -->|"miss"| L2["L2 Redis / Sentinel / Cluster"]
|
||||
L2 -->|"miss"| L3["L3 DB"]
|
||||
L3 --> L2
|
||||
L2 --> L1
|
||||
L1 -.->|"Redis不可用时降级"| Client
|
||||
```
|
||||
|
||||
| 方案 | 说明 |
|
||||
@@ -391,12 +383,12 @@ func (s *Service) GetWithNullCache(ctx context.Context, key string) (any, error)
|
||||
```mermaid
|
||||
graph LR
|
||||
Req["请求"] --> BF{"布隆过滤器"}
|
||||
BF -->|"一定不存在 ✗"| Reject["直接拒绝<br/>不查缓存/DB"]
|
||||
BF -->|"可能存在 ✓"| Cache{"Redis 缓存"}
|
||||
Cache -->|hit| Return["返回数据"]
|
||||
Cache -->|miss| DB["查询 DB"]
|
||||
DB -->|有数据| Cache
|
||||
DB -->|无数据| Null["缓存空对象<br/>短 TTL"]
|
||||
BF -->|"一定不存在"| Reject["直接拒绝 / 不查缓存和DB"]
|
||||
BF -->|"可能存在"| Cache{"Redis 缓存"}
|
||||
Cache -->|"hit"| Return["返回数据"]
|
||||
Cache -->|"miss"| DB["查询 DB"]
|
||||
DB -->|"有数据"| Cache2["回写缓存"]
|
||||
DB -->|"无数据"| Null["缓存空对象 / 短TTL"]
|
||||
```
|
||||
|
||||
```go
|
||||
|
||||
Reference in New Issue
Block a user