nixevol/NixMsg
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

test: 补齐 Q2 验收与本机 Q3 弱网崩溃压测

Browse Source
This commit is contained in:
Nixevol
2026-09-30 08:59:20 +08:00
parent 1f715e4e51
commit a22912d95a
13 changed files with 1299 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
test/chaos/README.md
+7 -46
View File
@@ -1,61 +1,22 @@
# 混沌 / 弱网测试辅助（Q 线）
用 toxiproxy 官方镜像做延迟和断开；Linux 丢包用容器内 netem。所有 Docker 资源名带 `q` 前缀，用完删除。
用 toxiproxy 官方镜像做延迟和断开；Linux 丢包用容器内 netem。Q3 相关 Docker 资源名带 `q3` 前缀，用完删除。
## 启动 toxiproxy
在仓库根目录（或本目录）执行：
```powershell
docker compose -p q-chaos -f test/chaos/compose.yml up -d
```
API 默认映射到本机随机/固定端口见 compose 注释。查看实际端口：
```powershell
docker compose -p q-chaos -f test/chaos/compose.yml port toxiproxy 8474
docker compose -p q3-chaos -f test/chaos/compose.yml up -d
docker compose -p q3-chaos -f test/chaos/compose.yml port toxiproxy 8474
```
清理：
```powershell
docker compose -p q-chaos -f test/chaos/compose.yml down -v
docker compose -p q3-chaos -f test/chaos/compose.yml down -v
```
## 对运行中的服务注入故障
集成测 `TestQ3ToxiproxyOfflineKeepDelivered` 会自动起停上述 compose。
不要写死 `7443`。上游地址取自当前实例的实际监听端口（例如测试 harness 写入的 `listen.addr`，或你启动时打印的地址）。
## netem
1. 本机起好 NixMsg（或任意 TCP 服务），记下 `HOST:PORT`。
2. 在 Windows 上，容器访问本机服务用 `host.docker.internal:PORT`。
3. 用本包客户端或 curl 建代理（listen 用 `0.0.0.0:0` 让 toxiproxy 分配端口）：
```go
c := chaos.NewClient("http://127.0.0.1:<api端口>")
p, err := c.CreateProxy("q-nixmsg", "0.0.0.0:0", "host.docker.internal:"+strconv.Itoa(port))
// 客户端改连 p.Listen（把 0.0.0.0 换成 127.0.0.1）
_, _ = c.AddLatency("q-nixmsg", "lag", 200, 50, "")
_, _ = c.AddResetPeer("q-nixmsg", "rst", 0, "")
```
等价 curl：
```bash
curl -s -X POST http://127.0.0.1:<api>/proxies \
-H 'Content-Type: application/json' \
-d '{"name":"q-nixmsg","listen":"0.0.0.0:0","upstream":"host.docker.internal:<PORT>","enabled":true}'
```
延迟：`POST .../proxies/q-nixmsg/toxics`，`type=latency`，`attributes.latency` / `jitter`（毫秒）。
断开：`type=reset_peer`，`attributes.timeout`（毫秒，0 表示尽快 RST）。
## 单元测试
```powershell
go test ./test/chaos/ -count=1
```
## netem 丢包
见 [netem/README.md](./netem/README.md)。脚本必须在 Linux 容器内执行（本机是 Windows）。
见 [netem/README.md](./netem/README.md)。本机 Windows 上 Q3 将 netem 20% 丢包记为未测（见 `docs/DEVIATIONS.md` 测试交付 Q）。
test/chaos/compose.yml
+2 -4
View File
@@ -1,15 +1,13 @@
# toxiproxy：延迟与断开。项目名用 -p q-chaos，容器名带 q 前缀。
# toxiproxy：延迟与断开。项目名用 -p q3-chaos，容器名带 q3 前缀。
# API 与代理端口映射到本机，具体宿主机端口用 `docker compose port` 查询，不要写死业务端口。
services:
toxiproxy:
image: ghcr.io/shopify/toxiproxy:2.12.0
container_name: q-toxiproxy
# 8474=API；8475 起可手工映射代理端口，或在 CreateProxy 时用 0.0.0.0:0 再 docker port 查询
container_name: q3-toxiproxy
ports:
- "8474"
- "8475"
- "8476"
# 允许代理到本机服务（Docker Desktop / Windows）
extra_hosts:
- "host.docker.internal:host-gateway"
test/chaos/q3_crash_test.go
+56
View File
@@ -0,0 +1,56 @@
package chaos_test
import (
"testing"
"time"
"git.asio.asia/nixevol/NixMsg/test/accept"
"git.asio.asia/nixevol/NixMsg/test/harness"
)
// TestQ3CrashSubmitThenRestart：提交返回成功后杀进程，重启后续传（离线保留消息）。
func TestQ3CrashSubmitThenRestart(t *testing.T) {
srv, err := accept.StartManaged()
if err != nil {
t.Fatal(err)
}
defer func() { _ = srv.Cleanup() }()
hs := &harness.Server{
HTTPBase: srv.HTTPBase,
AdminHTTPBase: srv.AdminHTTPBase,
AdminPassword: srv.AdminPassword,
}
ac := accept.AdminLogin(t, hs)
accept.CreateEndpoint(t, ac, "q3crasha1", epPassword)
accept.CreateEndpoint(t, ac, "q3crashb1", epPassword)
alice := accept.MQTTLogin(t, srv.HTTPBase, "q3crasha1", epPassword)
sendOff := alice.Request(t, map[string]any{
"v": 1, "type": "send", "rid": "cr1", "id": "q3-crash-1",
"to": map[string]any{"kind": "endpoint", "id": "q3crashb1"},
"body": map[string]any{"enc": "utf8", "data": "after-crash"},
"delay_ms": int64(0),
"offline": map[string]any{"keep": true, "ttl_seconds": int64(86400)},
})
if !sendOff.OK {
t.Fatalf("submit before crash: %+v", sendOff)
}
alice.Close()
if err := srv.Kill(); err != nil {
t.Fatal(err)
}
time.Sleep(200 * time.Millisecond)
if err := srv.Restart(); err != nil {
t.Fatalf("restart: %v", err)
}
bob := accept.MQTTLogin(t, srv.HTTPBase, "q3crashb1", epPassword)
defer bob.Close()
msg := bob.WaitType(t, "msg", 20*time.Second)
if msg["id"] != "q3-crash-1" {
t.Fatalf("want q3-crash-1 after restart, got %v", msg)
}
bob.Request(t, map[string]any{"v": 1, "type": "ack", "rid": "cra1", "from": "q3crasha1", "id": "q3-crash-1"})
}
test/chaos/q3_weak_test.go
+180
View File
@@ -0,0 +1,180 @@
package chaos_test
import (
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strings"
"testing"
"time"
"git.asio.asia/nixevol/NixMsg/test/accept"
"git.asio.asia/nixevol/NixMsg/test/chaos"
"git.asio.asia/nixevol/NixMsg/test/harness"
)
const (
q3ComposeProject = "q3-chaos"
q3ProxyName = "q3-nixmsg"
epPassword = "password1234"
)
// TestQ3ToxiproxyOfflineKeepDelivered：延迟 + 断开后，离线保留期内消息最终送达。
func TestQ3ToxiproxyOfflineKeepDelivered(t *testing.T) {
if _, err := exec.LookPath("docker"); err != nil {
t.Skip("docker 不可用，跳过 toxiproxy 弱网")
}
composeFile := filepath.Join(moduleRoot(t), "test", "chaos", "compose.yml")
down := func() {
cmd := exec.Command("docker", "compose", "-p", q3ComposeProject, "-f", composeFile, "down", "-v", "--remove-orphans")
_ = cmd.Run()
}
down()
t.Cleanup(down)
up := exec.Command("docker", "compose", "-p", q3ComposeProject, "-f", composeFile, "up", "-d")
if out, err := up.CombinedOutput(); err != nil {
t.Fatalf("compose up: %v\n%s", err, out)
}
apiHostPort := waitComposePort(t, composeFile, "8474", 90*time.Second)
listenHostPort := waitComposePort(t, composeFile, "8475", 90*time.Second)
apiURL := "http://127.0.0.1:" + apiHostPort
waitHTTP(t, apiURL+"/version", 90*time.Second)
srv, err := harness.Start(harness.Options{})
if err != nil {
t.Fatal(err)
}
defer func() { _ = srv.Stop() }()
_, nixPort, err := net.SplitHostPort(srv.Addr)
if err != nil {
t.Fatal(err)
}
c := chaos.NewClient(apiURL)
_ = c.DeleteProxy(q3ProxyName)
if _, err := c.CreateProxy(q3ProxyName, "0.0.0.0:8475", "host.docker.internal:"+nixPort); err != nil {
t.Fatalf("CreateProxy: %v", err)
}
t.Cleanup(func() { _ = c.DeleteProxy(q3ProxyName) })
proxyBase := "http://127.0.0.1:" + listenHostPort
if _, err := c.AddLatency(q3ProxyName, "q3-lag", 150, 50, ""); err != nil {
t.Fatalf("AddLatency: %v", err)
}
ac := accept.AdminLogin(t, srv)
accept.CreateEndpoint(t, ac, "q3alice01", epPassword)
accept.CreateEndpoint(t, ac, "q3bob0001", epPassword)
alice := accept.MQTTLogin(t, proxyBase, "q3alice01", epPassword)
defer alice.Close()
sendOff := alice.Request(t, map[string]any{
"v": 1, "type": "send", "rid": "q3s1", "id": "q3-off-keep",
"to": map[string]any{"kind": "endpoint", "id": "q3bob0001"},
"body": map[string]any{"enc": "utf8", "data": "keep-me"},
"delay_ms": int64(0),
"offline": map[string]any{"keep": true, "ttl_seconds": int64(86400)},
})
if !sendOff.OK {
t.Fatalf("send offline keep: %+v", sendOff)
}
if _, err := c.AddResetPeer(q3ProxyName, "q3-rst", 0, ""); err != nil {
t.Fatalf("AddResetPeer: %v", err)
}
time.Sleep(400 * time.Millisecond)
_ = c.RemoveToxic(q3ProxyName, "q3-rst")
_ = c.RemoveToxic(q3ProxyName, "q3-lag")
bob := accept.MQTTLogin(t, proxyBase, "q3bob0001", epPassword)
defer bob.Close()
msg := bob.WaitType(t, "msg", 20*time.Second)
if msg["id"] != "q3-off-keep" {
t.Fatalf("want q3-off-keep got %v", msg)
}
bob.Request(t, map[string]any{"v": 1, "type": "ack", "rid": "q3a1", "from": "q3alice01", "id": "q3-off-keep"})
}
// TestQ3NetemUntested 记录本机 Windows 上 netem 20% 丢包未测原因。
func TestQ3NetemUntested(t *testing.T) {
t.Log("未测：Linux netem 20% 丢包。本机为 Windows；无法在宿主直接用 tc/netem。" +
"若只对 toxiproxy 容器网卡挂 netshoot，不等于 NixMsg 业务端到端丢包验收。" +
"本波用 toxiproxy 延迟/断开覆盖弱网；netem 需 Linux 宿主或把服务放进同网络 Linux 容器后再测。")
}
func moduleRoot(t *testing.T) string {
t.Helper()
dir, err := os.Getwd()
if err != nil {
t.Fatal(err)
}
for {
if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
return dir
}
parent := filepath.Dir(dir)
if parent == dir {
t.Fatal("go.mod not found")
}
dir = parent
}
}
func waitComposePort(t *testing.T, composeFile, containerPort string, timeout time.Duration) string {
t.Helper()
deadline := time.Now().Add(timeout)
var last string
for time.Now().Before(deadline) {
cmd := exec.Command("docker", "compose", "-p", q3ComposeProject, "-f", composeFile, "port", "toxiproxy", containerPort)
out, err := cmd.CombinedOutput()
last = strings.TrimSpace(string(out))
if err == nil && last != "" {
_, port, perr := net.SplitHostPort(normalizeComposeAddr(last))
if perr == nil && port != "" {
return port
}
}
time.Sleep(400 * time.Millisecond)
}
t.Fatalf("compose port %s timeout, last=%q", containerPort, last)
return ""
}
func normalizeComposeAddr(s string) string {
s = strings.TrimSpace(s)
if strings.HasPrefix(s, "[::]") {
return "127.0.0.1" + s[len("[::]"):]
}
if strings.HasPrefix(s, "0.0.0.0:") {
return "127.0.0.1" + s[len("0.0.0.0"):]
}
return s
}
func waitHTTP(t *testing.T, url string, timeout time.Duration) {
t.Helper()
client := &http.Client{Timeout: 2 * time.Second}
deadline := time.Now().Add(timeout)
var last error
for time.Now().Before(deadline) {
resp, err := client.Get(url)
if err == nil {
_ = resp.Body.Close()
if resp.StatusCode < 500 {
return
}
last = err
} else {
last = err
}
time.Sleep(400 * time.Millisecond)
}
t.Fatalf("wait HTTP %s: %v", url, last)
}