feat: 测试交付 Q1 混沌压测骨架与 Q4 镜像骨架
This commit is contained in:
Nixevol
@@ -0,0 +1,61 @@
|
||||
# 混沌 / 弱网测试辅助（Q 线）
|
||||
|
||||
用 toxiproxy 官方镜像做延迟和断开；Linux 丢包用容器内 netem。所有 Docker 资源名带 `q` 前缀，用完删除。
|
||||
|
||||
## 启动 toxiproxy
|
||||
|
||||
在仓库根目录（或本目录）执行：
|
||||
|
||||
```powershell
|
||||
docker compose -p q-chaos -f test/chaos/compose.yml up -d
|
||||
```
|
||||
|
||||
API 默认映射到本机随机/固定端口见 compose 注释。查看实际端口：
|
||||
|
||||
```powershell
|
||||
docker compose -p q-chaos -f test/chaos/compose.yml port toxiproxy 8474
|
||||
```
|
||||
|
||||
清理：
|
||||
|
||||
```powershell
|
||||
docker compose -p q-chaos -f test/chaos/compose.yml down -v
|
||||
```
|
||||
|
||||
## 对运行中的服务注入故障
|
||||
|
||||
不要写死 `7443`。上游地址取自当前实例的实际监听端口（例如测试 harness 写入的 `listen.addr`，或你启动时打印的地址）。
|
||||
|
||||
1. 本机起好 NixMsg（或任意 TCP 服务），记下 `HOST:PORT`。
|
||||
2. 在 Windows 上，容器访问本机服务用 `host.docker.internal:PORT`。
|
||||
3. 用本包客户端或 curl 建代理（listen 用 `0.0.0.0:0` 让 toxiproxy 分配端口）：
|
||||
|
||||
```go
|
||||
c := chaos.NewClient("http://127.0.0.1:<api端口>")
|
||||
p, err := c.CreateProxy("q-nixmsg", "0.0.0.0:0", "host.docker.internal:"+strconv.Itoa(port))
|
||||
// 客户端改连 p.Listen（把 0.0.0.0 换成 127.0.0.1）
|
||||
_, _ = c.AddLatency("q-nixmsg", "lag", 200, 50, "")
|
||||
_, _ = c.AddResetPeer("q-nixmsg", "rst", 0, "")
|
||||
```
|
||||
|
||||
等价 curl：
|
||||
|
||||
```bash
|
||||
curl -s -X POST http://127.0.0.1:<api>/proxies \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"name":"q-nixmsg","listen":"0.0.0.0:0","upstream":"host.docker.internal:<PORT>","enabled":true}'
|
||||
```
|
||||
|
||||
延迟：`POST .../proxies/q-nixmsg/toxics`，`type=latency`，`attributes.latency` / `jitter`（毫秒）。
|
||||
|
||||
断开：`type=reset_peer`，`attributes.timeout`（毫秒，0 表示尽快 RST）。
|
||||
|
||||
## 单元测试
|
||||
|
||||
```powershell
|
||||
go test ./test/chaos/ -count=1
|
||||
```
|
||||
|
||||
## netem 丢包
|
||||
|
||||
见 [netem/README.md](./netem/README.md)。脚本必须在 Linux 容器内执行（本机是 Windows）。
|
||||
@@ -0,0 +1,15 @@
|
||||
# toxiproxy：延迟与断开。项目名用 -p q-chaos，容器名带 q 前缀。
|
||||
# API 与代理端口映射到本机，具体宿主机端口用 `docker compose port` 查询，不要写死业务端口。
|
||||
|
||||
services:
|
||||
toxiproxy:
|
||||
image: ghcr.io/shopify/toxiproxy:2.12.0
|
||||
container_name: q-toxiproxy
|
||||
# 8474=API；8475 起可手工映射代理端口，或在 CreateProxy 时用 0.0.0.0:0 再 docker port 查询
|
||||
ports:
|
||||
- "8474"
|
||||
- "8475"
|
||||
- "8476"
|
||||
# 允许代理到本机服务（Docker Desktop / Windows）
|
||||
extra_hosts:
|
||||
- "host.docker.internal:host-gateway"
|
||||
@@ -0,0 +1,192 @@
|
||||
// Package chaos 提供基于 toxiproxy 的弱网故障注入辅助。
|
||||
// 上游地址由调用方传入，不写死端口。
|
||||
package chaos
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Client 调用 toxiproxy HTTP API。
|
||||
type Client struct {
|
||||
BaseURL string
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// NewClient 创建客户端。baseURL 形如 http://127.0.0.1:8474。
|
||||
func NewClient(baseURL string) *Client {
|
||||
return &Client{
|
||||
BaseURL: strings.TrimRight(baseURL, "/"),
|
||||
HTTPClient: &http.Client{
|
||||
Timeout: 10 * time.Second,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Proxy 描述一个 toxiproxy 代理。
|
||||
type Proxy struct {
|
||||
Name string `json:"name"`
|
||||
Listen string `json:"listen"`
|
||||
Upstream string `json:"upstream"`
|
||||
Enabled bool `json:"enabled"`
|
||||
}
|
||||
|
||||
// Toxic 描述一条故障规则。
|
||||
type Toxic struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Stream string `json:"stream,omitempty"`
|
||||
Toxicity float32 `json:"toxicity,omitempty"`
|
||||
Attributes map[string]any `json:"attributes,omitempty"`
|
||||
}
|
||||
|
||||
// CreateProxy 创建或覆盖同名代理。listen 可用 host:0 让 toxiproxy 分配端口。
|
||||
func (c *Client) CreateProxy(name, listen, upstream string) (*Proxy, error) {
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("proxy name required")
|
||||
}
|
||||
if listen == "" {
|
||||
return nil, fmt.Errorf("listen required")
|
||||
}
|
||||
if upstream == "" {
|
||||
return nil, fmt.Errorf("upstream required")
|
||||
}
|
||||
body := Proxy{
|
||||
Name: name,
|
||||
Listen: listen,
|
||||
Upstream: upstream,
|
||||
Enabled: true,
|
||||
}
|
||||
var out Proxy
|
||||
if err := c.doJSON(http.MethodPost, "/proxies", body, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// DeleteProxy 删除代理；不存在时忽略。
|
||||
func (c *Client) DeleteProxy(name string) error {
|
||||
req, err := http.NewRequest(http.MethodDelete, c.BaseURL+"/proxies/"+name, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("delete proxy: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
// GetProxy 读取代理（含实际 listen 地址）。
|
||||
func (c *Client) GetProxy(name string) (*Proxy, error) {
|
||||
var out Proxy
|
||||
if err := c.doJSON(http.MethodGet, "/proxies/"+name, nil, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
// AddLatency 注入下行/上行延迟（毫秒）。stream 为空时默认 downstream。
|
||||
func (c *Client) AddLatency(proxyName, toxicName string, latencyMs, jitterMs int, stream string) (*Toxic, error) {
|
||||
if stream == "" {
|
||||
stream = "downstream"
|
||||
}
|
||||
t := Toxic{
|
||||
Name: toxicName,
|
||||
Type: "latency",
|
||||
Stream: stream,
|
||||
Toxicity: 1,
|
||||
Attributes: map[string]any{
|
||||
"latency": latencyMs,
|
||||
"jitter": jitterMs,
|
||||
},
|
||||
}
|
||||
return c.addToxic(proxyName, t)
|
||||
}
|
||||
|
||||
// AddResetPeer 在连接上注入 TCP RST（断开）。timeoutMs 为触发前等待。
|
||||
func (c *Client) AddResetPeer(proxyName, toxicName string, timeoutMs int, stream string) (*Toxic, error) {
|
||||
if stream == "" {
|
||||
stream = "downstream"
|
||||
}
|
||||
t := Toxic{
|
||||
Name: toxicName,
|
||||
Type: "reset_peer",
|
||||
Stream: stream,
|
||||
Toxicity: 1,
|
||||
Attributes: map[string]any{
|
||||
"timeout": timeoutMs,
|
||||
},
|
||||
}
|
||||
return c.addToxic(proxyName, t)
|
||||
}
|
||||
|
||||
// RemoveToxic 删除一条 toxic。
|
||||
func (c *Client) RemoveToxic(proxyName, toxicName string) error {
|
||||
req, err := http.NewRequest(http.MethodDelete, c.BaseURL+"/proxies/"+proxyName+"/toxics/"+toxicName, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
if resp.StatusCode == http.StatusNotFound || resp.StatusCode == http.StatusNoContent || resp.StatusCode == http.StatusOK {
|
||||
return nil
|
||||
}
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
return fmt.Errorf("remove toxic: %s: %s", resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
|
||||
func (c *Client) addToxic(proxyName string, t Toxic) (*Toxic, error) {
|
||||
var out Toxic
|
||||
if err := c.doJSON(http.MethodPost, "/proxies/"+proxyName+"/toxics", t, &out); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &out, nil
|
||||
}
|
||||
|
||||
func (c *Client) doJSON(method, path string, in any, out any) error {
|
||||
var body io.Reader
|
||||
if in != nil {
|
||||
b, err := json.Marshal(in)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
body = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequest(method, c.BaseURL+path, body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if in != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
resp, err := c.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = resp.Body.Close() }()
|
||||
respBody, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("%s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(respBody)))
|
||||
}
|
||||
if out == nil || len(respBody) == 0 {
|
||||
return nil
|
||||
}
|
||||
return json.Unmarshal(respBody, out)
|
||||
}
|
||||
@@ -0,0 +1,113 @@
|
||||
package chaos
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestClientCreateLatencyAndReset(t *testing.T) {
|
||||
t.Parallel()
|
||||
|
||||
toxics := map[string][]Toxic{}
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch {
|
||||
case r.Method == http.MethodPost && r.URL.Path == "/proxies":
|
||||
var p Proxy
|
||||
if err := json.NewDecoder(r.Body).Decode(&p); err != nil {
|
||||
t.Errorf("decode proxy: %v", err)
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
if p.Listen == "0.0.0.0:0" {
|
||||
p.Listen = "0.0.0.0:18080"
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(p)
|
||||
|
||||
case r.Method == http.MethodPost && strings.HasSuffix(r.URL.Path, "/toxics"):
|
||||
var toxic Toxic
|
||||
if err := json.NewDecoder(r.Body).Decode(&toxic); err != nil {
|
||||
http.Error(w, err.Error(), 400)
|
||||
return
|
||||
}
|
||||
name := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/proxies/"), "/toxics")
|
||||
toxics[name] = append(toxics[name], toxic)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_ = json.NewEncoder(w).Encode(toxic)
|
||||
|
||||
case r.Method == http.MethodDelete:
|
||||
w.WriteHeader(http.StatusNoContent)
|
||||
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
|
||||
c := NewClient(srv.URL)
|
||||
p, err := c.CreateProxy("q-demo", "0.0.0.0:0", "127.0.0.1:19000")
|
||||
if err != nil {
|
||||
t.Fatalf("CreateProxy: %v", err)
|
||||
}
|
||||
if p.Upstream != "127.0.0.1:19000" {
|
||||
t.Fatalf("upstream = %q", p.Upstream)
|
||||
}
|
||||
if p.Listen != "0.0.0.0:18080" {
|
||||
t.Fatalf("listen = %q", p.Listen)
|
||||
}
|
||||
|
||||
lat, err := c.AddLatency("q-demo", "lag", 200, 50, "")
|
||||
if err != nil {
|
||||
t.Fatalf("AddLatency: %v", err)
|
||||
}
|
||||
if lat.Type != "latency" {
|
||||
t.Fatalf("type = %q", lat.Type)
|
||||
}
|
||||
if _, err := c.AddResetPeer("q-demo", "drop", 0, ""); err != nil {
|
||||
t.Fatalf("AddResetPeer: %v", err)
|
||||
}
|
||||
if got := toxics["q-demo"]; len(got) != 2 {
|
||||
t.Fatalf("toxics len = %d", len(got))
|
||||
}
|
||||
if got := toxics["q-demo"][0]; got.Type != "latency" {
|
||||
t.Fatalf("first toxic type = %q", got.Type)
|
||||
}
|
||||
if err := c.RemoveToxic("q-demo", "lag"); err != nil {
|
||||
t.Fatalf("RemoveToxic: %v", err)
|
||||
}
|
||||
if err := c.DeleteProxy("q-demo"); err != nil {
|
||||
t.Fatalf("DeleteProxy: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestClientRejectsEmptyFields(t *testing.T) {
|
||||
t.Parallel()
|
||||
c := NewClient("http://127.0.0.1:1")
|
||||
if _, err := c.CreateProxy("", "0.0.0.0:0", "127.0.0.1:1"); err == nil {
|
||||
t.Fatal("expected error for empty name")
|
||||
}
|
||||
if _, err := c.CreateProxy("x", "", "127.0.0.1:1"); err == nil {
|
||||
t.Fatal("expected error for empty listen")
|
||||
}
|
||||
if _, err := c.CreateProxy("x", "0.0.0.0:0", ""); err == nil {
|
||||
t.Fatal("expected error for empty upstream")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateProxyErrorStatus(t *testing.T) {
|
||||
t.Parallel()
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
w.WriteHeader(http.StatusConflict)
|
||||
_, _ = io.WriteString(w, "already exists")
|
||||
}))
|
||||
t.Cleanup(srv.Close)
|
||||
c := NewClient(srv.URL)
|
||||
if _, err := c.CreateProxy("q", "0.0.0.0:0", "127.0.0.1:9"); err == nil {
|
||||
t.Fatal("expected error")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
# Linux 容器内用 netem 做丢包（本机 Windows 不能直接跑 tc）
|
||||
|
||||
## 用法
|
||||
|
||||
对已经在同一网络里的目标容器网卡注入 20% 丢包（需要 `NET_ADMIN`）：
|
||||
|
||||
```powershell
|
||||
# 示例：起一个带 net-tools/iproute2 的旁路容器，对网卡 eth0 丢包
|
||||
docker run --rm --name q-netem --cap-add=NET_ADMIN --network container:q-toxiproxy `
|
||||
nicolaka/netshoot `
|
||||
bash -lc "tc qdisc replace dev eth0 root netem loss 20%"
|
||||
```
|
||||
|
||||
或把本目录脚本挂进去：
|
||||
|
||||
```powershell
|
||||
docker run --rm --name q-netem --cap-add=NET_ADMIN --network container:<目标容器名> `
|
||||
-v ${PWD}/test/chaos/netem:/scripts:ro `
|
||||
nicolaka/netshoot `
|
||||
bash /scripts/apply-loss.sh eth0 20
|
||||
```
|
||||
|
||||
清除：
|
||||
|
||||
```powershell
|
||||
docker run --rm --name q-netem-clear --cap-add=NET_ADMIN --network container:<目标容器名> `
|
||||
nicolaka/netshoot `
|
||||
bash -lc "tc qdisc del dev eth0 root 2>/dev/null || true"
|
||||
```
|
||||
|
||||
用完删除容器（上面 `--rm` 已自动删）。不要把 NixMsg 业务端口写死进脚本。
|
||||
@@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# 在 Linux 容器内执行：对指定网卡注入丢包。用法：apply-loss.sh <iface> <loss_percent>
|
||||
set -euo pipefail
|
||||
|
||||
IFACE="${1:-eth0}"
|
||||
LOSS="${2:-20}"
|
||||
|
||||
if ! command -v tc >/dev/null 2>&1; then
|
||||
echo "tc not found; use an image with iproute2 (e.g. nicolaka/netshoot)" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
tc qdisc replace dev "$IFACE" root netem loss "${LOSS}%"
|
||||
echo "netem: iface=${IFACE} loss=${LOSS}%"
|
||||
tc qdisc show dev "$IFACE"
|
||||
Reference in New Issue
Block a user