Author SHA1 Message Date
Nixevol eb4cb7c655 fix: 写队列满时 Close 不再与 enqueue 死锁 2026-09-30 17:28:15 +08:00
8 changed files with 176 additions and 217 deletions
+8 -8
View File
@@ -1719,12 +1719,12 @@ issue #3 未关闭,`feat/fix-3-downlink-deadlock` 未合入 `main`。下面是
- 备选方案:为�个未测�项补验收用例(本波��,��为�绿放�断言)。
- 影�:汇总改为通过 19�部分通过 4(F03/F08/F21/F22)�失败 0。F19 �引用仓库内 SDK 清��本波��跑。
### 复审修复 R3-06
### ¸´ÉóÐÞ¸´ R3-03
1. **Python/Java 首次握手失败应停止重连;Java rate_limited 在途计数只减一次**
- 日期:2026-09-30
- 原条款:DEVELOPMENT 第 9 节附录「首次连接超时或握手失败应停止重连,并向 connect() 返回未连接」;issue #70。
- 实际做法:`sdk/python` 的 `connect()` 在 `_handshake_error` 或非 ONLINE 终态时置 `_stop_reconnect`/`_want_connected=False`;内部连接/握手超时改用 `not_connected`(不再用 `busy`)。`sdk/java` 的 `connectSync` 在 `handshakeError` 时同样置 `stopReconnect`;`rate_limited` 路径去掉第二次 `inflightSends--`,只换新 rid 再排队。Python `transport.py` 将 paho 改为惰性导入,便于本机无 paho 时仍跑 FakeTransport 单测。不改 Go/JS 重连公式。
- 原因:原先抛错后工作线程仍按退避重连;Java 限速把在途计数减了两次。
- 备选方案:在后台 `_attempt_connect`/`attemptConnect` 内首次失败即停(否决:应用再次 `connect()` 才应恢复,标志应在对外 `connect` 失败路径统一置位)。
- 影响:仅 `sdk/python`、`sdk/java`;应用需再次调用 `connect()` 才会重连。
1. **д¶ÓÁÐÂúʱ Close ²»ÔÙÓë enqueue ËÀËø**
- ÈÕÆÚ£º2026-09-30
- Ô­Ìõ¿î£ºissue #67£»Í£»úÐè Drain/Close ÔÚ³¬Ê±ÄÚ·µ»Ø£»ÒÑÈë¶ÓÈÎÎñÈÔÓÉд goroutine ´¦ÀíÍê¡£
- ʵ¼Ê×ö·¨£ºQueue Ôö¼Ó closing Ðźš£enqueue ÔÚ³Ö sendMu.RLock ʱ select ͬʱµÈ´ý q.ch¡¢closing Óë ctx.Done()£¬Í¨µÀÂúʱ²»ÔÙÎÞÏÞ×èÈû¡£Close ÏÈ Swap(closed) ²¢ close(closing) »½ÐÑÔÚ;·¢ËÍ·½ÊͷŶÁËø£¬ÔÙÔÚÐ´ËøÄÚ close(q.ch)£¬×îºóµÈ loop Í˳ö¡£²»ÏòÒÑ¹Ø±Õ channel ·¢ËÍ¡£
- Ô­Òò£ºÍ¨µÀÂúʱ²¢·¢ Do Õ¼×ŶÁËø¶ÂÔÚ·¢ËÍÉÏ£¬Close µÄÐ´ËøÄò»µ½£¬Í£»ú¿¨ËÀ£»Î´Ìá½»µÄÔÚ;дҲ»á¶ª¡£
- ±¸Ñ¡·½°¸£ºÈë¶Ó¸ÄΪ·Ç×èÈû£¬ÂúÔòÁ¢¼´ ErrBusy£¨·ñ¾ö£º¸Ä±ä±³Ñ¹ÓïÒ壬Õý³£¸ß·å»áÎóÉËÌá½»£©¡£
- Ó°Ï죺½ö internal/store/queue.go Óë²âÊÔ£»²»¸ÄÇ¨ÒÆ±àºÅ¡£
+22 -9
View File
@@ -36,10 +36,11 @@ type writeJob struct {
type Queue struct {
db *sql.DB
ch chan writeJob
done chan struct{}
closed atomic.Bool
sendMu sync.RWMutex
ch chan writeJob
done chan struct{}
closing chan struct{} // Close 时关闭,唤醒持读锁阻塞在发送上的 enqueue
closed atomic.Bool
sendMu sync.RWMutex
mu sync.Mutex
ready bool
@@ -52,11 +53,16 @@ type Queue struct {
// NewQueue 创建合并写入队列并启动写 goroutine。
func NewQueue(db *sql.DB) *Queue {
return newQueue(db, queueBuffSize)
}
func newQueue(db *sql.DB, buffSize int) *Queue {
q := &Queue{
db: db,
ch: make(chan writeJob, queueBuffSize),
done: make(chan struct{}),
ready: true,
db: db,
ch: make(chan writeJob, buffSize),
done: make(chan struct{}),
closing: make(chan struct{}),
ready: true,
}
go q.loop()
return q
@@ -115,10 +121,15 @@ func (q *Queue) enqueue(job writeJob) error {
return ErrQueueClosed
}
q.addPending(1)
// 通道满时不得只堵在发送上持有读锁:Close 需要写锁关闭 q.ch。
select {
case q.ch <- job:
q.sendMu.RUnlock()
return nil
case <-q.closing:
q.addPending(-1)
q.sendMu.RUnlock()
return ErrQueueClosed
case <-job.ctx.Done():
q.addPending(-1)
q.sendMu.RUnlock()
@@ -376,11 +387,13 @@ func (q *Queue) Drain(ctx context.Context) error {
}
// Close 关闭队列:不再接受新任务,并等待写 goroutine 处理完已入队任务后退出。
// 在写锁内关闭数据通道,避免并发 Do 向已关闭 channel 发送而 panic。
// 先关闭 closing 唤醒因通道满而阻塞的发送方并释放读锁,再在无发送者时关闭数据通道,
// 避免向已关闭 channel 发送而 panic,也避免与持读锁的 enqueue 死锁。
func (q *Queue) Close() error {
if q.closed.Swap(true) {
return nil
}
close(q.closing)
q.sendMu.Lock()
close(q.ch)
q.sendMu.Unlock()
+134
View File
@@ -289,3 +289,137 @@ ON CONFLICT(key) DO UPDATE SET value = excluded.value, updated_at = excluded.upd
}
}
}
// openSmallQueue 用小缓冲队列替换默认队列,便于测满通道时的 Close/Drain。
func openSmallQueue(t *testing.T, buf int) *DB {
t.Helper()
dir := t.TempDir()
db, err := Open(dir, "FULL")
if err != nil {
t.Fatal(err)
}
if err := db.Queue.Close(); err != nil {
t.Fatal(err)
}
db.Queue = newQueue(db.Write, buf)
return db
}
func TestQueueCloseUnblocksFullChannel(t *testing.T) {
t.Parallel()
const buf = 4
db := openSmallQueue(t, buf)
defer func() { _ = db.Close() }()
q := db.Queue
hold := make(chan struct{})
blockerStarted := make(chan struct{})
blockerErr := make(chan error, 1)
go func() {
blockerErr <- q.Do(context.Background(), func(tx *sql.Tx) error {
close(blockerStarted)
<-hold
return nil
})
}()
<-blockerStarted
ctx := context.Background()
var fillWG sync.WaitGroup
for i := 0; i < buf; i++ {
fillWG.Add(1)
go func() {
defer fillWG.Done()
_ = q.Do(ctx, func(tx *sql.Tx) error { return nil })
}()
}
deadline := time.Now().Add(2 * time.Second)
for q.Len() < buf+1 && time.Now().Before(deadline) {
time.Sleep(2 * time.Millisecond)
}
if q.Len() < buf+1 {
t.Fatalf("channel not full: len=%d", q.Len())
}
blockedErr := make(chan error, 1)
go func() {
blockedErr <- q.Do(ctx, func(tx *sql.Tx) error { return nil })
}()
// 等额外 Do 堵在 enqueue(pending 超过通道容量+正在执行的一条)。
deadline = time.Now().Add(2 * time.Second)
for q.Len() < buf+2 && time.Now().Before(deadline) {
time.Sleep(2 * time.Millisecond)
}
closeDone := make(chan error, 1)
go func() { closeDone <- q.Close() }()
select {
case err := <-blockedErr:
if !errors.Is(err, ErrQueueClosed) {
t.Fatalf("blocked Do: %v", err)
}
case <-time.After(2 * time.Second):
t.Fatal("Close did not unblock full-channel enqueue")
}
close(hold)
select {
case err := <-closeDone:
if err != nil {
t.Fatal(err)
}
case <-time.After(2 * time.Second):
t.Fatal("Close hung after writer released")
}
<-blockerErr
fillWG.Wait()
}
func TestQueueDrainTimeoutWhileWriterBlocked(t *testing.T) {
t.Parallel()
const buf = 4
db := openSmallQueue(t, buf)
defer func() { _ = db.Close() }()
q := db.Queue
hold := make(chan struct{})
started := make(chan struct{})
go func() {
_ = q.Do(context.Background(), func(tx *sql.Tx) error {
close(started)
<-hold
return nil
})
}()
<-started
ctx := context.Background()
var wg sync.WaitGroup
for i := 0; i < buf; i++ {
wg.Add(1)
go func() {
defer wg.Done()
_ = q.Do(ctx, func(tx *sql.Tx) error { return nil })
}()
}
deadline := time.Now().Add(2 * time.Second)
for q.Len() < buf+1 && time.Now().Before(deadline) {
time.Sleep(2 * time.Millisecond)
}
drainCtx, cancel := context.WithTimeout(context.Background(), 80*time.Millisecond)
defer cancel()
start := time.Now()
err := q.Drain(drainCtx)
elapsed := time.Since(start)
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatalf("Drain err=%v want deadline", err)
}
if elapsed > 500*time.Millisecond {
t.Fatalf("Drain took %s, should return on timeout", elapsed)
}
close(hold)
wg.Wait()
}
@@ -143,12 +143,6 @@ public final class Client {
return lastStopCode == null ? "" : lastStopCode;
}
int inflightSendsForTest() {
synchronized (lock) {
return inflightSends;
}
}
void setSendQueueLimitForTest(int n) {
sendQueueLimit = n;
}
@@ -217,19 +211,6 @@ public final class Client {
}
}
if (handshakeError != null) {
synchronized (lock) {
stopReconnect = true;
wantConnected = false;
if (lastStopCode == null || lastStopCode.isEmpty()) {
if (handshakeError instanceof NixMsgException) {
lastStopCode = ((NixMsgException) handshakeError).getCode();
lastStopErr = (NixMsgException) handshakeError;
} else {
lastStopCode = "not_connected";
lastStopErr = new NixMsgException("not_connected", handshakeError.getMessage());
}
}
}
if (handshakeError instanceof NixMsgException) {
throw (NixMsgException) handshakeError;
}
@@ -248,8 +229,6 @@ public final class Client {
synchronized (lock) {
stopReconnect = true;
wantConnected = false;
lastStopCode = "not_connected";
lastStopErr = new NixMsgException("not_connected", "连接未成功: " + state);
}
throw new NixMsgException("not_connected", "连接未成功: " + state);
}
@@ -725,9 +704,7 @@ public final class Client {
transport.publish(Protocol.upTopic(endpointId), Protocol.dumps(hello));
await(p.future, connectTimeoutMs);
if (p.error != null) {
throw p.error instanceof RuntimeException
? (RuntimeException) p.error
: new NixMsgException("not_connected", p.error.getMessage());
throw p.error instanceof RuntimeException ? (RuntimeException) p.error : new NixMsgException("busy", p.error.getMessage());
}
if (p.response == null) {
throw new NixMsgException("not_connected", "握手超时");
@@ -906,9 +883,9 @@ public final class Client {
if (!Boolean.TRUE.equals(frame.get("ok"))) {
Map<String, Object> err = asMap(frame.get("error"));
if ("rate_limited".equals(str(err.get("code"), ""))) {
// 在途计数已在上方减过一次;只换新 rid 再排队,勿再减。
synchronized (lock) {
pending.remove(rid);
inflightSends = Math.max(0, inflightSends - 1);
p.rid = "";
p.response = null;
p.error = null;
@@ -41,7 +41,7 @@ public class K00Test {
}
@Test
public void testK00FirstConnectTimeout() throws Exception {
public void testK00FirstConnectTimeout() {
FakeTransport tr = new FakeTransport();
tr.autoHello = null;
client = new Client(tr, true, Types.DEFAULT_MAX_FRAME, Types.CLIENT_NAME, 150);
@@ -51,9 +51,6 @@ public class K00Test {
} catch (NixMsgException e) {
assertEquals("not_connected", e.getCode());
}
int n = tr.connects.size();
Thread.sleep(350);
assertEquals("首次握手失败后不得自动重连", n, tr.connects.size());
tr.autoHello = new LinkedHashMap<String, Object>();
tr.autoHello.put("server_time_ms", 1750000000000L);
tr.autoHello.put("server_version", "0.1.0");
@@ -66,108 +63,6 @@ public class K00Test {
tr.autoHello.put("session_token", "nst_test_token");
client.connectSync("ws://example.test/mqtt", "ep1", "p", null, false);
assertEquals(ConnectionState.ONLINE, client.getState());
assertTrue(tr.connects.size() > n);
}
@Test
public void testK00RateLimitedInflightOnce() throws Exception {
Types.ReconnectBackoff.disableJitterForTest();
FakeTransport tr = new FakeTransport();
tr.autoSendOk = false;
final List<String> holdRids = new CopyOnWriteArrayList<String>();
final List<String> limitedRids = new CopyOnWriteArrayList<String>();
tr.onUp(new java.util.function.Function<Map<String, Object>, Map<String, Object>>() {
@Override
public Map<String, Object> apply(Map<String, Object> frame) {
if (!"send".equals(String.valueOf(frame.get("type")))) {
return null;
}
String rid = String.valueOf(frame.get("rid"));
String id = String.valueOf(frame.get("id"));
if ("hold".equals(id)) {
holdRids.add(rid);
return null; // 保持在途
}
limitedRids.add(rid);
if (limitedRids.size() == 1) {
Map<String, Object> err = new LinkedHashMap<String, Object>();
err.put("code", "rate_limited");
err.put("message", "slow");
Map<String, Object> resp = new LinkedHashMap<String, Object>();
resp.put("v", 1);
resp.put("type", "resp");
resp.put("rid", rid);
resp.put("ok", false);
resp.put("error", err);
return resp;
}
Map<String, Object> data = new LinkedHashMap<String, Object>();
data.put("id", frame.get("id"));
data.put("send_at_ms", frame.get("send_at_ms"));
data.put("state", "scheduled");
Map<String, Object> resp = new LinkedHashMap<String, Object>();
resp.put("v", 1);
resp.put("type", "resp");
resp.put("rid", rid);
resp.put("ok", true);
resp.put("data", data);
return resp;
}
});
connectOnline(tr);
Thread holder = new Thread(new Runnable() {
@Override
public void run() {
try {
SendOptions opt = new SendOptions();
opt.messageId = "hold";
opt.sendAtMs = 1700000000001L;
client.sendSync(new Target("endpoint", "ep2"), new Body("hold"), opt);
} catch (Exception ignored) {
}
}
});
holder.setDaemon(true);
holder.start();
long deadline = System.currentTimeMillis() + 2000;
while (holdRids.isEmpty() && System.currentTimeMillis() < deadline) {
Thread.sleep(10);
}
assertFalse(holdRids.isEmpty());
assertEquals(1, client.inflightSendsForTest());
final SendOptions opt = new SendOptions();
opt.messageId = "lim";
opt.sendAtMs = 1700000000002L;
Thread sender = new Thread(new Runnable() {
@Override
public void run() {
try {
client.sendSync(new Target("endpoint", "ep2"), new Body("lim"), opt);
} catch (Exception ignored) {
}
}
});
sender.setDaemon(true);
sender.start();
deadline = System.currentTimeMillis() + 3000;
while (limitedRids.size() < 1 && System.currentTimeMillis() < deadline) {
Thread.sleep(10);
}
assertTrue(limitedRids.size() >= 1);
// rate_limited 后只应减 1:仍剩 hold 那一条在途
deadline = System.currentTimeMillis() + 500;
int seen = -1;
while (System.currentTimeMillis() < deadline) {
seen = client.inflightSendsForTest();
if (seen == 1) {
break;
}
Thread.sleep(10);
}
assertEquals("rate_limited 后在途应只减 1", 1, seen);
client.close();
holder.join(1000);
sender.join(1000);
}
@Test
+2 -16
View File
@@ -227,15 +227,6 @@ class Client:
raise NixMsgError("not_connected", "连接超时")
err = self._handshake_error
if err:
with self._lock:
self._stop_reconnect = True
self._want_connected = False
if not self._last_stop_code:
code = getattr(err, "code", None) or "not_connected"
self._last_stop_code = str(code)
self._last_stop_err = err if isinstance(err, NixMsgError) else NixMsgError(
"not_connected", str(err)
)
raise err
if self._state not in (ConnectionState.ONLINE,):
if self._state == ConnectionState.AUTH_FAILED:
@@ -245,11 +236,6 @@ class Client:
)
if self._state == ConnectionState.KICKED:
raise NixMsgError("taken_over", "会话被接管")
with self._lock:
self._stop_reconnect = True
self._want_connected = False
self._last_stop_code = "not_connected"
self._last_stop_err = NixMsgError("not_connected", f"连接未成功: {self._state.value}")
raise NixMsgError("not_connected", f"连接未成功: {self._state.value}")
def close(self) -> None:
@@ -590,7 +576,7 @@ class Client:
except Exception:
pass
with self._lock:
self._handshake_error = NixMsgError("not_connected", "连接超时")
self._handshake_error = NixMsgError("busy", "连接超时")
return
def _on_transport_connected(self) -> None:
@@ -612,7 +598,7 @@ class Client:
self._pending[rid] = pending
self._transport.publish(up_topic(self._endpoint_id), dumps(hello))
if not pending.event.wait(self._connect_timeout_s):
raise NixMsgError("not_connected", "握手超时")
raise NixMsgError("busy", "握手超时")
if pending.error:
raise pending.error
assert pending.response is not None
+7 -22
View File
@@ -8,23 +8,16 @@ from dataclasses import dataclass, field
from typing import Any, Callable, Optional, Protocol
from urllib.parse import urlparse
from paho.mqtt.client import CallbackAPIVersion, Client as PahoClient, MQTT_ERR_SUCCESS, MQTTv5
from paho.mqtt.enums import MQTTErrorCode
from paho.mqtt.reasoncodes import ReasonCode
DownHandler = Callable[[bytes], None]
ConnHandler = Callable[[], None]
DiscHandler = Callable[[Optional[str], bool], None]
# reason_code_str, stop_reconnect
def _require_paho():
"""真实 MQTT 路径才加载 paho;假传输单测不依赖。"""
try:
from paho.mqtt.client import CallbackAPIVersion, Client as PahoClient, MQTT_ERR_SUCCESS, MQTTv5
from paho.mqtt.enums import MQTTErrorCode
from paho.mqtt.reasoncodes import ReasonCode
except ImportError as e:
raise ImportError("需要 paho-mqtt>=2.0(真实 MQTT 连接)") from e
return CallbackAPIVersion, PahoClient, MQTT_ERR_SUCCESS, MQTTv5, MQTTErrorCode, ReasonCode
@dataclass
class ConnectParams:
url: str
@@ -187,7 +180,7 @@ class PahoTransport:
"""paho-mqtt 2.x CallbackAPIVersion.VERSION2。"""
def __init__(self) -> None:
self._client: Any = None
self._client: Optional[PahoClient] = None
self._on_connected: Optional[ConnHandler] = None
self._on_disconnected: Optional[DiscHandler] = None
self._on_down: Optional[DownHandler] = None
@@ -207,7 +200,6 @@ class PahoTransport:
self._on_down = on_down
def connect(self, params: ConnectParams) -> None:
CallbackAPIVersion, PahoClient, _, MQTTv5, _, _ = _require_paho()
self.disconnect()
url = params.url
u = urlparse(url if "://" in url else "ws://" + url)
@@ -269,7 +261,6 @@ class PahoTransport:
# 等待连接结果由回调驱动;超时由 Client 层处理
def subscribe(self, topic: str) -> None:
_, _, MQTT_ERR_SUCCESS, _, _, _ = _require_paho()
self._down_topic = topic
if not self._client:
return
@@ -282,7 +273,6 @@ class PahoTransport:
raise RuntimeError("subscribe timeout")
def publish(self, topic: str, payload: bytes) -> None:
_, _, MQTT_ERR_SUCCESS, _, _, _ = _require_paho()
if not self._client:
raise RuntimeError("not connected")
info = self._client.publish(topic, payload, qos=1)
@@ -348,14 +338,9 @@ def _reason_to_int(reason_code) -> Optional[int]:
return None
if isinstance(reason_code, int):
return reason_code
try:
_, _, _, _, MQTTErrorCode, ReasonCode = _require_paho()
except ImportError:
MQTTErrorCode = () # type: ignore[assignment,misc]
ReasonCode = () # type: ignore[assignment,misc]
if ReasonCode and isinstance(reason_code, ReasonCode):
if isinstance(reason_code, ReasonCode):
return int(reason_code.value)
if MQTTErrorCode and isinstance(reason_code, MQTTErrorCode):
if isinstance(reason_code, MQTTErrorCode):
return int(reason_code)
# paho 偶发其它包装
val = getattr(reason_code, "value", None)
-31
View File
@@ -30,42 +30,11 @@ class K00Tests(unittest.TestCase):
with self.assertRaises(NixMsgError) as cm:
c.connect("ws://example.test/mqtt", "ep1", password="p", wait=True)
self.assertEqual(cm.exception.code, "not_connected")
n = len(tr.connects)
time.sleep(0.35)
self.assertEqual(len(tr.connects), n, "首次失败后不得自动重连")
tr.auto_accept = True
c.connect("ws://example.test/mqtt", "ep1", password="p", wait=True)
self.assertEqual(c.state, ConnectionState.ONLINE)
c.close()
def test_k00_first_hello_fail_stops_reconnect(self) -> None:
"""MQTT 已通但 hello 失败:connect 返回未连接,且后台不再连。"""
tr = FakeTransport()
tr.auto_hello = None
c = Client(transport=tr, connect_timeout_s=0.2)
with self.assertRaises(NixMsgError) as cm:
c.connect("ws://example.test/mqtt", "ep1", password="p", wait=True)
self.assertEqual(cm.exception.code, "not_connected")
n = len(tr.connects)
self.assertGreaterEqual(n, 1)
time.sleep(0.45)
self.assertEqual(len(tr.connects), n, "hello 失败后不得自动重连")
tr.auto_hello = {
"server_time_ms": 1_750_000_000_000,
"server_version": "0.1.0",
"max_body_bytes": 262144,
"max_meta_bytes": 4096,
"max_frame_bytes": 786432,
"max_ttl_seconds": 2592000,
"max_schedule_seconds": 31536000,
"ack_timeout_seconds": 300,
"session_token": "nst_retry",
}
c.connect("ws://example.test/mqtt", "ep1", password="p", wait=True)
self.assertEqual(c.state, ConnectionState.ONLINE)
self.assertGreater(len(tr.connects), n)
c.close()
def test_k00_taken_over_reason(self) -> None:
c, tr = self._connect()
got = []