|
|
@@ -37,9 +37,47 @@ import {
|
|
|
CHANNEL_ID,
|
|
|
getChannelCfg,
|
|
|
normalizeAllowFrom,
|
|
|
+ resolveAllowFrom,
|
|
|
resolveApiBase,
|
|
|
+ resolveBatchWindowMs,
|
|
|
+ resolveDmPolicy,
|
|
|
+ resolveFaq,
|
|
|
+ resolveGroupEnabled,
|
|
|
+ resolveGroupKeywords,
|
|
|
+ resolveGroupMentionTokens,
|
|
|
+ resolveGroupReplyAtSender,
|
|
|
+ resolveIgnoreMessageTypes,
|
|
|
+ resolveIgnoreWxidExact,
|
|
|
+ resolveIgnoreWxidPrefixes,
|
|
|
+ resolveHighThroughputOutboundAttemptsPer5Min,
|
|
|
+ resolveMetricsLogIntervalSec,
|
|
|
+ resolveOutboundQueueMax,
|
|
|
+ resolveOutboundRatePerSec,
|
|
|
+ resolvePollIntervalMs,
|
|
|
+ resolveReplyCooldownSecPerWxid,
|
|
|
+ resolveSelfWxid,
|
|
|
} from "./config.js";
|
|
|
import type { Logger, RawWechatMessage, WechatAgentConfig } from "./types.js";
|
|
|
+import {
|
|
|
+ advanceCursor,
|
|
|
+ compareMsgId,
|
|
|
+ isAfterCursor,
|
|
|
+ loadCursor,
|
|
|
+ saveCursor,
|
|
|
+ toEpochMs,
|
|
|
+ type InboundCursor,
|
|
|
+} from "./cursor-store.js";
|
|
|
+import { join } from "node:path";
|
|
|
+import { withRetry } from "./retry.js";
|
|
|
+import { configureOutboundQueue, enqueueOutboundSend } from "./outbound-queue.js";
|
|
|
+import {
|
|
|
+ getOutboundMetricsSnapshot,
|
|
|
+ recordOutboundAttempt,
|
|
|
+ recordOutboundFailure,
|
|
|
+ recordOutboundSuccess,
|
|
|
+ shouldEmitOutboundFailureAlert,
|
|
|
+ shouldSkipDuplicateSend,
|
|
|
+} from "./outbound-observability.js";
|
|
|
|
|
|
export type MonitorOpts = {
|
|
|
cfg: OpenClawConfig;
|
|
|
@@ -62,6 +100,10 @@ const SEEN_MAX = 2000;
|
|
|
const SEEN_TRIM_TO = 1000;
|
|
|
const DEFAULT_GROUP_KEYWORDS = ["帮我", "请问"];
|
|
|
const DEFAULT_GROUP_MENTION_TOKENS = ["@bot", "@助手"];
|
|
|
+const POLL_ALERT_MIN_TICKS = 10;
|
|
|
+const POLL_ALERT_FAIL_RATE = 0.5;
|
|
|
+const HIGH_THROUGHPUT_LOG_INTERVAL_MS = 300 * 1000;
|
|
|
+const THROUGHPUT_WINDOW_MS = 5 * 60 * 1000;
|
|
|
|
|
|
export async function startMonitor(opts: MonitorOpts): Promise<MonitorResult> {
|
|
|
const channelCfg = getChannelCfg(opts.cfg);
|
|
|
@@ -84,13 +126,35 @@ export async function startMonitor(opts: MonitorOpts): Promise<MonitorResult> {
|
|
|
const client = createClient(apiBase, channelCfg.appId);
|
|
|
const pollIntervalMs = Math.max(
|
|
|
POLL_MIN_INTERVAL_MS,
|
|
|
- channelCfg.pollIntervalMs ?? 10000
|
|
|
+ resolvePollIntervalMs(channelCfg) ?? 10000
|
|
|
+ );
|
|
|
+ const batchWindowMs = Math.max(0, resolveBatchWindowMs(channelCfg) ?? 3000);
|
|
|
+ const replyCooldownMs = Math.max(0, (resolveReplyCooldownSecPerWxid(channelCfg) ?? 5) * 1000);
|
|
|
+ configureOutboundQueue(
|
|
|
+ {
|
|
|
+ ratePerSec: resolveOutboundRatePerSec(channelCfg),
|
|
|
+ maxQueue: resolveOutboundQueueMax(channelCfg),
|
|
|
+ },
|
|
|
+ log
|
|
|
);
|
|
|
- const batchWindowMs = Math.max(0, channelCfg.batchWindowMs ?? 3000);
|
|
|
- const replyCooldownMs = Math.max(0, (channelCfg.replyCooldownSecPerWxid ?? 5) * 1000);
|
|
|
+ const baseMetricsLogIntervalMs = Math.max(30 * 1000, resolveMetricsLogIntervalSec(channelCfg) * 1000);
|
|
|
+ const highThroughputAttemptsThreshold = resolveHighThroughputOutboundAttemptsPer5Min(channelCfg);
|
|
|
+
|
|
|
+ // Persistent inbound cursor: best-effort resume after restarts.
|
|
|
+ // We store it alongside OpenClaw's session store so admins can back it up
|
|
|
+ // together with other channel state.
|
|
|
+ const core = getRuntime();
|
|
|
+ const sessionCfg = (opts.cfg as Record<string, unknown>).session as
|
|
|
+ | { store?: string }
|
|
|
+ | undefined;
|
|
|
+ const cursorStoreDir = core.channel.session.resolveStorePath(sessionCfg?.store, {
|
|
|
+ agentId: CHANNEL_ID,
|
|
|
+ });
|
|
|
+ const cursorPath = join(cursorStoreDir, `${CHANNEL_ID}.inbound-cursor.json`);
|
|
|
+ let cursor: InboundCursor | null = await loadCursor(cursorPath);
|
|
|
|
|
|
// Resolve bot's own wxid so we can filter self-echoes.
|
|
|
- let selfWxid: string | null = channelCfg.selfWxid?.trim() || null;
|
|
|
+ let selfWxid: string | null = resolveSelfWxid(channelCfg)?.trim() || null;
|
|
|
if (!selfWxid) {
|
|
|
try {
|
|
|
selfWxid = await client.getSelfWxid();
|
|
|
@@ -119,6 +183,10 @@ export async function startMonitor(opts: MonitorOpts): Promise<MonitorResult> {
|
|
|
outboundCount: 0,
|
|
|
faqHits: 0,
|
|
|
};
|
|
|
+ let metricsLastLogAt = Date.now();
|
|
|
+ let throughputWindowStartAt = Date.now();
|
|
|
+ let throughputWindowLastAttempts = 0;
|
|
|
+ let adaptiveMetricsLogIntervalMs = baseMetricsLogIntervalMs;
|
|
|
|
|
|
// Message dedupe window
|
|
|
const seenMsgIds = new Set<string>();
|
|
|
@@ -179,10 +247,23 @@ export async function startMonitor(opts: MonitorOpts): Promise<MonitorResult> {
|
|
|
direction: "received",
|
|
|
limit: 50,
|
|
|
signal: opts.abortSignal,
|
|
|
+ sinceTimestampMs: cursor?.lastTimestampMs,
|
|
|
+ sinceMsgId: cursor?.lastMsgId,
|
|
|
});
|
|
|
counters.pollsOk++;
|
|
|
+ maybeEmitHealthLogs();
|
|
|
+
|
|
|
+ // Always process in stable order so agent context is consistent.
|
|
|
+ messages.sort((a, b) => {
|
|
|
+ const ta = toEpochMs(a.timestamp);
|
|
|
+ const tb = toEpochMs(b.timestamp);
|
|
|
+ if (ta !== tb) return ta - tb;
|
|
|
+ return compareMsgId(String(a.msgId), String(b.msgId));
|
|
|
+ });
|
|
|
|
|
|
for (const m of messages) {
|
|
|
+ // Cursor gate first: don't let old messages fill the dedupe window.
|
|
|
+ if (!isAfterCursor(m, cursor)) continue;
|
|
|
if (!markSeen(m.msgId)) continue;
|
|
|
const ignoredReason = shouldIgnoreMessage(m, channelCfg, selfWxid);
|
|
|
if (ignoredReason) {
|
|
|
@@ -201,7 +282,7 @@ export async function startMonitor(opts: MonitorOpts): Promise<MonitorResult> {
|
|
|
opts.patchStatus({ lastInboundAt: Date.now() });
|
|
|
|
|
|
// FAQ fast-path
|
|
|
- const match = matchFaq(m.content, channelCfg.faq);
|
|
|
+ const match = matchFaq(m.content, resolveFaq(channelCfg));
|
|
|
if (match) {
|
|
|
counters.faqHits++;
|
|
|
log.info(
|
|
|
@@ -220,17 +301,69 @@ export async function startMonitor(opts: MonitorOpts): Promise<MonitorResult> {
|
|
|
|
|
|
// Batch + dispatch
|
|
|
cooldown.enqueue(m, resolveConversationId(m));
|
|
|
+
|
|
|
+ // Advance cursor after enqueue. The actual agent dispatch is async,
|
|
|
+ // but enqueueing is the "accepted for processing" point for this
|
|
|
+ // channel. We still keep a bounded dedupe window to tolerate crashes
|
|
|
+ // between enqueue and outbound delivery.
|
|
|
+ cursor = advanceCursor(cursor, m);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Best-effort persist cursor once per tick.
|
|
|
+ if (cursor) {
|
|
|
+ await saveCursor(cursorPath, cursor);
|
|
|
}
|
|
|
} catch (err) {
|
|
|
counters.pollsFail++;
|
|
|
const msg = err instanceof Error ? err.message : String(err);
|
|
|
opts.patchStatus({ lastError: msg });
|
|
|
log.warn(`poll error: ${msg}`);
|
|
|
+ maybeEmitHealthLogs();
|
|
|
} finally {
|
|
|
pollInFlight = false;
|
|
|
}
|
|
|
};
|
|
|
|
|
|
+ const maybeEmitHealthLogs = () => {
|
|
|
+ const now = Date.now();
|
|
|
+ const ticks = counters.pollsOk + counters.pollsFail;
|
|
|
+ const outboundNow = getOutboundMetricsSnapshot();
|
|
|
+ if (now - throughputWindowStartAt >= THROUGHPUT_WINDOW_MS) {
|
|
|
+ const attemptsInWindow = outboundNow.attempts - throughputWindowLastAttempts;
|
|
|
+ throughputWindowStartAt = now;
|
|
|
+ throughputWindowLastAttempts = outboundNow.attempts;
|
|
|
+ const nextInterval =
|
|
|
+ attemptsInWindow >= highThroughputAttemptsThreshold
|
|
|
+ ? Math.max(baseMetricsLogIntervalMs, HIGH_THROUGHPUT_LOG_INTERVAL_MS)
|
|
|
+ : baseMetricsLogIntervalMs;
|
|
|
+ if (nextInterval !== adaptiveMetricsLogIntervalMs) {
|
|
|
+ adaptiveMetricsLogIntervalMs = nextInterval;
|
|
|
+ log.info(
|
|
|
+ `metrics interval adjusted to ${Math.round(adaptiveMetricsLogIntervalMs / 1000)}s (outboundAttempts5m=${attemptsInWindow}, threshold=${highThroughputAttemptsThreshold})`
|
|
|
+ );
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (now - metricsLastLogAt >= adaptiveMetricsLogIntervalMs) {
|
|
|
+ metricsLastLogAt = now;
|
|
|
+ log.info(
|
|
|
+ `metrics pollsOk=${counters.pollsOk} pollsFail=${counters.pollsFail} inbound=${counters.inboundCount} outbound=${counters.outboundCount} faqHits=${counters.faqHits} outAttempts=${outboundNow.attempts} outSuccess=${outboundNow.success} outFailed=${outboundNow.failed} outDeduped=${outboundNow.deduped} outAvgLatencyMs=${outboundNow.avgLatencyMs} metricsIntervalSec=${Math.round(adaptiveMetricsLogIntervalMs / 1000)}`
|
|
|
+ );
|
|
|
+ }
|
|
|
+ if (ticks >= POLL_ALERT_MIN_TICKS) {
|
|
|
+ const failRate = counters.pollsFail / ticks;
|
|
|
+ if (failRate >= POLL_ALERT_FAIL_RATE) {
|
|
|
+ log.warn(
|
|
|
+ `[ALERT] high poll failure rate fail=${counters.pollsFail} total=${ticks} ratio=${failRate.toFixed(2)}`
|
|
|
+ );
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (shouldEmitOutboundFailureAlert()) {
|
|
|
+ log.warn(
|
|
|
+ `[ALERT] outbound failure rate high attempts=${outboundNow.attempts} failed=${outboundNow.failed} ratio=${outboundNow.failRate.toFixed(2)}`
|
|
|
+ );
|
|
|
+ }
|
|
|
+ };
|
|
|
+
|
|
|
const timer = setInterval(() => {
|
|
|
tick().catch((err) => log.error(`tick crashed: ${String(err)}`));
|
|
|
}, pollIntervalMs);
|
|
|
@@ -415,17 +548,46 @@ async function dispatchBatchToAgent(params: DispatchParams): Promise<void> {
|
|
|
|
|
|
for (const chunk of parts) {
|
|
|
if (!chunk) continue;
|
|
|
- const result = await client.sendText({
|
|
|
- toWxid: conversationId,
|
|
|
- content: chunk,
|
|
|
- ats: isGroup && channelCfg.groupReplyAtSender && senderWxid ? senderWxid : "",
|
|
|
- });
|
|
|
- if (!result.ok) {
|
|
|
- log.error(`send-text failed ret=${String(result.ret)} msg=${result.msg ?? ""}`);
|
|
|
- throw new Error(
|
|
|
- `send-text failed: ${result.msg ?? `ret=${String(result.ret)}`}`
|
|
|
- );
|
|
|
+ if (shouldSkipDuplicateSend("agent-reply", conversationId, chunk)) {
|
|
|
+ log.warn(`skip duplicate agent reply conversation=${conversationId}`);
|
|
|
+ continue;
|
|
|
+ }
|
|
|
+ const startedAt = Date.now();
|
|
|
+ recordOutboundAttempt("agent-reply");
|
|
|
+ try {
|
|
|
+ await enqueueOutboundSend(`agent-reply:${conversationId}`, () =>
|
|
|
+ withRetry(
|
|
|
+ async () => {
|
|
|
+ const res = await client.sendText({
|
|
|
+ toWxid: conversationId,
|
|
|
+ content: chunk,
|
|
|
+ ats:
|
|
|
+ isGroup && resolveGroupReplyAtSender(channelCfg) && senderWxid
|
|
|
+ ? senderWxid
|
|
|
+ : "",
|
|
|
+ });
|
|
|
+ if (!res.ok) {
|
|
|
+ throw new Error(res.msg ?? `ret=${String(res.ret)}`);
|
|
|
+ }
|
|
|
+ return res;
|
|
|
+ },
|
|
|
+ {
|
|
|
+ tries: 4,
|
|
|
+ baseDelayMs: 500,
|
|
|
+ maxDelayMs: 4000,
|
|
|
+ isRetryable: (err) => {
|
|
|
+ const msg = err instanceof Error ? err.message : String(err);
|
|
|
+ if (/timeout|aborted|ECONN|ENOTFOUND|EAI_AGAIN/i.test(msg)) return true;
|
|
|
+ if (/HTTP 5\d\d/i.test(msg)) return true;
|
|
|
+ return false;
|
|
|
+ },
|
|
|
+ }
|
|
|
+ ));
|
|
|
+ } catch (err) {
|
|
|
+ recordOutboundFailure("agent-reply");
|
|
|
+ throw err;
|
|
|
}
|
|
|
+ recordOutboundSuccess("agent-reply", conversationId, chunk, Date.now() - startedAt);
|
|
|
}
|
|
|
log.info(`delivered reply to ${conversationId} (${text.length} chars)`);
|
|
|
onOutboundSent();
|
|
|
@@ -461,14 +623,25 @@ async function sendFaqReply(
|
|
|
): Promise<void> {
|
|
|
const conversationId = resolveConversationId(m);
|
|
|
const senderWxid = resolveSenderWxid(m);
|
|
|
- const result = await client.sendText({
|
|
|
- toWxid: conversationId,
|
|
|
- content: reply,
|
|
|
- ats: m.isGroup && cfg.groupReplyAtSender && senderWxid ? senderWxid : "",
|
|
|
- });
|
|
|
+ if (shouldSkipDuplicateSend("faq", conversationId, reply)) {
|
|
|
+ log.warn(`skip duplicate faq reply to ${conversationId}`);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ recordOutboundAttempt("faq");
|
|
|
+ const startedAt = Date.now();
|
|
|
+ const result = await enqueueOutboundSend(`faq:${conversationId}`, () =>
|
|
|
+ client.sendText({
|
|
|
+ toWxid: conversationId,
|
|
|
+ content: reply,
|
|
|
+ ats:
|
|
|
+ m.isGroup && resolveGroupReplyAtSender(cfg) && senderWxid ? senderWxid : "",
|
|
|
+ })
|
|
|
+ );
|
|
|
if (result.ok) {
|
|
|
+ recordOutboundSuccess("faq", conversationId, reply, Date.now() - startedAt);
|
|
|
log.info(`faq reply sent to ${conversationId}`);
|
|
|
} else {
|
|
|
+ recordOutboundFailure("faq");
|
|
|
log.warn(
|
|
|
`faq reply failed to ${conversationId}: ret=${String(result.ret)} msg=${result.msg ?? ""}`
|
|
|
);
|
|
|
@@ -487,23 +660,23 @@ function shouldIgnoreMessage(
|
|
|
if (selfWxid && m.chatroomMemberWxid === selfWxid) return "self-group-echo";
|
|
|
|
|
|
// Group switch
|
|
|
- if (m.isGroup && cfg.groupEnabled === false) return "group-disabled";
|
|
|
+ if (m.isGroup && resolveGroupEnabled(cfg) === false) return "group-disabled";
|
|
|
|
|
|
// Type filter
|
|
|
- const ignoreTypes = cfg.ignoreMessageTypes ?? [];
|
|
|
+ const ignoreTypes = resolveIgnoreMessageTypes(cfg) ?? [];
|
|
|
if (ignoreTypes.includes(m.type)) return `ignored-type:${m.type}`;
|
|
|
|
|
|
// v0.2: inbound still text-only
|
|
|
if (m.type !== "text") return "non-text";
|
|
|
|
|
|
// Wxid prefix filter (official accounts etc.)
|
|
|
- const prefixes = cfg.ignoreWxidPrefixes ?? [];
|
|
|
+ const prefixes = resolveIgnoreWxidPrefixes(cfg) ?? [];
|
|
|
for (const p of prefixes) {
|
|
|
if (p && m.fromWxid.startsWith(p)) return `ignored-prefix:${p}`;
|
|
|
}
|
|
|
|
|
|
// Exact wxid filter (system contacts)
|
|
|
- const exacts = cfg.ignoreWxidExact ?? [];
|
|
|
+ const exacts = resolveIgnoreWxidExact(cfg) ?? [];
|
|
|
if (exacts.includes(m.fromWxid)) return `ignored-wxid:${m.fromWxid}`;
|
|
|
|
|
|
// Empty content
|
|
|
@@ -513,10 +686,10 @@ function shouldIgnoreMessage(
|
|
|
}
|
|
|
|
|
|
function passesDmPolicy(m: RawWechatMessage, cfg: WechatAgentConfig): boolean {
|
|
|
- const dmPolicy = cfg.dmPolicy ?? "open";
|
|
|
+ const dmPolicy = resolveDmPolicy(cfg) ?? "open";
|
|
|
if (dmPolicy === "open") return true;
|
|
|
|
|
|
- const allowFrom = normalizeAllowFrom(cfg.allowFrom);
|
|
|
+ const allowFrom = normalizeAllowFrom(resolveAllowFrom(cfg));
|
|
|
if (allowFrom.includes(m.fromWxid.toLowerCase())) return true;
|
|
|
|
|
|
// pairing is handled upstream in core; here we treat it like allowlist.
|
|
|
@@ -543,13 +716,15 @@ function isGroupTriggered(
|
|
|
): boolean {
|
|
|
if (!m.isGroup) return true;
|
|
|
const content = m.content.toLowerCase();
|
|
|
- const keywords = (cfg.groupKeywords?.length ? cfg.groupKeywords : DEFAULT_GROUP_KEYWORDS)
|
|
|
+ const groupKeywords = resolveGroupKeywords(cfg);
|
|
|
+ const keywords = (groupKeywords?.length ? groupKeywords : DEFAULT_GROUP_KEYWORDS)
|
|
|
.map((s) => s.trim().toLowerCase())
|
|
|
.filter(Boolean);
|
|
|
if (keywords.some((kw) => content.includes(kw))) return true;
|
|
|
|
|
|
+ const groupMentionTokens = resolveGroupMentionTokens(cfg);
|
|
|
const mentionTokens = (
|
|
|
- cfg.groupMentionTokens?.length ? cfg.groupMentionTokens : DEFAULT_GROUP_MENTION_TOKENS
|
|
|
+ groupMentionTokens?.length ? groupMentionTokens : DEFAULT_GROUP_MENTION_TOKENS
|
|
|
)
|
|
|
.map((s) => s.trim().toLowerCase())
|
|
|
.filter(Boolean);
|