#!/usr/bin/env tsx /** * 人工 docx 10 组 — RPA kernel 直调 + 人工价对比(开发调试用) * 系统 API 验收请用:npm run probe:human-doc-system * * npm run benchmark:human-doc-10pairs */ import { randomUUID } from "node:crypto"; import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { loadDevEnv } from "@/lib/dev-env"; import { hostFetchMothershipCandidates } from "@/lib/frontend/api-client"; import { applyMothershipCandidate } from "@/lib/frontend/mothership-address"; import { RPAContext } from "@/workers/rpa/kernel/context"; import { runMothershipQuoteKernel } from "@/workers/rpa/kernel/orchestrator"; import { stabilizeQuoteLandingPage, waitForQuoterWidgetHydrated, } from "@/workers/rpa/page-prep"; import { closeBrowser, openQuotePage } from "@/workers/rpa/session-manager"; import { quotePageAdapter } from "@/workers/rpa/quote-page-adapter"; import type { QuoteItem, QuoteRequest } from "@/modules/providers/quote-provider"; import { HUMAN_DOC_10_PAIRS, HUMAN_DOC_CARGO, type HumanDocPair, } from "./lib/human-doc-10pairs"; process.env.RPA_MOCK_MODE = "false"; process.env.RPA_ADDRESS_MODE = "real"; /** 报价策略:Direct 优先,Widget 仅 fallback(勿设 RPA_QUOTE_MODE=widget) */ process.env.RPA_VISUAL_RUSH = process.env.RPA_VISUAL_RUSH ?? "false"; /** 默认与 benchmark 一致 rush 提交;设 RPA_HUMAN_SUBMIT_STRICT=true 启用 submit 前闸门 */ process.env.RPA_HUMAN_SUBMIT_STRICT = process.env.RPA_HUMAN_SUBMIT_STRICT ?? "false"; process.env.RPA_DUAL_ADDRESS_PATCH = process.env.RPA_DUAL_ADDRESS_PATCH ?? "true"; process.env.RPA_HEADLESS = process.env.RPA_HEADLESS ?? "true"; process.env.RPA_KEEP_BROWSER_OPEN = "false"; process.env.RPA_SLOW_MO_MS = "0"; process.env.RPA_HUMAN_PACING = "false"; process.env.RPA_BENCHMARK_QUOTE_MS = "45000"; process.env.RPA_REQUIRE_QUOTE_RATES = "true"; loadDevEnv(); const BASE = process.env.PROBE_BASE_URL ?? "http://localhost:3000"; const TOKEN = "demo-host-token"; const CUSTOMER_ID = "CUST_001"; /** 单档可接受:≤12% 或 ≤$50 */ const TIER_OK_PCT = 0.12; const TIER_OK_ABS = 50; /** 触发重试:>20% 且 >$80 */ const TIER_RETRY_PCT = 0.2; const TIER_RETRY_ABS = 80; const MAX_ATTEMPTS = 3; const CARGO = { palletCount: HUMAN_DOC_CARGO.palletCount, weightLb: HUMAN_DOC_CARGO.weightLb, dimsIn: HUMAN_DOC_CARGO.dimsIn, cargoType: HUMAN_DOC_CARGO.cargoType, }; type HumanRef = HumanDocPair["humanQuotes"][number]; type HumanPair = HumanDocPair; const PAIRS = HUMAN_DOC_10_PAIRS; type TierCmp = { key: string; human: number; rpa: number | null; absDiff: number | null; pctDiff: number | null; status: "match" | "close" | "missing" | "retry"; }; type RowResult = { id: string; route: string; pickup: { input: { street: string; city: string; state: string }; selectedSuggestion: string; optionId: string; }; delivery: { input: { street: string; city: string; state: string }; selectedSuggestion: string; optionId: string; }; cargo: typeof CARGO; attempts: number; timing: { totalMs: number; kernelMs: number }; comparisons: TierCmp[]; quotes: Array<{ serviceLevel: QuoteItem["serviceLevel"]; rateOption: QuoteItem["rateOption"]; rawTotal: number; carrier: string; transitDays: string; }>; result: { ok: boolean; compareOk: boolean; tierCount: number; finalUrl: string }; error?: string; }; function tierKey(serviceLevel: string, rateOption: string): string { return `${serviceLevel}/${rateOption}`; } function fmtMs(ms: number): string { if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`; const m = Math.floor(ms / 60_000); const s = ((ms % 60_000) / 1000).toFixed(0); return `${m}m${s}s`; } function fmtUsd(n: number): string { return `$${n.toFixed(2)}`; } function draftAddress(d: { street: string; city: string; state: string }) { return { street: d.street.trim(), city: d.city.trim(), state: d.state.trim(), zip: "", place_id: `draft_${d.street}_${d.city}_${d.state}`.replace(/\s+/g, "_").slice(0, 64), formatted_address: `${d.street}, ${d.city}, ${d.state}`, selected_from_suggestions: true, }; } function compareToHuman( humanQuotes: HumanRef[], rpaItems: QuoteItem[], ): { comparisons: TierCmp[]; compareOk: boolean; shouldRetry: boolean } { const rpaMap = new Map( rpaItems.map((q) => [tierKey(q.serviceLevel, q.rateOption), q.rawTotal]), ); const comparisons: TierCmp[] = []; let shouldRetry = false; for (const h of humanQuotes) { const key = tierKey(h.serviceLevel, h.rateOption); const rpa = rpaMap.get(key) ?? null; if (rpa === null) { comparisons.push({ key, human: h.rawTotal, rpa: null, absDiff: null, pctDiff: null, status: "retry", }); shouldRetry = true; continue; } const absDiff = Math.abs(rpa - h.rawTotal); const pctDiff = absDiff / h.rawTotal; let status: TierCmp["status"] = "match"; if (pctDiff <= TIER_OK_PCT || absDiff <= TIER_OK_ABS) { status = absDiff <= 1 ? "match" : "close"; } else if (pctDiff > TIER_RETRY_PCT && absDiff > TIER_RETRY_ABS) { status = "retry"; shouldRetry = true; } else { status = "close"; } comparisons.push({ key, human: h.rawTotal, rpa, absDiff, pctDiff, status }); } return { comparisons, compareOk: !shouldRetry, shouldRetry }; } async function waitForHttpOk(url: string, attempts = 20): Promise { for (let i = 0; i < attempts; i += 1) { try { const res = await fetch(url); if (res.ok) return; } catch { /* retry */ } await new Promise((r) => setTimeout(r, 2000)); } throw new Error(`服务未就绪: ${url}`); } async function fetchCandidatesWithRetry( pickup: ReturnType, delivery: ReturnType, attempts = 3, ) { let lastErr = "候选 API 失败"; for (let i = 0; i < attempts; i += 1) { try { const cand = await hostFetchMothershipCandidates( BASE, TOKEN, CUSTOMER_ID, pickup, delivery, ); if (cand.code === 0 && cand.data?.pickup_candidates[0] && cand.data.delivery_candidates[0]) { return cand; } lastErr = cand.message ?? lastErr; } catch (e) { lastErr = e instanceof Error ? e.message : String(e); } if (i < attempts - 1) await new Promise((r) => setTimeout(r, 2000)); } return { code: -1, message: lastErr, data: null }; } async function runOneCase(pair: HumanPair, attempt: number): Promise { const caseStart = Date.now(); const userPickup = draftAddress(pair.pickup); const userDelivery = draftAddress(pair.delivery); const base = (): Omit => ({ id: pair.id, route: pair.route, pickup: { input: { ...pair.pickup }, selectedSuggestion: "", optionId: "", }, delivery: { input: { ...pair.delivery }, selectedSuggestion: "", optionId: "", }, cargo: { ...CARGO }, }); const cand = await fetchCandidatesWithRetry(userPickup, userDelivery); if (cand.code !== 0 || !cand.data?.pickup_candidates[0] || !cand.data.delivery_candidates[0]) { return { ...base(), attempts: attempt, timing: { totalMs: Date.now() - caseStart, kernelMs: 0 }, comparisons: [], quotes: [], result: { ok: false, compareOk: false, tierCount: 0, finalUrl: "" }, error: cand.message ?? "候选 API 失败", }; } const pc = cand.data.pickup_candidates[0]; const dc = cand.data.delivery_candidates[0]; const pickupApplied = applyMothershipCandidate(userPickup, pc); const deliveryApplied = applyMothershipCandidate(userDelivery, dc); const row = base(); row.pickup.selectedSuggestion = pc.display_label; row.pickup.optionId = pc.option_id; row.delivery.selectedSuggestion = dc.display_label; row.delivery.optionId = dc.option_id; const req: QuoteRequest = { cargoHash: `human_${pair.id}_${randomUUID().slice(0, 8)}`, pickup: { street: pickupApplied.street, city: pickupApplied.city, state: pickupApplied.state, zip: pickupApplied.zip ?? "", placeId: pc.option_id, formattedAddress: pickupApplied.formatted_address, selectedFromSuggestions: true, mothershipOptionId: pc.option_id, mothershipDisplayLabel: pc.display_label, selectedFromMothership: true, }, delivery: { street: deliveryApplied.street, city: deliveryApplied.city, state: deliveryApplied.state, zip: deliveryApplied.zip ?? "", placeId: dc.option_id, formattedAddress: deliveryApplied.formatted_address, selectedFromSuggestions: true, mothershipOptionId: dc.option_id, mothershipDisplayLabel: dc.display_label, selectedFromMothership: true, }, palletCount: CARGO.palletCount, weightLb: CARGO.weightLb, dimsIn: { l: CARGO.dimsIn.length, w: CARGO.dimsIn.width, h: CARGO.dimsIn.height }, cargoType: CARGO.cargoType, }; let opened: Awaited> | null = null; try { opened = await openQuotePage(quotePageAdapter, false, undefined, { freshCargoContext: true, }); const ctx = RPAContext.fromSession(opened.page, opened.context); await stabilizeQuoteLandingPage(ctx); await waitForQuoterWidgetHydrated(ctx.page, 90_000); const kernelStart = Date.now(); const items = await runMothershipQuoteKernel(ctx, req); const kernelMs = Date.now() - kernelStart; const finalUrl = opened.page.url(); const { comparisons, compareOk, shouldRetry } = compareToHuman(pair.humanQuotes, items); const quotes = items.map((q) => ({ serviceLevel: q.serviceLevel, rateOption: q.rateOption, rawTotal: q.rawTotal, carrier: q.carrier, transitDays: q.transitDays, })); const ok = items.length > 0; return { ...row, attempts: attempt, timing: { totalMs: Date.now() - caseStart, kernelMs }, comparisons, quotes, result: { ok, compareOk, tierCount: items.length, finalUrl }, error: ok ? compareOk ? undefined : shouldRetry ? "与人工报价偏差过大(链路已成功)" : undefined : "未抓到报价档位", }; } catch (e) { return { ...row, attempts: attempt, timing: { totalMs: Date.now() - caseStart, kernelMs: 0 }, comparisons: [], quotes: [], result: { ok: false, compareOk: false, tierCount: 0, finalUrl: opened?.page.url() ?? "" }, error: e instanceof Error ? e.message : String(e), }; } finally { await opened?.page.close().catch(() => undefined); await opened?.context.close().catch(() => undefined); } } async function runOneWithRetry(pair: HumanPair): Promise { let last: RowResult | null = null; for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) { if (attempt > 1) { const reason = last?.quotes.length === 0 ? "报价未抓到" : last?.comparisons.some((c) => c.status === "retry") ? "与人工偏差过大" : "链路失败"; console.log(`[human-bench] ${pair.id} ${reason},重试 ${attempt}/${MAX_ATTEMPTS} …`); await new Promise((r) => setTimeout(r, 3000)); } last = await runOneCase(pair, attempt); if (last.result.ok && last.quotes.length > 0) { return last; } if (last.quotes.length === 0) continue; } return last!; } function printCmpLine(c: TierCmp): string { if (c.rpa === null) return `${c.key}: 人工${fmtUsd(c.human)} RPA=缺失`; const pct = c.pctDiff !== null ? `${(c.pctDiff * 100).toFixed(1)}%` : "—"; return `${c.key}: 人工${fmtUsd(c.human)} RPA${fmtUsd(c.rpa)} Δ${fmtUsd(c.absDiff ?? 0)}(${pct}) [${c.status}]`; } async function main(): Promise { const runId = randomUUID().slice(0, 8); const startedAt = Date.now(); await waitForHttpOk(`${BASE}/`); console.log(`[human-bench] 人工 docx 10 组 widget 对比 ${runId}`); console.log( `[human-bench] 链路: widget + visual-rush=${process.env.RPA_VISUAL_RUSH === "true"} dual-patch=${process.env.RPA_DUAL_ADDRESS_PATCH === "true"} | 货物: 2托 500lb 50×50×50 | 容忍 ≤${TIER_OK_PCT * 100}%或≤$${TIER_OK_ABS} | 重试 >${TIER_RETRY_PCT * 100}%且>$${TIER_RETRY_ABS}\n`, ); const rows: RowResult[] = []; const only = process.env.BENCH_HUMAN_ONLY?.trim(); const onlyIds = only ? only.split(/[,;]/).map((s) => s.trim()).filter(Boolean) : null; const pairs = onlyIds ? PAIRS.filter((p) => onlyIds.includes(p.id)) : PAIRS; if (pairs.length === 0) { throw new Error(`BENCH_HUMAN_ONLY 无效: ${only}`); } for (let i = 0; i < pairs.length; i += 1) { const pair = pairs[i]!; console.log(`[human-bench] ${i + 1}/${pairs.length} ${pair.route} …`); const result = await runOneWithRetry(pair); rows.push(result); console.log( ` → ${result.result.ok ? "OK" : "FAIL"} attempts=${result.attempts} tiers=${result.result.tierCount} ${fmtMs(result.timing.totalMs)}`, ); console.log(` pickup: ${result.pickup.selectedSuggestion || "—"}`); console.log(` delivery: ${result.delivery.selectedSuggestion || "—"}`); for (const c of result.comparisons) { console.log(` ${printCmpLine(c)}`); } if (result.error) console.log(` err: ${result.error}`); } await closeBrowser(); const pass = rows.filter((r) => r.result.ok).length; const report = { runId, sourceDoc: "docs/美国跨州货物运输报价汇总_edited3.docx", finishedAt: new Date().toISOString(), durationMs: Date.now() - startedAt, tolerance: { okPct: TIER_OK_PCT, okAbs: TIER_OK_ABS, retryPct: TIER_RETRY_PCT, retryAbs: TIER_RETRY_ABS }, summary: { pass, fail: rows.length - pass }, cargoTemplate: CARGO, env: { quoteStrategy: "direct-first-widget-fallback", RPA_VISUAL_RUSH: process.env.RPA_VISUAL_RUSH === "true", RPA_HUMAN_SUBMIT_STRICT: process.env.RPA_HUMAN_SUBMIT_STRICT === "true", RPA_REQUIRE_QUOTE_RATES: true, RPA_BENCHMARK_QUOTE_MS: 45_000, RPA_HEADLESS: process.env.RPA_HEADLESS === "true", }, rows, }; const outDir = join(process.cwd(), ".rpa", "human-doc-benchmark"); mkdirSync(outDir, { recursive: true }); const outPath = join(outDir, `human-${runId}.json`); writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); console.log("\n| # | ID | Route | Result | Attempts | 比对 |"); console.log("|---|-----|-------|--------|----------|------|"); for (let i = 0; i < rows.length; i += 1) { const r = rows[i]!; const cmpSummary = r.comparisons .map((c) => `${c.key.split("/")[1]}:${c.status === "match" ? "✓" : c.status === "close" ? "~" : "✗"}`) .join(" "); console.log( `| ${i + 1} | ${r.id} | ${r.route} | ${r.result.ok ? "PASS" : "FAIL"} | ${r.attempts} | ${cmpSummary} |`, ); } console.log(`\n[human-bench] 报告: ${outPath}`); process.exit(pass === rows.length ? 0 : 1); } main().catch(async (e) => { console.error(e); await closeBrowser().catch(() => undefined); process.exit(1); });