|
|
#!/usr/bin/env tsx
|
|
|
/**
|
|
|
* 人工 docx 10 组 — RPA kernel 直调 + 人工价对比（开发调试用）
|
|
|
* 系统 API 验收请用：npm run probe:human-doc-system
|
|
|
*
|
|
|
* npm run benchmark:human-doc-10pairs
|
|
|
*/
|
|
|
import { randomUUID } from "node:crypto";
|
|
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
import { join } from "node:path";
|
|
|
import { loadDevEnv } from "@/lib/dev-env";
|
|
|
import { hostFetchMothershipCandidates } from "@/lib/frontend/api-client";
|
|
|
import { applyMothershipCandidate } from "@/lib/frontend/mothership-address";
|
|
|
import { RPAContext } from "@/workers/rpa/kernel/context";
|
|
|
import { runMothershipQuoteKernel } from "@/workers/rpa/kernel/orchestrator";
|
|
|
import {
|
|
|
stabilizeQuoteLandingPage,
|
|
|
waitForQuoterWidgetHydrated,
|
|
|
} from "@/workers/rpa/page-prep";
|
|
|
import { closeBrowser, openQuotePage } from "@/workers/rpa/session-manager";
|
|
|
import { quotePageAdapter } from "@/workers/rpa/quote-page-adapter";
|
|
|
import type { QuoteItem, QuoteRequest } from "@/modules/providers/quote-provider";
|
|
|
import {
|
|
|
HUMAN_DOC_10_PAIRS,
|
|
|
HUMAN_DOC_CARGO,
|
|
|
type HumanDocPair,
|
|
|
} from "./lib/human-doc-10pairs";
|
|
|
|
|
|
process.env.RPA_MOCK_MODE = "false";
|
|
|
process.env.RPA_ADDRESS_MODE = "real";
|
|
|
/** 报价策略：Direct 优先，Widget 仅 fallback（勿设 RPA_QUOTE_MODE=widget） */
|
|
|
process.env.RPA_VISUAL_RUSH = process.env.RPA_VISUAL_RUSH ?? "false";
|
|
|
/** 默认与 benchmark 一致 rush 提交；设 RPA_HUMAN_SUBMIT_STRICT=true 启用 submit 前闸门 */
|
|
|
process.env.RPA_HUMAN_SUBMIT_STRICT = process.env.RPA_HUMAN_SUBMIT_STRICT ?? "false";
|
|
|
process.env.RPA_DUAL_ADDRESS_PATCH = process.env.RPA_DUAL_ADDRESS_PATCH ?? "true";
|
|
|
process.env.RPA_HEADLESS = process.env.RPA_HEADLESS ?? "true";
|
|
|
process.env.RPA_KEEP_BROWSER_OPEN = "false";
|
|
|
process.env.RPA_SLOW_MO_MS = "0";
|
|
|
process.env.RPA_HUMAN_PACING = "false";
|
|
|
process.env.RPA_BENCHMARK_QUOTE_MS = "45000";
|
|
|
process.env.RPA_REQUIRE_QUOTE_RATES = "true";
|
|
|
|
|
|
loadDevEnv();
|
|
|
|
|
|
const BASE = process.env.PROBE_BASE_URL ?? "http://localhost:3000";
|
|
|
const TOKEN = "demo-host-token";
|
|
|
const CUSTOMER_ID = "CUST_001";
|
|
|
|
|
|
/** 单档可接受：≤12% 或 ≤$50 */
|
|
|
const TIER_OK_PCT = 0.12;
|
|
|
const TIER_OK_ABS = 50;
|
|
|
/** 触发重试：>20% 且 >$80 */
|
|
|
const TIER_RETRY_PCT = 0.2;
|
|
|
const TIER_RETRY_ABS = 80;
|
|
|
const MAX_ATTEMPTS = 3;
|
|
|
|
|
|
const CARGO = {
|
|
|
palletCount: HUMAN_DOC_CARGO.palletCount,
|
|
|
weightLb: HUMAN_DOC_CARGO.weightLb,
|
|
|
dimsIn: HUMAN_DOC_CARGO.dimsIn,
|
|
|
cargoType: HUMAN_DOC_CARGO.cargoType,
|
|
|
};
|
|
|
|
|
|
type HumanRef = HumanDocPair["humanQuotes"][number];
|
|
|
type HumanPair = HumanDocPair;
|
|
|
const PAIRS = HUMAN_DOC_10_PAIRS;
|
|
|
|
|
|
type TierCmp = {
|
|
|
key: string;
|
|
|
human: number;
|
|
|
rpa: number | null;
|
|
|
absDiff: number | null;
|
|
|
pctDiff: number | null;
|
|
|
status: "match" | "close" | "missing" | "retry";
|
|
|
};
|
|
|
|
|
|
type RowResult = {
|
|
|
id: string;
|
|
|
route: string;
|
|
|
pickup: {
|
|
|
input: { street: string; city: string; state: string };
|
|
|
selectedSuggestion: string;
|
|
|
optionId: string;
|
|
|
};
|
|
|
delivery: {
|
|
|
input: { street: string; city: string; state: string };
|
|
|
selectedSuggestion: string;
|
|
|
optionId: string;
|
|
|
};
|
|
|
cargo: typeof CARGO;
|
|
|
attempts: number;
|
|
|
timing: { totalMs: number; kernelMs: number };
|
|
|
comparisons: TierCmp[];
|
|
|
quotes: Array<{
|
|
|
serviceLevel: QuoteItem["serviceLevel"];
|
|
|
rateOption: QuoteItem["rateOption"];
|
|
|
rawTotal: number;
|
|
|
carrier: string;
|
|
|
transitDays: string;
|
|
|
}>;
|
|
|
result: { ok: boolean; compareOk: boolean; tierCount: number; finalUrl: string };
|
|
|
error?: string;
|
|
|
};
|
|
|
|
|
|
function tierKey(serviceLevel: string, rateOption: string): string {
|
|
|
return `${serviceLevel}/${rateOption}`;
|
|
|
}
|
|
|
|
|
|
function fmtMs(ms: number): string {
|
|
|
if (ms < 60_000) return `${(ms / 1000).toFixed(1)}s`;
|
|
|
const m = Math.floor(ms / 60_000);
|
|
|
const s = ((ms % 60_000) / 1000).toFixed(0);
|
|
|
return `${m}m${s}s`;
|
|
|
}
|
|
|
|
|
|
function fmtUsd(n: number): string {
|
|
|
return `$${n.toFixed(2)}`;
|
|
|
}
|
|
|
|
|
|
function draftAddress(d: { street: string; city: string; state: string }) {
|
|
|
return {
|
|
|
street: d.street.trim(),
|
|
|
city: d.city.trim(),
|
|
|
state: d.state.trim(),
|
|
|
zip: "",
|
|
|
place_id: `draft_${d.street}_${d.city}_${d.state}`.replace(/\s+/g, "_").slice(0, 64),
|
|
|
formatted_address: `${d.street}, ${d.city}, ${d.state}`,
|
|
|
selected_from_suggestions: true,
|
|
|
};
|
|
|
}
|
|
|
|
|
|
function compareToHuman(
|
|
|
humanQuotes: HumanRef[],
|
|
|
rpaItems: QuoteItem[],
|
|
|
): { comparisons: TierCmp[]; compareOk: boolean; shouldRetry: boolean } {
|
|
|
const rpaMap = new Map(
|
|
|
rpaItems.map((q) => [tierKey(q.serviceLevel, q.rateOption), q.rawTotal]),
|
|
|
);
|
|
|
const comparisons: TierCmp[] = [];
|
|
|
let shouldRetry = false;
|
|
|
|
|
|
for (const h of humanQuotes) {
|
|
|
const key = tierKey(h.serviceLevel, h.rateOption);
|
|
|
const rpa = rpaMap.get(key) ?? null;
|
|
|
if (rpa === null) {
|
|
|
comparisons.push({
|
|
|
key,
|
|
|
human: h.rawTotal,
|
|
|
rpa: null,
|
|
|
absDiff: null,
|
|
|
pctDiff: null,
|
|
|
status: "retry",
|
|
|
});
|
|
|
shouldRetry = true;
|
|
|
continue;
|
|
|
}
|
|
|
const absDiff = Math.abs(rpa - h.rawTotal);
|
|
|
const pctDiff = absDiff / h.rawTotal;
|
|
|
let status: TierCmp["status"] = "match";
|
|
|
if (pctDiff <= TIER_OK_PCT || absDiff <= TIER_OK_ABS) {
|
|
|
status = absDiff <= 1 ? "match" : "close";
|
|
|
} else if (pctDiff > TIER_RETRY_PCT && absDiff > TIER_RETRY_ABS) {
|
|
|
status = "retry";
|
|
|
shouldRetry = true;
|
|
|
} else {
|
|
|
status = "close";
|
|
|
}
|
|
|
comparisons.push({ key, human: h.rawTotal, rpa, absDiff, pctDiff, status });
|
|
|
}
|
|
|
|
|
|
return { comparisons, compareOk: !shouldRetry, shouldRetry };
|
|
|
}
|
|
|
|
|
|
async function waitForHttpOk(url: string, attempts = 20): Promise<void> {
|
|
|
for (let i = 0; i < attempts; i += 1) {
|
|
|
try {
|
|
|
const res = await fetch(url);
|
|
|
if (res.ok) return;
|
|
|
} catch {
|
|
|
/* retry */
|
|
|
}
|
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
}
|
|
|
throw new Error(`服务未就绪: ${url}`);
|
|
|
}
|
|
|
|
|
|
async function fetchCandidatesWithRetry(
|
|
|
pickup: ReturnType<typeof draftAddress>,
|
|
|
delivery: ReturnType<typeof draftAddress>,
|
|
|
attempts = 3,
|
|
|
) {
|
|
|
let lastErr = "候选 API 失败";
|
|
|
for (let i = 0; i < attempts; i += 1) {
|
|
|
try {
|
|
|
const cand = await hostFetchMothershipCandidates(
|
|
|
BASE,
|
|
|
TOKEN,
|
|
|
CUSTOMER_ID,
|
|
|
pickup,
|
|
|
delivery,
|
|
|
);
|
|
|
if (cand.code === 0 && cand.data?.pickup_candidates[0] && cand.data.delivery_candidates[0]) {
|
|
|
return cand;
|
|
|
}
|
|
|
lastErr = cand.message ?? lastErr;
|
|
|
} catch (e) {
|
|
|
lastErr = e instanceof Error ? e.message : String(e);
|
|
|
}
|
|
|
if (i < attempts - 1) await new Promise((r) => setTimeout(r, 2000));
|
|
|
}
|
|
|
return { code: -1, message: lastErr, data: null };
|
|
|
}
|
|
|
|
|
|
async function runOneCase(pair: HumanPair, attempt: number): Promise<RowResult> {
|
|
|
const caseStart = Date.now();
|
|
|
const userPickup = draftAddress(pair.pickup);
|
|
|
const userDelivery = draftAddress(pair.delivery);
|
|
|
|
|
|
const base = (): Omit<RowResult, "timing" | "comparisons" | "quotes" | "result" | "attempts"> => ({
|
|
|
id: pair.id,
|
|
|
route: pair.route,
|
|
|
pickup: {
|
|
|
input: { ...pair.pickup },
|
|
|
selectedSuggestion: "",
|
|
|
optionId: "",
|
|
|
},
|
|
|
delivery: {
|
|
|
input: { ...pair.delivery },
|
|
|
selectedSuggestion: "",
|
|
|
optionId: "",
|
|
|
},
|
|
|
cargo: { ...CARGO },
|
|
|
});
|
|
|
|
|
|
const cand = await fetchCandidatesWithRetry(userPickup, userDelivery);
|
|
|
if (cand.code !== 0 || !cand.data?.pickup_candidates[0] || !cand.data.delivery_candidates[0]) {
|
|
|
return {
|
|
|
...base(),
|
|
|
attempts: attempt,
|
|
|
timing: { totalMs: Date.now() - caseStart, kernelMs: 0 },
|
|
|
comparisons: [],
|
|
|
quotes: [],
|
|
|
result: { ok: false, compareOk: false, tierCount: 0, finalUrl: "" },
|
|
|
error: cand.message ?? "候选 API 失败",
|
|
|
};
|
|
|
}
|
|
|
|
|
|
const pc = cand.data.pickup_candidates[0];
|
|
|
const dc = cand.data.delivery_candidates[0];
|
|
|
const pickupApplied = applyMothershipCandidate(userPickup, pc);
|
|
|
const deliveryApplied = applyMothershipCandidate(userDelivery, dc);
|
|
|
|
|
|
const row = base();
|
|
|
row.pickup.selectedSuggestion = pc.display_label;
|
|
|
row.pickup.optionId = pc.option_id;
|
|
|
row.delivery.selectedSuggestion = dc.display_label;
|
|
|
row.delivery.optionId = dc.option_id;
|
|
|
|
|
|
const req: QuoteRequest = {
|
|
|
cargoHash: `human_${pair.id}_${randomUUID().slice(0, 8)}`,
|
|
|
pickup: {
|
|
|
street: pickupApplied.street,
|
|
|
city: pickupApplied.city,
|
|
|
state: pickupApplied.state,
|
|
|
zip: pickupApplied.zip ?? "",
|
|
|
placeId: pc.option_id,
|
|
|
formattedAddress: pickupApplied.formatted_address,
|
|
|
selectedFromSuggestions: true,
|
|
|
mothershipOptionId: pc.option_id,
|
|
|
mothershipDisplayLabel: pc.display_label,
|
|
|
selectedFromMothership: true,
|
|
|
},
|
|
|
delivery: {
|
|
|
street: deliveryApplied.street,
|
|
|
city: deliveryApplied.city,
|
|
|
state: deliveryApplied.state,
|
|
|
zip: deliveryApplied.zip ?? "",
|
|
|
placeId: dc.option_id,
|
|
|
formattedAddress: deliveryApplied.formatted_address,
|
|
|
selectedFromSuggestions: true,
|
|
|
mothershipOptionId: dc.option_id,
|
|
|
mothershipDisplayLabel: dc.display_label,
|
|
|
selectedFromMothership: true,
|
|
|
},
|
|
|
palletCount: CARGO.palletCount,
|
|
|
weightLb: CARGO.weightLb,
|
|
|
dimsIn: { l: CARGO.dimsIn.length, w: CARGO.dimsIn.width, h: CARGO.dimsIn.height },
|
|
|
cargoType: CARGO.cargoType,
|
|
|
};
|
|
|
|
|
|
let opened: Awaited<ReturnType<typeof openQuotePage>> | null = null;
|
|
|
try {
|
|
|
opened = await openQuotePage(quotePageAdapter, false, undefined, {
|
|
|
freshCargoContext: true,
|
|
|
});
|
|
|
const ctx = RPAContext.fromSession(opened.page, opened.context);
|
|
|
await stabilizeQuoteLandingPage(ctx);
|
|
|
await waitForQuoterWidgetHydrated(ctx.page, 90_000);
|
|
|
|
|
|
const kernelStart = Date.now();
|
|
|
const items = await runMothershipQuoteKernel(ctx, req);
|
|
|
const kernelMs = Date.now() - kernelStart;
|
|
|
const finalUrl = opened.page.url();
|
|
|
const { comparisons, compareOk, shouldRetry } = compareToHuman(pair.humanQuotes, items);
|
|
|
const quotes = items.map((q) => ({
|
|
|
serviceLevel: q.serviceLevel,
|
|
|
rateOption: q.rateOption,
|
|
|
rawTotal: q.rawTotal,
|
|
|
carrier: q.carrier,
|
|
|
transitDays: q.transitDays,
|
|
|
}));
|
|
|
const ok = items.length > 0;
|
|
|
|
|
|
return {
|
|
|
...row,
|
|
|
attempts: attempt,
|
|
|
timing: { totalMs: Date.now() - caseStart, kernelMs },
|
|
|
comparisons,
|
|
|
quotes,
|
|
|
result: { ok, compareOk, tierCount: items.length, finalUrl },
|
|
|
error: ok
|
|
|
? compareOk
|
|
|
? undefined
|
|
|
: shouldRetry
|
|
|
? "与人工报价偏差过大（链路已成功）"
|
|
|
: undefined
|
|
|
: "未抓到报价档位",
|
|
|
};
|
|
|
} catch (e) {
|
|
|
return {
|
|
|
...row,
|
|
|
attempts: attempt,
|
|
|
timing: { totalMs: Date.now() - caseStart, kernelMs: 0 },
|
|
|
comparisons: [],
|
|
|
quotes: [],
|
|
|
result: { ok: false, compareOk: false, tierCount: 0, finalUrl: opened?.page.url() ?? "" },
|
|
|
error: e instanceof Error ? e.message : String(e),
|
|
|
};
|
|
|
} finally {
|
|
|
await opened?.page.close().catch(() => undefined);
|
|
|
await opened?.context.close().catch(() => undefined);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
async function runOneWithRetry(pair: HumanPair): Promise<RowResult> {
|
|
|
let last: RowResult | null = null;
|
|
|
for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt += 1) {
|
|
|
if (attempt > 1) {
|
|
|
const reason =
|
|
|
last?.quotes.length === 0
|
|
|
? "报价未抓到"
|
|
|
: last?.comparisons.some((c) => c.status === "retry")
|
|
|
? "与人工偏差过大"
|
|
|
: "链路失败";
|
|
|
console.log(`[human-bench] ${pair.id} ${reason}，重试 ${attempt}/${MAX_ATTEMPTS} …`);
|
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
|
}
|
|
|
last = await runOneCase(pair, attempt);
|
|
|
if (last.result.ok && last.quotes.length > 0) {
|
|
|
return last;
|
|
|
}
|
|
|
if (last.quotes.length === 0) continue;
|
|
|
}
|
|
|
return last!;
|
|
|
}
|
|
|
|
|
|
function printCmpLine(c: TierCmp): string {
|
|
|
if (c.rpa === null) return `${c.key}: 人工${fmtUsd(c.human)} RPA=缺失`;
|
|
|
const pct = c.pctDiff !== null ? `${(c.pctDiff * 100).toFixed(1)}%` : "—";
|
|
|
return `${c.key}: 人工${fmtUsd(c.human)} RPA${fmtUsd(c.rpa)} Δ${fmtUsd(c.absDiff ?? 0)}(${pct}) [${c.status}]`;
|
|
|
}
|
|
|
|
|
|
async function main(): Promise<void> {
|
|
|
const runId = randomUUID().slice(0, 8);
|
|
|
const startedAt = Date.now();
|
|
|
|
|
|
await waitForHttpOk(`${BASE}/`);
|
|
|
console.log(`[human-bench] 人工 docx 10 组 widget 对比 ${runId}`);
|
|
|
console.log(
|
|
|
`[human-bench] 链路: widget + visual-rush=${process.env.RPA_VISUAL_RUSH === "true"} dual-patch=${process.env.RPA_DUAL_ADDRESS_PATCH === "true"} | 货物: 2托 500lb 50×50×50 | 容忍 ≤${TIER_OK_PCT * 100}%或≤$${TIER_OK_ABS} | 重试 >${TIER_RETRY_PCT * 100}%且>$${TIER_RETRY_ABS}\n`,
|
|
|
);
|
|
|
|
|
|
const rows: RowResult[] = [];
|
|
|
|
|
|
const only = process.env.BENCH_HUMAN_ONLY?.trim();
|
|
|
const onlyIds = only
|
|
|
? only.split(/[,;]/).map((s) => s.trim()).filter(Boolean)
|
|
|
: null;
|
|
|
const pairs = onlyIds ? PAIRS.filter((p) => onlyIds.includes(p.id)) : PAIRS;
|
|
|
if (pairs.length === 0) {
|
|
|
throw new Error(`BENCH_HUMAN_ONLY 无效: ${only}`);
|
|
|
}
|
|
|
|
|
|
for (let i = 0; i < pairs.length; i += 1) {
|
|
|
const pair = pairs[i]!;
|
|
|
console.log(`[human-bench] ${i + 1}/${pairs.length} ${pair.route} …`);
|
|
|
const result = await runOneWithRetry(pair);
|
|
|
rows.push(result);
|
|
|
console.log(
|
|
|
` → ${result.result.ok ? "OK" : "FAIL"} attempts=${result.attempts} tiers=${result.result.tierCount} ${fmtMs(result.timing.totalMs)}`,
|
|
|
);
|
|
|
console.log(` pickup: ${result.pickup.selectedSuggestion || "—"}`);
|
|
|
console.log(` delivery: ${result.delivery.selectedSuggestion || "—"}`);
|
|
|
for (const c of result.comparisons) {
|
|
|
console.log(` ${printCmpLine(c)}`);
|
|
|
}
|
|
|
if (result.error) console.log(` err: ${result.error}`);
|
|
|
}
|
|
|
|
|
|
await closeBrowser();
|
|
|
|
|
|
const pass = rows.filter((r) => r.result.ok).length;
|
|
|
const report = {
|
|
|
runId,
|
|
|
sourceDoc: "docs/美国跨州货物运输报价汇总_edited3.docx",
|
|
|
finishedAt: new Date().toISOString(),
|
|
|
durationMs: Date.now() - startedAt,
|
|
|
tolerance: { okPct: TIER_OK_PCT, okAbs: TIER_OK_ABS, retryPct: TIER_RETRY_PCT, retryAbs: TIER_RETRY_ABS },
|
|
|
summary: { pass, fail: rows.length - pass },
|
|
|
cargoTemplate: CARGO,
|
|
|
env: {
|
|
|
quoteStrategy: "direct-first-widget-fallback",
|
|
|
RPA_VISUAL_RUSH: process.env.RPA_VISUAL_RUSH === "true",
|
|
|
RPA_HUMAN_SUBMIT_STRICT: process.env.RPA_HUMAN_SUBMIT_STRICT === "true",
|
|
|
RPA_REQUIRE_QUOTE_RATES: true,
|
|
|
RPA_BENCHMARK_QUOTE_MS: 45_000,
|
|
|
RPA_HEADLESS: process.env.RPA_HEADLESS === "true",
|
|
|
},
|
|
|
rows,
|
|
|
};
|
|
|
|
|
|
const outDir = join(process.cwd(), ".rpa", "human-doc-benchmark");
|
|
|
mkdirSync(outDir, { recursive: true });
|
|
|
const outPath = join(outDir, `human-${runId}.json`);
|
|
|
writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
|
|
|
|
console.log("\n| # | ID | Route | Result | Attempts | 比对 |");
|
|
|
console.log("|---|-----|-------|--------|----------|------|");
|
|
|
for (let i = 0; i < rows.length; i += 1) {
|
|
|
const r = rows[i]!;
|
|
|
const cmpSummary = r.comparisons
|
|
|
.map((c) => `${c.key.split("/")[1]}:${c.status === "match" ? "✓" : c.status === "close" ? "~" : "✗"}`)
|
|
|
.join(" ");
|
|
|
console.log(
|
|
|
`| ${i + 1} | ${r.id} | ${r.route} | ${r.result.ok ? "PASS" : "FAIL"} | ${r.attempts} | ${cmpSummary} |`,
|
|
|
);
|
|
|
}
|
|
|
console.log(`\n[human-bench] 报告: ${outPath}`);
|
|
|
process.exit(pass === rows.length ? 0 : 1);
|
|
|
}
|
|
|
|
|
|
main().catch(async (e) => {
|
|
|
console.error(e);
|
|
|
await closeBrowser().catch(() => undefined);
|
|
|
process.exit(1);
|
|
|
});
|
