#!/usr/bin/env tsx /** * 10 组固定随机地址全链耗时基准(无头 + visual-rush) * 输出:英文路线、货物、选中联想地址、各档真实报价金额 * * npm run benchmark:quote-10pairs */ import { randomUUID } from "node:crypto"; import { mkdirSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { loadDevEnv } from "@/lib/dev-env"; import { hostFetchMothershipCandidates } from "@/lib/frontend/api-client"; import { applyMothershipCandidate } from "@/lib/frontend/mothership-address"; import { RPAContext } from "@/workers/rpa/kernel/context"; import { runMothershipQuoteKernel } from "@/workers/rpa/kernel/orchestrator"; import { stabilizeQuoteLandingPage, waitForQuoterWidgetHydrated, } from "@/workers/rpa/page-prep"; import { closeBrowser, openQuotePage } from "@/workers/rpa/session-manager"; import { quotePageAdapter } from "@/workers/rpa/quote-page-adapter"; import type { QuoteItem, QuoteRequest } from "@/modules/providers/quote-provider"; process.env.RPA_MOCK_MODE = "false"; process.env.RPA_ADDRESS_MODE = "real"; process.env.RPA_VISUAL_RUSH = "true"; process.env.RPA_DUAL_ADDRESS_PATCH = "false"; process.env.RPA_HEADLESS = "true"; process.env.RPA_KEEP_BROWSER_OPEN = "false"; process.env.RPA_SLOW_MO_MS = "0"; process.env.RPA_HUMAN_PACING = "false"; /** 基准拉长抓价窗口;首组冷启动 widget 较慢 */ process.env.RPA_BENCHMARK_QUOTE_MS = "45000"; process.env.RPA_REQUIRE_QUOTE_RATES = "true"; loadDevEnv(); const BASE = process.env.PROBE_BASE_URL ?? "http://localhost:3000"; const TOKEN = "demo-host-token"; const CUSTOMER_ID = "CUST_001"; const CARGO = { palletCount: 2, weightLb: 500, dimsIn: { length: 48, width: 40, height: 48 }, cargoType: "general_freight" as const, }; /** 与录证/探针池不同的 10 组路线(英文标签) */ const PAIRS = [ { id: "B01", route: "Portland OR → Austin TX", pickup: { street: "1221 SW 4th Ave", city: "Portland", state: "OR" }, delivery: { street: "301 Congress Ave", city: "Austin", state: "TX" }, }, { id: "B02", route: "Houston TX → San Antonio TX", pickup: { street: "901 Bagby St", city: "Houston", state: "TX" }, delivery: { street: "300 Alamo Plaza", city: "San Antonio", state: "TX" }, }, { id: "B03", route: "Philadelphia PA → Baltimore MD", pickup: { street: "1500 Market St", city: "Philadelphia", state: "PA" }, delivery: { street: "100 Light St", city: "Baltimore", state: "MD" }, }, { id: "B04", route: "Detroit MI → Columbus OH", pickup: { street: "1 Woodward Ave", city: "Detroit", state: "MI" }, delivery: { street: "50 W Gay St", city: "Columbus", state: "OH" }, }, { id: "B05", route: "Kansas City MO → St Louis MO", pickup: { street: "1100 Walnut St", city: "Kansas City", state: "MO" }, delivery: { street: "701 Clark Ave", city: "St Louis", state: "MO" }, }, { id: "B06", route: "Minneapolis MN → Milwaukee WI", pickup: { street: "225 S 6th St", city: "Minneapolis", state: "MN" }, delivery: { street: "777 E Wisconsin Ave", city: "Milwaukee", state: "WI" }, }, { id: "B07", route: "New Orleans LA → Houston TX", pickup: { street: "600 Decatur St", city: "New Orleans", state: "LA" }, delivery: { street: "1600 Smith St", city: "Houston", state: "TX" }, }, { id: "B08", route: "Salt Lake City UT → Boise ID", pickup: { street: "50 S Main St", city: "Salt Lake City", state: "UT" }, delivery: { street: "150 N Capitol Blvd", city: "Boise", state: "ID" }, }, { id: "B09", route: "Richmond VA → Raleigh NC", pickup: { street: "701 E Broad St", city: "Richmond", state: "VA" }, delivery: { street: "222 Fayetteville St", city: "Raleigh", state: "NC" }, }, { id: "B10", route: "Albuquerque NM → Tucson AZ", pickup: { street: "400 Marquette Ave NW", city: "Albuquerque", state: "NM" }, delivery: { street: "50 E Congress St", city: "Tucson", state: "AZ" }, }, ] as const; type BenchQuote = { serviceLevel: QuoteItem["serviceLevel"]; rateOption: QuoteItem["rateOption"]; carrier: string; transitDays: string; transitDescription: string; rawFreight: number; surcharges: number; rawTotal: number; }; type BenchRow = { id: string; route: string; pickup: { input: { street: string; city: string; state: string }; selectedSuggestion: string; optionId: string; formattedAddress: string; }; delivery: { input: { street: string; city: string; state: string }; selectedSuggestion: string; optionId: string; formattedAddress: string; }; cargo: typeof CARGO; timing: { totalMs: number; kernelMs: number }; result: { ok: boolean; finalUrl: string; quotePageReached: boolean; tierCount: number }; quotes: BenchQuote[]; error?: string; }; function draftAddress(d: { street: string; city: string; state: string }) { return { street: d.street.trim(), city: d.city.trim(), state: d.state.trim(), zip: "", place_id: `draft_${d.street}_${d.city}_${d.state}`.replace(/\s+/g, "_").slice(0, 64), formatted_address: `${d.street}, ${d.city}, ${d.state}`, selected_from_suggestions: true, }; } function fmtMs(ms: number): string { if (ms < 60_000) { return `${(ms / 1000).toFixed(1)}s`; } const m = Math.floor(ms / 60_000); const s = ((ms % 60_000) / 1000).toFixed(0); return `${m}m${s}s`; } function fmtUsd(n: number): string { return `$${n.toFixed(2)}`; } function toBenchQuotes(items: QuoteItem[]): BenchQuote[] { return items.map((q) => ({ serviceLevel: q.serviceLevel, rateOption: q.rateOption, carrier: q.carrier, transitDays: q.transitDays, transitDescription: q.transitDescription, rawFreight: q.rawFreight, surcharges: q.surcharges, rawTotal: q.rawTotal, })); } async function waitForHttpOk(url: string, attempts = 20): Promise { for (let i = 0; i < attempts; i += 1) { try { const res = await fetch(url); if (res.ok) { return; } } catch { /* retry */ } await new Promise((r) => setTimeout(r, 2000)); } throw new Error(`服务未就绪: ${url}`); } async function fetchCandidatesWithRetry( pickup: ReturnType, delivery: ReturnType, attempts = 3, ) { let lastErr = "候选 API 失败"; for (let i = 0; i < attempts; i += 1) { try { const cand = await hostFetchMothershipCandidates( BASE, TOKEN, CUSTOMER_ID, pickup, delivery, ); if (cand.code === 0 && cand.data?.pickup_candidates[0] && cand.data.delivery_candidates[0]) { return cand; } lastErr = cand.message ?? lastErr; } catch (e) { lastErr = e instanceof Error ? e.message : String(e); } if (i < attempts - 1) { await new Promise((r) => setTimeout(r, 2000)); } } return { code: -1, message: lastErr, data: null }; } async function runOneCase(pair: (typeof PAIRS)[number]): Promise { const caseStart = Date.now(); const userPickup = draftAddress(pair.pickup); const userDelivery = draftAddress(pair.delivery); const baseRow = (): Omit => ({ id: pair.id, route: pair.route, pickup: { input: { ...pair.pickup }, selectedSuggestion: "", optionId: "", formattedAddress: "", }, delivery: { input: { ...pair.delivery }, selectedSuggestion: "", optionId: "", formattedAddress: "", }, cargo: { ...CARGO }, }); const cand = await fetchCandidatesWithRetry(userPickup, userDelivery); if (cand.code !== 0 || !cand.data?.pickup_candidates[0] || !cand.data.delivery_candidates[0]) { return { ...baseRow(), timing: { totalMs: Date.now() - caseStart, kernelMs: 0 }, result: { ok: false, finalUrl: "", quotePageReached: false, tierCount: 0 }, quotes: [], error: cand.message ?? "候选 API 失败", }; } const pc = cand.data.pickup_candidates[0]; const dc = cand.data.delivery_candidates[0]; const pickupApplied = applyMothershipCandidate(userPickup, pc); const deliveryApplied = applyMothershipCandidate(userDelivery, dc); const row = baseRow(); row.pickup.selectedSuggestion = pc.display_label; row.pickup.optionId = pc.option_id; row.pickup.formattedAddress = pickupApplied.formatted_address; row.delivery.selectedSuggestion = dc.display_label; row.delivery.optionId = dc.option_id; row.delivery.formattedAddress = deliveryApplied.formatted_address; const req: QuoteRequest = { cargoHash: `bench_${pair.id}_${randomUUID().slice(0, 8)}`, pickup: { street: pickupApplied.street, city: pickupApplied.city, state: pickupApplied.state, zip: pickupApplied.zip ?? "", placeId: pc.option_id, formattedAddress: pickupApplied.formatted_address, selectedFromSuggestions: true, mothershipOptionId: pc.option_id, mothershipDisplayLabel: pc.display_label, selectedFromMothership: true, }, delivery: { street: deliveryApplied.street, city: deliveryApplied.city, state: deliveryApplied.state, zip: deliveryApplied.zip ?? "", placeId: dc.option_id, formattedAddress: deliveryApplied.formatted_address, selectedFromSuggestions: true, mothershipOptionId: dc.option_id, mothershipDisplayLabel: dc.display_label, selectedFromMothership: true, }, palletCount: CARGO.palletCount, weightLb: CARGO.weightLb, dimsIn: { l: CARGO.dimsIn.length, w: CARGO.dimsIn.width, h: CARGO.dimsIn.height }, cargoType: CARGO.cargoType, }; let opened: Awaited> | null = null; try { opened = await openQuotePage(quotePageAdapter, false, undefined, { freshCargoContext: true, }); const ctx = RPAContext.fromSession(opened.page, opened.context); await stabilizeQuoteLandingPage(ctx); await waitForQuoterWidgetHydrated(ctx.page, 90_000); const kernelStart = Date.now(); const items = await runMothershipQuoteKernel(ctx, req); const kernelMs = Date.now() - kernelStart; const finalUrl = opened.page.url(); const quotePageReached = finalUrl.includes("sign-up-quote"); const tierCount = items.length; const ok = tierCount > 0; return { ...row, timing: { totalMs: Date.now() - caseStart, kernelMs }, result: { ok, finalUrl, quotePageReached, tierCount }, quotes: toBenchQuotes(items), error: ok ? undefined : quotePageReached ? "已落 sign-up-quote 但未抓到报价档位" : "无报价档位且未落 sign-up-quote", }; } catch (e) { return { ...row, timing: { totalMs: Date.now() - caseStart, kernelMs: 0 }, result: { ok: false, finalUrl: opened?.page.url() ?? "", quotePageReached: (opened?.page.url() ?? "").includes("sign-up-quote"), tierCount: 0, }, quotes: [], error: e instanceof Error ? e.message : String(e), }; } finally { await opened?.page.close().catch(() => undefined); await opened?.context.close().catch(() => undefined); } } function printQuoteSummary(quotes: BenchQuote[]): string { if (quotes.length === 0) { return "—"; } return quotes .map((q) => `${q.serviceLevel}/${q.rateOption}=${fmtUsd(q.rawTotal)}`) .join("; "); } async function runOneCaseWithRetry( pair: (typeof PAIRS)[number], maxAttempts = 3, ): Promise { let last: BenchRow | null = null; for (let attempt = 1; attempt <= maxAttempts; attempt += 1) { if (attempt > 1) { console.log(`[bench] ${pair.id} 重试 ${attempt}/${maxAttempts} …`); await new Promise((r) => setTimeout(r, 3000)); } last = await runOneCase(pair); if (last.result.ok && last.quotes.length > 0) { return last; } } return last!; } async function main(): Promise { const runId = randomUUID().slice(0, 8); const startedAt = Date.now(); await waitForHttpOk(`${BASE}/`); console.log(`[bench] 10-pair quote benchmark ${runId} (headless + visual-rush)\n`); const rows: BenchRow[] = []; for (let i = 0; i < PAIRS.length; i += 1) { const pair = PAIRS[i]!; console.log(`[bench] ${i + 1}/10 ${pair.route} …`); const result = await runOneCaseWithRetry(pair); rows.push(result); console.log( ` → ${result.result.ok ? "OK" : "FAIL"} total=${fmtMs(result.timing.totalMs)} kernel=${fmtMs(result.timing.kernelMs)} tiers=${result.result.tierCount}`, ); console.log(` pickup: ${result.pickup.selectedSuggestion || "—"}`); console.log(` delivery: ${result.delivery.selectedSuggestion || "—"}`); console.log(` quotes: ${printQuoteSummary(result.quotes)}`); if (result.error) { console.log(` err: ${result.error.slice(0, 80)}`); } } await closeBrowser(); const pass = rows.filter((r) => r.result.ok && r.quotes.length > 0).length; const withRates = pass; const avgTotal = Math.round(rows.reduce((s, r) => s + r.timing.totalMs, 0) / rows.length); const avgKernel = Math.round(rows.reduce((s, r) => s + r.timing.kernelMs, 0) / rows.length); const report = { runId, finishedAt: new Date().toISOString(), durationMs: Date.now() - startedAt, summary: { pass, fail: rows.length - pass, withRates, avgTotalMs: avgTotal, avgKernelMs: avgKernel, }, cargoTemplate: CARGO, env: { RPA_HEADLESS: true, RPA_VISUAL_RUSH: true, RPA_DUAL_ADDRESS_PATCH: false, RPA_BENCHMARK_QUOTE_MS: 45_000, RPA_REQUIRE_QUOTE_RATES: true, }, rows, }; const outDir = join(process.cwd(), ".rpa", "benchmark-10pairs"); mkdirSync(outDir, { recursive: true }); const outPath = join(outDir, `bench-${runId}.json`); writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); /** 固定文件名供对比/引用 */ const canonicalPath = join(outDir, "bench-1cdb927f.json"); writeFileSync(canonicalPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); console.log("\n| # | ID | Route | Result | Total | Kernel | Rates |"); console.log("|---|-----|-------|--------|-------|--------|-------|"); for (let i = 0; i < rows.length; i += 1) { const r = rows[i]!; console.log( `| ${i + 1} | ${r.id} | ${r.route} | ${r.result.ok ? "PASS" : "FAIL"} | ${fmtMs(r.timing.totalMs)} | ${fmtMs(r.timing.kernelMs)} | ${printQuoteSummary(r.quotes)} |`, ); } console.log( `\n| avg | — | — | ${pass}/10 (${withRates} with $) | ${fmtMs(avgTotal)} | ${fmtMs(avgKernel)} | — |`, ); console.log(`\n[bench] report: ${outPath}`); console.log(`[bench] canonical: ${canonicalPath}`); process.exit(pass === rows.length ? 0 : 1); } main().catch(async (e) => { console.error(e); await closeBrowser().catch(() => undefined); process.exit(1); });