|
|
#!/usr/bin/env tsx
|
|
|
/**
|
|
|
* 10 组固定随机地址全链耗时基准（无头 + visual-rush）
|
|
|
* 输出：英文路线、货物、选中联想地址、各档真实报价金额
|
|
|
*
|
|
|
* npm run benchmark:quote-10pairs
|
|
|
*/
|
|
|
import { randomUUID } from "node:crypto";
|
|
|
import { mkdirSync, writeFileSync } from "node:fs";
|
|
|
import { join } from "node:path";
|
|
|
import { loadDevEnv } from "@/lib/dev-env";
|
|
|
import { hostFetchMothershipCandidates } from "@/lib/frontend/api-client";
|
|
|
import { applyMothershipCandidate } from "@/lib/frontend/mothership-address";
|
|
|
import { RPAContext } from "@/workers/rpa/kernel/context";
|
|
|
import { runMothershipQuoteKernel } from "@/workers/rpa/kernel/orchestrator";
|
|
|
import {
|
|
|
stabilizeQuoteLandingPage,
|
|
|
waitForQuoterWidgetHydrated,
|
|
|
} from "@/workers/rpa/page-prep";
|
|
|
import { closeBrowser, openQuotePage } from "@/workers/rpa/session-manager";
|
|
|
import { quotePageAdapter } from "@/workers/rpa/quote-page-adapter";
|
|
|
import type { QuoteItem, QuoteRequest } from "@/modules/providers/quote-provider";
|
|
|
|
|
|
process.env.RPA_MOCK_MODE = "false";
|
|
|
process.env.RPA_ADDRESS_MODE = "real";
|
|
|
process.env.RPA_VISUAL_RUSH = "true";
|
|
|
process.env.RPA_DUAL_ADDRESS_PATCH = "false";
|
|
|
process.env.RPA_HEADLESS = "true";
|
|
|
process.env.RPA_KEEP_BROWSER_OPEN = "false";
|
|
|
process.env.RPA_SLOW_MO_MS = "0";
|
|
|
process.env.RPA_HUMAN_PACING = "false";
|
|
|
/** 基准拉长抓价窗口；首组冷启动 widget 较慢 */
|
|
|
process.env.RPA_BENCHMARK_QUOTE_MS = "45000";
|
|
|
process.env.RPA_REQUIRE_QUOTE_RATES = "true";
|
|
|
|
|
|
loadDevEnv();
|
|
|
|
|
|
const BASE = process.env.PROBE_BASE_URL ?? "http://localhost:3000";
|
|
|
const TOKEN = "demo-host-token";
|
|
|
const CUSTOMER_ID = "CUST_001";
|
|
|
|
|
|
const CARGO = {
|
|
|
palletCount: 2,
|
|
|
weightLb: 500,
|
|
|
dimsIn: { length: 48, width: 40, height: 48 },
|
|
|
cargoType: "general_freight" as const,
|
|
|
};
|
|
|
|
|
|
/** 与录证/探针池不同的 10 组路线（英文标签） */
|
|
|
const PAIRS = [
|
|
|
{
|
|
|
id: "B01",
|
|
|
route: "Portland OR → Austin TX",
|
|
|
pickup: { street: "1221 SW 4th Ave", city: "Portland", state: "OR" },
|
|
|
delivery: { street: "301 Congress Ave", city: "Austin", state: "TX" },
|
|
|
},
|
|
|
{
|
|
|
id: "B02",
|
|
|
route: "Houston TX → San Antonio TX",
|
|
|
pickup: { street: "901 Bagby St", city: "Houston", state: "TX" },
|
|
|
delivery: { street: "300 Alamo Plaza", city: "San Antonio", state: "TX" },
|
|
|
},
|
|
|
{
|
|
|
id: "B03",
|
|
|
route: "Philadelphia PA → Baltimore MD",
|
|
|
pickup: { street: "1500 Market St", city: "Philadelphia", state: "PA" },
|
|
|
delivery: { street: "100 Light St", city: "Baltimore", state: "MD" },
|
|
|
},
|
|
|
{
|
|
|
id: "B04",
|
|
|
route: "Detroit MI → Columbus OH",
|
|
|
pickup: { street: "1 Woodward Ave", city: "Detroit", state: "MI" },
|
|
|
delivery: { street: "50 W Gay St", city: "Columbus", state: "OH" },
|
|
|
},
|
|
|
{
|
|
|
id: "B05",
|
|
|
route: "Kansas City MO → St Louis MO",
|
|
|
pickup: { street: "1100 Walnut St", city: "Kansas City", state: "MO" },
|
|
|
delivery: { street: "701 Clark Ave", city: "St Louis", state: "MO" },
|
|
|
},
|
|
|
{
|
|
|
id: "B06",
|
|
|
route: "Minneapolis MN → Milwaukee WI",
|
|
|
pickup: { street: "225 S 6th St", city: "Minneapolis", state: "MN" },
|
|
|
delivery: { street: "777 E Wisconsin Ave", city: "Milwaukee", state: "WI" },
|
|
|
},
|
|
|
{
|
|
|
id: "B07",
|
|
|
route: "New Orleans LA → Houston TX",
|
|
|
pickup: { street: "600 Decatur St", city: "New Orleans", state: "LA" },
|
|
|
delivery: { street: "1600 Smith St", city: "Houston", state: "TX" },
|
|
|
},
|
|
|
{
|
|
|
id: "B08",
|
|
|
route: "Salt Lake City UT → Boise ID",
|
|
|
pickup: { street: "50 S Main St", city: "Salt Lake City", state: "UT" },
|
|
|
delivery: { street: "150 N Capitol Blvd", city: "Boise", state: "ID" },
|
|
|
},
|
|
|
{
|
|
|
id: "B09",
|
|
|
route: "Richmond VA → Raleigh NC",
|
|
|
pickup: { street: "701 E Broad St", city: "Richmond", state: "VA" },
|
|
|
delivery: { street: "222 Fayetteville St", city: "Raleigh", state: "NC" },
|
|
|
},
|
|
|
{
|
|
|
id: "B10",
|
|
|
route: "Albuquerque NM → Tucson AZ",
|
|
|
pickup: { street: "400 Marquette Ave NW", city: "Albuquerque", state: "NM" },
|
|
|
delivery: { street: "50 E Congress St", city: "Tucson", state: "AZ" },
|
|
|
},
|
|
|
] as const;
|
|
|
|
|
|
type BenchQuote = {
|
|
|
serviceLevel: QuoteItem["serviceLevel"];
|
|
|
rateOption: QuoteItem["rateOption"];
|
|
|
carrier: string;
|
|
|
transitDays: string;
|
|
|
transitDescription: string;
|
|
|
rawFreight: number;
|
|
|
surcharges: number;
|
|
|
rawTotal: number;
|
|
|
};
|
|
|
|
|
|
type BenchRow = {
|
|
|
id: string;
|
|
|
route: string;
|
|
|
pickup: {
|
|
|
input: { street: string; city: string; state: string };
|
|
|
selectedSuggestion: string;
|
|
|
optionId: string;
|
|
|
formattedAddress: string;
|
|
|
};
|
|
|
delivery: {
|
|
|
input: { street: string; city: string; state: string };
|
|
|
selectedSuggestion: string;
|
|
|
optionId: string;
|
|
|
formattedAddress: string;
|
|
|
};
|
|
|
cargo: typeof CARGO;
|
|
|
timing: { totalMs: number; kernelMs: number };
|
|
|
result: { ok: boolean; finalUrl: string; quotePageReached: boolean; tierCount: number };
|
|
|
quotes: BenchQuote[];
|
|
|
error?: string;
|
|
|
};
|
|
|
|
|
|
function draftAddress(d: { street: string; city: string; state: string }) {
|
|
|
return {
|
|
|
street: d.street.trim(),
|
|
|
city: d.city.trim(),
|
|
|
state: d.state.trim(),
|
|
|
zip: "",
|
|
|
place_id: `draft_${d.street}_${d.city}_${d.state}`.replace(/\s+/g, "_").slice(0, 64),
|
|
|
formatted_address: `${d.street}, ${d.city}, ${d.state}`,
|
|
|
selected_from_suggestions: true,
|
|
|
};
|
|
|
}
|
|
|
|
|
|
function fmtMs(ms: number): string {
|
|
|
if (ms < 60_000) {
|
|
|
return `${(ms / 1000).toFixed(1)}s`;
|
|
|
}
|
|
|
const m = Math.floor(ms / 60_000);
|
|
|
const s = ((ms % 60_000) / 1000).toFixed(0);
|
|
|
return `${m}m${s}s`;
|
|
|
}
|
|
|
|
|
|
function fmtUsd(n: number): string {
|
|
|
return `$${n.toFixed(2)}`;
|
|
|
}
|
|
|
|
|
|
function toBenchQuotes(items: QuoteItem[]): BenchQuote[] {
|
|
|
return items.map((q) => ({
|
|
|
serviceLevel: q.serviceLevel,
|
|
|
rateOption: q.rateOption,
|
|
|
carrier: q.carrier,
|
|
|
transitDays: q.transitDays,
|
|
|
transitDescription: q.transitDescription,
|
|
|
rawFreight: q.rawFreight,
|
|
|
surcharges: q.surcharges,
|
|
|
rawTotal: q.rawTotal,
|
|
|
}));
|
|
|
}
|
|
|
|
|
|
async function waitForHttpOk(url: string, attempts = 20): Promise<void> {
|
|
|
for (let i = 0; i < attempts; i += 1) {
|
|
|
try {
|
|
|
const res = await fetch(url);
|
|
|
if (res.ok) {
|
|
|
return;
|
|
|
}
|
|
|
} catch {
|
|
|
/* retry */
|
|
|
}
|
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
}
|
|
|
throw new Error(`服务未就绪: ${url}`);
|
|
|
}
|
|
|
|
|
|
async function fetchCandidatesWithRetry(
|
|
|
pickup: ReturnType<typeof draftAddress>,
|
|
|
delivery: ReturnType<typeof draftAddress>,
|
|
|
attempts = 3,
|
|
|
) {
|
|
|
let lastErr = "候选 API 失败";
|
|
|
for (let i = 0; i < attempts; i += 1) {
|
|
|
try {
|
|
|
const cand = await hostFetchMothershipCandidates(
|
|
|
BASE,
|
|
|
TOKEN,
|
|
|
CUSTOMER_ID,
|
|
|
pickup,
|
|
|
delivery,
|
|
|
);
|
|
|
if (cand.code === 0 && cand.data?.pickup_candidates[0] && cand.data.delivery_candidates[0]) {
|
|
|
return cand;
|
|
|
}
|
|
|
lastErr = cand.message ?? lastErr;
|
|
|
} catch (e) {
|
|
|
lastErr = e instanceof Error ? e.message : String(e);
|
|
|
}
|
|
|
if (i < attempts - 1) {
|
|
|
await new Promise((r) => setTimeout(r, 2000));
|
|
|
}
|
|
|
}
|
|
|
return { code: -1, message: lastErr, data: null };
|
|
|
}
|
|
|
|
|
|
async function runOneCase(pair: (typeof PAIRS)[number]): Promise<BenchRow> {
|
|
|
const caseStart = Date.now();
|
|
|
const userPickup = draftAddress(pair.pickup);
|
|
|
const userDelivery = draftAddress(pair.delivery);
|
|
|
|
|
|
const baseRow = (): Omit<BenchRow, "timing" | "result" | "quotes"> => ({
|
|
|
id: pair.id,
|
|
|
route: pair.route,
|
|
|
pickup: {
|
|
|
input: { ...pair.pickup },
|
|
|
selectedSuggestion: "",
|
|
|
optionId: "",
|
|
|
formattedAddress: "",
|
|
|
},
|
|
|
delivery: {
|
|
|
input: { ...pair.delivery },
|
|
|
selectedSuggestion: "",
|
|
|
optionId: "",
|
|
|
formattedAddress: "",
|
|
|
},
|
|
|
cargo: { ...CARGO },
|
|
|
});
|
|
|
|
|
|
const cand = await fetchCandidatesWithRetry(userPickup, userDelivery);
|
|
|
if (cand.code !== 0 || !cand.data?.pickup_candidates[0] || !cand.data.delivery_candidates[0]) {
|
|
|
return {
|
|
|
...baseRow(),
|
|
|
timing: { totalMs: Date.now() - caseStart, kernelMs: 0 },
|
|
|
result: { ok: false, finalUrl: "", quotePageReached: false, tierCount: 0 },
|
|
|
quotes: [],
|
|
|
error: cand.message ?? "候选 API 失败",
|
|
|
};
|
|
|
}
|
|
|
|
|
|
const pc = cand.data.pickup_candidates[0];
|
|
|
const dc = cand.data.delivery_candidates[0];
|
|
|
const pickupApplied = applyMothershipCandidate(userPickup, pc);
|
|
|
const deliveryApplied = applyMothershipCandidate(userDelivery, dc);
|
|
|
|
|
|
const row = baseRow();
|
|
|
row.pickup.selectedSuggestion = pc.display_label;
|
|
|
row.pickup.optionId = pc.option_id;
|
|
|
row.pickup.formattedAddress = pickupApplied.formatted_address;
|
|
|
row.delivery.selectedSuggestion = dc.display_label;
|
|
|
row.delivery.optionId = dc.option_id;
|
|
|
row.delivery.formattedAddress = deliveryApplied.formatted_address;
|
|
|
|
|
|
const req: QuoteRequest = {
|
|
|
cargoHash: `bench_${pair.id}_${randomUUID().slice(0, 8)}`,
|
|
|
pickup: {
|
|
|
street: pickupApplied.street,
|
|
|
city: pickupApplied.city,
|
|
|
state: pickupApplied.state,
|
|
|
zip: pickupApplied.zip ?? "",
|
|
|
placeId: pc.option_id,
|
|
|
formattedAddress: pickupApplied.formatted_address,
|
|
|
selectedFromSuggestions: true,
|
|
|
mothershipOptionId: pc.option_id,
|
|
|
mothershipDisplayLabel: pc.display_label,
|
|
|
selectedFromMothership: true,
|
|
|
},
|
|
|
delivery: {
|
|
|
street: deliveryApplied.street,
|
|
|
city: deliveryApplied.city,
|
|
|
state: deliveryApplied.state,
|
|
|
zip: deliveryApplied.zip ?? "",
|
|
|
placeId: dc.option_id,
|
|
|
formattedAddress: deliveryApplied.formatted_address,
|
|
|
selectedFromSuggestions: true,
|
|
|
mothershipOptionId: dc.option_id,
|
|
|
mothershipDisplayLabel: dc.display_label,
|
|
|
selectedFromMothership: true,
|
|
|
},
|
|
|
palletCount: CARGO.palletCount,
|
|
|
weightLb: CARGO.weightLb,
|
|
|
dimsIn: { l: CARGO.dimsIn.length, w: CARGO.dimsIn.width, h: CARGO.dimsIn.height },
|
|
|
cargoType: CARGO.cargoType,
|
|
|
};
|
|
|
|
|
|
let opened: Awaited<ReturnType<typeof openQuotePage>> | null = null;
|
|
|
try {
|
|
|
opened = await openQuotePage(quotePageAdapter, false, undefined, {
|
|
|
freshCargoContext: true,
|
|
|
});
|
|
|
const ctx = RPAContext.fromSession(opened.page, opened.context);
|
|
|
await stabilizeQuoteLandingPage(ctx);
|
|
|
await waitForQuoterWidgetHydrated(ctx.page, 90_000);
|
|
|
|
|
|
const kernelStart = Date.now();
|
|
|
const items = await runMothershipQuoteKernel(ctx, req);
|
|
|
const kernelMs = Date.now() - kernelStart;
|
|
|
const finalUrl = opened.page.url();
|
|
|
const quotePageReached = finalUrl.includes("sign-up-quote");
|
|
|
const tierCount = items.length;
|
|
|
const ok = tierCount > 0;
|
|
|
|
|
|
return {
|
|
|
...row,
|
|
|
timing: { totalMs: Date.now() - caseStart, kernelMs },
|
|
|
result: { ok, finalUrl, quotePageReached, tierCount },
|
|
|
quotes: toBenchQuotes(items),
|
|
|
error: ok ? undefined : quotePageReached ? "已落 sign-up-quote 但未抓到报价档位" : "无报价档位且未落 sign-up-quote",
|
|
|
};
|
|
|
} catch (e) {
|
|
|
return {
|
|
|
...row,
|
|
|
timing: { totalMs: Date.now() - caseStart, kernelMs: 0 },
|
|
|
result: {
|
|
|
ok: false,
|
|
|
finalUrl: opened?.page.url() ?? "",
|
|
|
quotePageReached: (opened?.page.url() ?? "").includes("sign-up-quote"),
|
|
|
tierCount: 0,
|
|
|
},
|
|
|
quotes: [],
|
|
|
error: e instanceof Error ? e.message : String(e),
|
|
|
};
|
|
|
} finally {
|
|
|
await opened?.page.close().catch(() => undefined);
|
|
|
await opened?.context.close().catch(() => undefined);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
function printQuoteSummary(quotes: BenchQuote[]): string {
|
|
|
if (quotes.length === 0) {
|
|
|
return "—";
|
|
|
}
|
|
|
return quotes
|
|
|
.map((q) => `${q.serviceLevel}/${q.rateOption}=${fmtUsd(q.rawTotal)}`)
|
|
|
.join("; ");
|
|
|
}
|
|
|
|
|
|
async function runOneCaseWithRetry(
|
|
|
pair: (typeof PAIRS)[number],
|
|
|
maxAttempts = 3,
|
|
|
): Promise<BenchRow> {
|
|
|
let last: BenchRow | null = null;
|
|
|
for (let attempt = 1; attempt <= maxAttempts; attempt += 1) {
|
|
|
if (attempt > 1) {
|
|
|
console.log(`[bench] ${pair.id} 重试 ${attempt}/${maxAttempts} …`);
|
|
|
await new Promise((r) => setTimeout(r, 3000));
|
|
|
}
|
|
|
last = await runOneCase(pair);
|
|
|
if (last.result.ok && last.quotes.length > 0) {
|
|
|
return last;
|
|
|
}
|
|
|
}
|
|
|
return last!;
|
|
|
}
|
|
|
|
|
|
async function main(): Promise<void> {
|
|
|
const runId = randomUUID().slice(0, 8);
|
|
|
const startedAt = Date.now();
|
|
|
|
|
|
await waitForHttpOk(`${BASE}/`);
|
|
|
console.log(`[bench] 10-pair quote benchmark ${runId} (headless + visual-rush)\n`);
|
|
|
|
|
|
const rows: BenchRow[] = [];
|
|
|
|
|
|
for (let i = 0; i < PAIRS.length; i += 1) {
|
|
|
const pair = PAIRS[i]!;
|
|
|
console.log(`[bench] ${i + 1}/10 ${pair.route} …`);
|
|
|
const result = await runOneCaseWithRetry(pair);
|
|
|
rows.push(result);
|
|
|
console.log(
|
|
|
` → ${result.result.ok ? "OK" : "FAIL"} total=${fmtMs(result.timing.totalMs)} kernel=${fmtMs(result.timing.kernelMs)} tiers=${result.result.tierCount}`,
|
|
|
);
|
|
|
console.log(` pickup: ${result.pickup.selectedSuggestion || "—"}`);
|
|
|
console.log(` delivery: ${result.delivery.selectedSuggestion || "—"}`);
|
|
|
console.log(` quotes: ${printQuoteSummary(result.quotes)}`);
|
|
|
if (result.error) {
|
|
|
console.log(` err: ${result.error.slice(0, 80)}`);
|
|
|
}
|
|
|
}
|
|
|
|
|
|
await closeBrowser();
|
|
|
|
|
|
const pass = rows.filter((r) => r.result.ok && r.quotes.length > 0).length;
|
|
|
const withRates = pass;
|
|
|
const avgTotal = Math.round(rows.reduce((s, r) => s + r.timing.totalMs, 0) / rows.length);
|
|
|
const avgKernel = Math.round(rows.reduce((s, r) => s + r.timing.kernelMs, 0) / rows.length);
|
|
|
|
|
|
const report = {
|
|
|
runId,
|
|
|
finishedAt: new Date().toISOString(),
|
|
|
durationMs: Date.now() - startedAt,
|
|
|
summary: {
|
|
|
pass,
|
|
|
fail: rows.length - pass,
|
|
|
withRates,
|
|
|
avgTotalMs: avgTotal,
|
|
|
avgKernelMs: avgKernel,
|
|
|
},
|
|
|
cargoTemplate: CARGO,
|
|
|
env: {
|
|
|
RPA_HEADLESS: true,
|
|
|
RPA_VISUAL_RUSH: true,
|
|
|
RPA_DUAL_ADDRESS_PATCH: false,
|
|
|
RPA_BENCHMARK_QUOTE_MS: 45_000,
|
|
|
RPA_REQUIRE_QUOTE_RATES: true,
|
|
|
},
|
|
|
rows,
|
|
|
};
|
|
|
|
|
|
const outDir = join(process.cwd(), ".rpa", "benchmark-10pairs");
|
|
|
mkdirSync(outDir, { recursive: true });
|
|
|
const outPath = join(outDir, `bench-${runId}.json`);
|
|
|
writeFileSync(outPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
|
/** 固定文件名供对比/引用 */
|
|
|
const canonicalPath = join(outDir, "bench-1cdb927f.json");
|
|
|
writeFileSync(canonicalPath, `${JSON.stringify(report, null, 2)}\n`, "utf8");
|
|
|
|
|
|
console.log("\n| # | ID | Route | Result | Total | Kernel | Rates |");
|
|
|
console.log("|---|-----|-------|--------|-------|--------|-------|");
|
|
|
for (let i = 0; i < rows.length; i += 1) {
|
|
|
const r = rows[i]!;
|
|
|
console.log(
|
|
|
`| ${i + 1} | ${r.id} | ${r.route} | ${r.result.ok ? "PASS" : "FAIL"} | ${fmtMs(r.timing.totalMs)} | ${fmtMs(r.timing.kernelMs)} | ${printQuoteSummary(r.quotes)} |`,
|
|
|
);
|
|
|
}
|
|
|
console.log(
|
|
|
`\n| avg | — | — | ${pass}/10 (${withRates} with $) | ${fmtMs(avgTotal)} | ${fmtMs(avgKernel)} | — |`,
|
|
|
);
|
|
|
console.log(`\n[bench] report: ${outPath}`);
|
|
|
console.log(`[bench] canonical: ${canonicalPath}`);
|
|
|
process.exit(pass === rows.length ? 0 : 1);
|
|
|
}
|
|
|
|
|
|
main().catch(async (e) => {
|
|
|
console.error(e);
|
|
|
await closeBrowser().catch(() => undefined);
|
|
|
process.exit(1);
|
|
|
});
|
