/** * Flock Network 抓包 POC:真实打开 get-a-quote,录 HAR + 旁路 XHR 日志, * 分析是否存在可直连的 quote/rate JSON 接口(对齐 MotherShip axel/quote 思路)。 * * 用法: * npm run capture:flock-har * RPA_HEADLESS=false npm run capture:flock-har */ import fs from "node:fs"; import path from "node:path"; import type { Page, Request, Response } from "playwright"; import { analyzeFlockHarEntries, analyzeFlockNetworkLog, mergeAndDedupeCandidates, summarizeFlockHosts, verdictFromCandidates, type FlockHarFile, type FlockNetworkLogEntry, type FlockQuoteApiCandidate, } from "@/lib/flock/har-analyze"; import { resolveFlockQuoteAccount } from "@/lib/flock/resolve-account"; import { getFlockQuoteUrl, getFlockQuoteWaitMs } from "@/lib/flock/env"; import { createRpaBrowserContext } from "@/lib/rpa/browser-context"; import { launchRpaBrowser } from "@/lib/rpa/browser-launch"; import { formatExternalSiteNavigationError, gotoWithResilience, isTransientNavigationError, } from "@/lib/rpa/page-goto"; import { DEFAULT_FLOCK_DEMO } from "@/workers/rpa/flock/demo-input"; import { clickFlockQuoteSubmit, fillFlockQuoteForm, waitForFlockQuoteForm, } from "@/workers/rpa/flock/form-interactions"; import { completeFlockRegistrationIfNeeded, isFlockQuoteResultsPage, isFlockRegistrationPage, } from "@/workers/rpa/flock/register-account"; import { extractFlockQuoteCards } from "@/workers/rpa/flock/quote-extract"; import { flockQuoteLimitMessage, flockRegistrationErrorMessage, } from "@/workers/rpa/flock/page-state"; import type { FlockQuoteInput } from "@/workers/rpa/flock/types"; const ROOT = process.cwd(); const OUT_DIR = path.join(ROOT, ".rpa", "flock-network-poc"); function stamp(): string { const d = new Date(); const p = (n: number) => String(n).padStart(2, "0"); return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`; } function demoToInput(): FlockQuoteInput { return { pickupDate: DEFAULT_FLOCK_DEMO.pickupDateDisplay, pickupZip: DEFAULT_FLOCK_DEMO.pickupZip, deliveryZip: DEFAULT_FLOCK_DEMO.deliveryZip, palletCount: DEFAULT_FLOCK_DEMO.palletCount, totalWeightLb: DEFAULT_FLOCK_DEMO.totalWeightLb, lengthIn: DEFAULT_FLOCK_DEMO.lengthIn, widthIn: DEFAULT_FLOCK_DEMO.widthIn, heightIn: DEFAULT_FLOCK_DEMO.heightIn, }; } function shouldCaptureRequest(req: Request): boolean { const type = req.resourceType(); if (type === "document" || type === "stylesheet" || type === "image" || type === "font" || type === "media") { return false; } const url = req.url(); if (/\.(js|css|png|jpe?g|gif|svg|woff2?|map)(\?|$)/i.test(url)) { return false; } return type === "xhr" || type === "fetch" || type === "other" || type === "websocket"; } async function attachNetworkLogger( page: Page, sink: FlockNetworkLogEntry[], ): Promise { page.on("response", async (res: Response) => { try { const req = res.request(); if (!shouldCaptureRequest(req)) return; const headers = res.headers(); const mime = headers["content-type"] ?? ""; let responseBodyPreview: string | undefined; let requestBodyPreview: string | undefined; try { const post = req.postData(); if (post) requestBodyPreview = post.slice(0, 4_000); } catch { /* ignore */ } if (/json|text|javascript|graphql/i.test(mime) || mime === "") { try { const body = await res.text(); responseBodyPreview = body.slice(0, 8_000); } catch { /* body may be unavailable */ } } sink.push({ method: req.method(), url: req.url(), status: res.status(), mimeType: mime, requestBodyPreview, responseBodyPreview, resourceType: req.resourceType(), }); } catch { /* ignore listener errors */ } }); } function writeSummaryMarkdown(opts: { outPath: string; harPath: string; logPath: string; verdict: string; quoteOk: boolean; quoteError?: string; prices?: string; hosts: Array<{ host: string; count: number; methods: string[] }>; candidates: FlockQuoteApiCandidate[]; }): void { const lines = [ "# Flock Network 抓包 POC 结论", "", `- 时间: ${new Date().toISOString()}`, `- HAR: \`${opts.harPath}\``, `- 旁路日志: \`${opts.logPath}\``, `- DOM 抓价成功: ${opts.quoteOk ? "是" : "否"}${opts.prices ? `(${opts.prices})` : ""}`, opts.quoteError ? `- DOM 失败原因: ${opts.quoteError}` : "", `- **裁决: \`${opts.verdict}\`**`, "", "## 非静态主机 Top", "", ...opts.hosts.slice(0, 20).map( (h) => `- \`${h.host}\` ×${h.count} methods=${h.methods.join(",")}`, ), "", "## quote/rate 候选接口", "", ]; if (opts.candidates.length === 0) { lines.push( "_未发现 score≥4 的 JSON quote/rate 候选。当前路径更可能依赖 DOM SSR/内嵌数据,与 MotherShip `axel/quote` 不同。_", "", ); } else { for (const [i, c] of opts.candidates.slice(0, 15).entries()) { lines.push( `### ${i + 1}. score=${c.score} ${c.method} ${c.status}`, "", `- URL: \`${c.urlWithoutQuery}\``, `- source: ${c.source}`, c.mimeType ? `- mime: ${c.mimeType}` : "", c.requestPreview ? `- request preview:\n\`\`\`\n${c.requestPreview}\n\`\`\`` : "", c.responsePreview ? `- response preview:\n\`\`\`\n${c.responsePreview}\n\`\`\`` : "", "", ); } } lines.push( "## 含义", "", "- `stable_json_quote_api_found`: 存在高分 JSON 报价接口,可继续做 Direct 风格 POC", "- `json_api_candidates_unclear`: 有 API 候选但契约不明,需人工核对 HAR", "- `no_quote_json_api_dom_only`: 未见稳定 quote JSON;继续 DOM RPA 或更深抓包", "", ); fs.writeFileSync( opts.outPath, lines.filter((l) => l !== "").join("\n") + "\n", "utf8", ); } async function main(): Promise { if (process.env.RPA_MOCK_MODE === "true") { console.error("capture:flock-har 需要真实站点;请勿设 RPA_MOCK_MODE=true"); process.exit(2); } fs.mkdirSync(OUT_DIR, { recursive: true }); const id = stamp(); const harPath = path.join(OUT_DIR, `flock-${id}.har`); const logPath = path.join(OUT_DIR, `flock-${id}-network.json`); const summaryJson = path.join(OUT_DIR, `flock-${id}-summary.json`); const summaryMd = path.join(OUT_DIR, `flock-${id}-summary.md`); const input = demoToInput(); const account = resolveFlockQuoteAccount(input); const url = getFlockQuoteUrl(); const headless = process.env.RPA_HEADLESS !== "false"; const quoteWaitMs = getFlockQuoteWaitMs(); const networkLog: FlockNetworkLogEntry[] = []; console.log(`[capture-flock-har] url=${url}`); console.log(`[capture-flock-har] sample ${input.pickupZip}→${input.deliveryZip} ${input.palletCount}pl ${input.totalWeightLb}lb`); console.log(`[capture-flock-har] email=${account.email} headless=${headless}`); console.log(`[capture-flock-har] har → ${harPath}`); const browser = await launchRpaBrowser({ headless, incognito: true }); let quoteOk = false; let quoteError: string | undefined; let prices: string | undefined; try { const context = await createRpaBrowserContext(browser, { recordHar: { path: harPath, mode: "full", content: "embed", }, }); const page = await context.newPage(); await attachNetworkLogger(page, networkLog); try { await gotoWithResilience(page, url); } catch (error) { const raw = error instanceof Error ? error.message : String(error); quoteError = isTransientNavigationError({ message: raw }) ? formatExternalSiteNavigationError("Flock Freight 官网", { message: raw, }) : raw; await context.close(); throw new Error(quoteError); } if (!(await waitForFlockQuoteForm(page))) { quoteError = (await flockQuoteLimitMessage(page)) ?? "QUOTE_ENTRY_UNAVAILABLE:表单未加载"; await context.close(); throw new Error(quoteError); } const filled = await fillFlockQuoteForm(page, input); if (!filled.ok) { quoteError = filled.error; await context.close(); throw new Error(quoteError ?? "填表失败"); } if (!(await clickFlockQuoteSubmit(page))) { quoteError = "STRUCT_CHANGE:未找到提交按钮"; await context.close(); throw new Error(quoteError); } const deadline = Date.now() + quoteWaitMs; while (Date.now() < deadline) { if (await flockQuoteLimitMessage(page)) break; if (await isFlockRegistrationPage(page)) { const ok = await completeFlockRegistrationIfNeeded(page, account); if (!ok) { quoteError = (await flockRegistrationErrorMessage(page)) ?? "进入注册页但无法完成注册"; break; } } if (await isFlockQuoteResultsPage(page)) break; await page.waitForTimeout(800); } if (!quoteError) { const limitMsg = await flockQuoteLimitMessage(page); if (limitMsg) { quoteError = limitMsg; } else if (await isFlockRegistrationPage(page)) { quoteError = (await flockRegistrationErrorMessage(page)) ?? "停在注册页,未到结果卡"; } else { const { quotes, reference } = await extractFlockQuoteCards(page); if (quotes.length >= 2) { quoteOk = true; prices = quotes.map((q) => `${q.label}:$${q.totalUsd}`).join(", "); console.log(`[capture-flock-har] DOM OK ref=${reference ?? "-"} ${prices}`); } else { quoteError = `仅抓到 ${quotes.length} 档 DOM 报价(仍可分析 HAR)`; } } } // 必须 close context 才能落盘 HAR await context.close(); } finally { await browser.close(); } fs.writeFileSync(logPath, JSON.stringify(networkLog, null, 2), "utf8"); let harEntries: NonNullable["entries"]> = []; if (fs.existsSync(harPath)) { const har = JSON.parse(fs.readFileSync(harPath, "utf8")) as FlockHarFile; harEntries = har.log?.entries ?? []; } const fromHar = analyzeFlockHarEntries(harEntries); const fromLog = analyzeFlockNetworkLog(networkLog); const candidates = mergeAndDedupeCandidates([fromHar, fromLog]); const hosts = summarizeFlockHosts(harEntries); const verdict = verdictFromCandidates(candidates); const payload = { capturedAt: new Date().toISOString(), harPath, logPath, quoteOk, quoteError: quoteError ?? null, prices: prices ?? null, harEntryCount: harEntries.length, networkLogCount: networkLog.length, verdict, hosts: hosts.slice(0, 30), candidates: candidates.slice(0, 20), }; fs.writeFileSync(summaryJson, JSON.stringify(payload, null, 2), "utf8"); writeSummaryMarkdown({ outPath: summaryMd, harPath, logPath, verdict, quoteOk, quoteError, prices, hosts, candidates, }); console.log(`[capture-flock-har] HAR entries=${payload.harEntryCount} networkLog=${payload.networkLogCount}`); console.log(`[capture-flock-har] candidates=${candidates.length} verdict=${verdict}`); console.log(`[capture-flock-har] summary → ${summaryMd}`); for (const c of candidates.slice(0, 8)) { console.log(` [${c.score}] ${c.method} ${c.status} ${c.urlWithoutQuery}`); } // 抓包本身成功即 exit 0;无候选时 exit 3 便于脚本判定 if (!fs.existsSync(harPath) || payload.harEntryCount < 5) { process.exit(1); } if (verdict === "no_quote_json_api_dom_only") { process.exit(3); } } main().catch((err) => { console.error("[capture-flock-har] FAIL:", err instanceof Error ? err.message : err); process.exit(1); });