chajia/scripts/capture-flock-network-har.ts

/**
 * Flock Network 抓包 POC：真实打开 get-a-quote，录 HAR + 旁路 XHR 日志，
 * 分析是否存在可直连的 quote/rate JSON 接口（对齐 MotherShip axel/quote 思路）。
 *
 * 用法:
 *   npm run capture:flock-har
 *   RPA_HEADLESS=false npm run capture:flock-har
 */
import fs from "node:fs";
import path from "node:path";
import type { Page, Request, Response } from "playwright";
import {
  analyzeFlockHarEntries,
  analyzeFlockNetworkLog,
  mergeAndDedupeCandidates,
  summarizeFlockHosts,
  verdictFromCandidates,
  type FlockHarFile,
  type FlockNetworkLogEntry,
  type FlockQuoteApiCandidate,
} from "@/lib/flock/har-analyze";
import { resolveFlockQuoteAccount } from "@/lib/flock/resolve-account";
import { getFlockQuoteUrl, getFlockQuoteWaitMs } from "@/lib/flock/env";
import { createRpaBrowserContext } from "@/lib/rpa/browser-context";
import { launchRpaBrowser } from "@/lib/rpa/browser-launch";
import {
  formatExternalSiteNavigationError,
  gotoWithResilience,
  isTransientNavigationError,
} from "@/lib/rpa/page-goto";
import { DEFAULT_FLOCK_DEMO } from "@/workers/rpa/flock/demo-input";
import {
  clickFlockQuoteSubmit,
  fillFlockQuoteForm,
  waitForFlockQuoteForm,
} from "@/workers/rpa/flock/form-interactions";
import {
  completeFlockRegistrationIfNeeded,
  isFlockQuoteResultsPage,
  isFlockRegistrationPage,
} from "@/workers/rpa/flock/register-account";
import { extractFlockQuoteCards } from "@/workers/rpa/flock/quote-extract";
import {
  flockQuoteLimitMessage,
  flockRegistrationErrorMessage,
} from "@/workers/rpa/flock/page-state";
import type { FlockQuoteInput } from "@/workers/rpa/flock/types";

const ROOT = process.cwd();
const OUT_DIR = path.join(ROOT, ".rpa", "flock-network-poc");

function stamp(): string {
  const d = new Date();
  const p = (n: number) => String(n).padStart(2, "0");
  return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
}

function demoToInput(): FlockQuoteInput {
  return {
    pickupDate: DEFAULT_FLOCK_DEMO.pickupDateDisplay,
    pickupZip: DEFAULT_FLOCK_DEMO.pickupZip,
    deliveryZip: DEFAULT_FLOCK_DEMO.deliveryZip,
    palletCount: DEFAULT_FLOCK_DEMO.palletCount,
    totalWeightLb: DEFAULT_FLOCK_DEMO.totalWeightLb,
    lengthIn: DEFAULT_FLOCK_DEMO.lengthIn,
    widthIn: DEFAULT_FLOCK_DEMO.widthIn,
    heightIn: DEFAULT_FLOCK_DEMO.heightIn,
  };
}

function shouldCaptureRequest(req: Request): boolean {
  const type = req.resourceType();
  if (type === "document" || type === "stylesheet" || type === "image" || type === "font" || type === "media") {
    return false;
  }
  const url = req.url();
  if (/\.(js|css|png|jpe?g|gif|svg|woff2?|map)(\?|$)/i.test(url)) {
    return false;
  }
  return type === "xhr" || type === "fetch" || type === "other" || type === "websocket";
}

async function attachNetworkLogger(
  page: Page,
  sink: FlockNetworkLogEntry[],
): Promise<void> {
  page.on("response", async (res: Response) => {
    try {
      const req = res.request();
      if (!shouldCaptureRequest(req)) return;
      const headers = res.headers();
      const mime = headers["content-type"] ?? "";
      let responseBodyPreview: string | undefined;
      let requestBodyPreview: string | undefined;
      try {
        const post = req.postData();
        if (post) requestBodyPreview = post.slice(0, 4_000);
      } catch {
        /* ignore */
      }
      if (/json|text|javascript|graphql/i.test(mime) || mime === "") {
        try {
          const body = await res.text();
          responseBodyPreview = body.slice(0, 8_000);
        } catch {
          /* body may be unavailable */
        }
      }
      sink.push({
        method: req.method(),
        url: req.url(),
        status: res.status(),
        mimeType: mime,
        requestBodyPreview,
        responseBodyPreview,
        resourceType: req.resourceType(),
      });
    } catch {
      /* ignore listener errors */
    }
  });
}

function writeSummaryMarkdown(opts: {
  outPath: string;
  harPath: string;
  logPath: string;
  verdict: string;
  quoteOk: boolean;
  quoteError?: string;
  prices?: string;
  hosts: Array<{ host: string; count: number; methods: string[] }>;
  candidates: FlockQuoteApiCandidate[];
}): void {
  const lines = [
    "# Flock Network 抓包 POC 结论",
    "",
    `- 时间: ${new Date().toISOString()}`,
    `- HAR: \`${opts.harPath}\``,
    `- 旁路日志: \`${opts.logPath}\``,
    `- DOM 抓价成功: ${opts.quoteOk ? "是" : "否"}${opts.prices ? `（${opts.prices}）` : ""}`,
    opts.quoteError ? `- DOM 失败原因: ${opts.quoteError}` : "",
    `- **裁决: \`${opts.verdict}\`**`,
    "",
    "## 非静态主机 Top",
    "",
    ...opts.hosts.slice(0, 20).map(
      (h) => `- \`${h.host}\` ×${h.count} methods=${h.methods.join(",")}`,
    ),
    "",
    "## quote/rate 候选接口",
    "",
  ];
  if (opts.candidates.length === 0) {
    lines.push(
      "_未发现 score≥4 的 JSON quote/rate 候选。当前路径更可能依赖 DOM SSR/内嵌数据，与 MotherShip `axel/quote` 不同。_",
      "",
    );
  } else {
    for (const [i, c] of opts.candidates.slice(0, 15).entries()) {
      lines.push(
        `### ${i + 1}. score=${c.score} ${c.method} ${c.status}`,
        "",
        `- URL: \`${c.urlWithoutQuery}\``,
        `- source: ${c.source}`,
        c.mimeType ? `- mime: ${c.mimeType}` : "",
        c.requestPreview
          ? `- request preview:\n\`\`\`\n${c.requestPreview}\n\`\`\``
          : "",
        c.responsePreview
          ? `- response preview:\n\`\`\`\n${c.responsePreview}\n\`\`\``
          : "",
        "",
      );
    }
  }
  lines.push(
    "## 含义",
    "",
    "- `stable_json_quote_api_found`: 存在高分 JSON 报价接口，可继续做 Direct 风格 POC",
    "- `json_api_candidates_unclear`: 有 API 候选但契约不明，需人工核对 HAR",
    "- `no_quote_json_api_dom_only`: 未见稳定 quote JSON；继续 DOM RPA 或更深抓包",
    "",
  );
  fs.writeFileSync(
    opts.outPath,
    lines.filter((l) => l !== "").join("\n") + "\n",
    "utf8",
  );
}

async function main(): Promise<void> {
  if (process.env.RPA_MOCK_MODE === "true") {
    console.error("capture:flock-har 需要真实站点；请勿设 RPA_MOCK_MODE=true");
    process.exit(2);
  }

  fs.mkdirSync(OUT_DIR, { recursive: true });
  const id = stamp();
  const harPath = path.join(OUT_DIR, `flock-${id}.har`);
  const logPath = path.join(OUT_DIR, `flock-${id}-network.json`);
  const summaryJson = path.join(OUT_DIR, `flock-${id}-summary.json`);
  const summaryMd = path.join(OUT_DIR, `flock-${id}-summary.md`);

  const input = demoToInput();
  const account = resolveFlockQuoteAccount(input);
  const url = getFlockQuoteUrl();
  const headless = process.env.RPA_HEADLESS !== "false";
  const quoteWaitMs = getFlockQuoteWaitMs();
  const networkLog: FlockNetworkLogEntry[] = [];

  console.log(`[capture-flock-har] url=${url}`);
  console.log(`[capture-flock-har] sample ${input.pickupZip}→${input.deliveryZip} ${input.palletCount}pl ${input.totalWeightLb}lb`);
  console.log(`[capture-flock-har] email=${account.email} headless=${headless}`);
  console.log(`[capture-flock-har] har → ${harPath}`);

  const browser = await launchRpaBrowser({ headless, incognito: true });
  let quoteOk = false;
  let quoteError: string | undefined;
  let prices: string | undefined;

  try {
    const context = await createRpaBrowserContext(browser, {
      recordHar: {
        path: harPath,
        mode: "full",
        content: "embed",
      },
    });
    const page = await context.newPage();
    await attachNetworkLogger(page, networkLog);

    try {
      await gotoWithResilience(page, url);
    } catch (error) {
      const raw = error instanceof Error ? error.message : String(error);
      quoteError = isTransientNavigationError({ message: raw })
        ? formatExternalSiteNavigationError("Flock Freight 官网", {
            message: raw,
          })
        : raw;
      await context.close();
      throw new Error(quoteError);
    }

    if (!(await waitForFlockQuoteForm(page))) {
      quoteError =
        (await flockQuoteLimitMessage(page)) ??
        "QUOTE_ENTRY_UNAVAILABLE：表单未加载";
      await context.close();
      throw new Error(quoteError);
    }

    const filled = await fillFlockQuoteForm(page, input);
    if (!filled.ok) {
      quoteError = filled.error;
      await context.close();
      throw new Error(quoteError ?? "填表失败");
    }
    if (!(await clickFlockQuoteSubmit(page))) {
      quoteError = "STRUCT_CHANGE：未找到提交按钮";
      await context.close();
      throw new Error(quoteError);
    }

    const deadline = Date.now() + quoteWaitMs;
    while (Date.now() < deadline) {
      if (await flockQuoteLimitMessage(page)) break;
      if (await isFlockRegistrationPage(page)) {
        const ok = await completeFlockRegistrationIfNeeded(page, account);
        if (!ok) {
          quoteError =
            (await flockRegistrationErrorMessage(page)) ??
            "进入注册页但无法完成注册";
          break;
        }
      }
      if (await isFlockQuoteResultsPage(page)) break;
      await page.waitForTimeout(800);
    }

    if (!quoteError) {
      const limitMsg = await flockQuoteLimitMessage(page);
      if (limitMsg) {
        quoteError = limitMsg;
      } else if (await isFlockRegistrationPage(page)) {
        quoteError =
          (await flockRegistrationErrorMessage(page)) ??
          "停在注册页，未到结果卡";
      } else {
        const { quotes, reference } = await extractFlockQuoteCards(page);
        if (quotes.length >= 2) {
          quoteOk = true;
          prices = quotes.map((q) => `${q.label}:$${q.totalUsd}`).join(", ");
          console.log(`[capture-flock-har] DOM OK ref=${reference ?? "-"} ${prices}`);
        } else {
          quoteError = `仅抓到 ${quotes.length} 档 DOM 报价（仍可分析 HAR）`;
        }
      }
    }

    // 必须 close context 才能落盘 HAR
    await context.close();
  } finally {
    await browser.close();
  }

  fs.writeFileSync(logPath, JSON.stringify(networkLog, null, 2), "utf8");

  let harEntries: NonNullable<NonNullable<FlockHarFile["log"]>["entries"]> = [];
  if (fs.existsSync(harPath)) {
    const har = JSON.parse(fs.readFileSync(harPath, "utf8")) as FlockHarFile;
    harEntries = har.log?.entries ?? [];
  }

  const fromHar = analyzeFlockHarEntries(harEntries);
  const fromLog = analyzeFlockNetworkLog(networkLog);
  const candidates = mergeAndDedupeCandidates([fromHar, fromLog]);
  const hosts = summarizeFlockHosts(harEntries);
  const verdict = verdictFromCandidates(candidates);

  const payload = {
    capturedAt: new Date().toISOString(),
    harPath,
    logPath,
    quoteOk,
    quoteError: quoteError ?? null,
    prices: prices ?? null,
    harEntryCount: harEntries.length,
    networkLogCount: networkLog.length,
    verdict,
    hosts: hosts.slice(0, 30),
    candidates: candidates.slice(0, 20),
  };
  fs.writeFileSync(summaryJson, JSON.stringify(payload, null, 2), "utf8");
  writeSummaryMarkdown({
    outPath: summaryMd,
    harPath,
    logPath,
    verdict,
    quoteOk,
    quoteError,
    prices,
    hosts,
    candidates,
  });

  console.log(`[capture-flock-har] HAR entries=${payload.harEntryCount} networkLog=${payload.networkLogCount}`);
  console.log(`[capture-flock-har] candidates=${candidates.length} verdict=${verdict}`);
  console.log(`[capture-flock-har] summary → ${summaryMd}`);
  for (const c of candidates.slice(0, 8)) {
    console.log(`  [${c.score}] ${c.method} ${c.status} ${c.urlWithoutQuery}`);
  }

  // 抓包本身成功即 exit 0；无候选时 exit 3 便于脚本判定
  if (!fs.existsSync(harPath) || payload.harEntryCount < 5) {
    process.exit(1);
  }
  if (verdict === "no_quote_json_api_dom_only") {
    process.exit(3);
  }
}

main().catch((err) => {
  console.error("[capture-flock-har] FAIL:", err instanceof Error ? err.message : err);
  process.exit(1);
});