|
|
/**
|
|
|
* Flock Network POC：从 HAR / 旁路日志中识别 quote/rate 类 JSON 接口
|
|
|
*/
|
|
|
export type FlockHarRequest = {
|
|
|
url: string;
|
|
|
method: string;
|
|
|
};
|
|
|
|
|
|
export type FlockHarResponse = {
|
|
|
status: number;
|
|
|
content?: { mimeType?: string; text?: string; size?: number };
|
|
|
};
|
|
|
|
|
|
export type FlockHarEntry = {
|
|
|
request: FlockHarRequest;
|
|
|
response: FlockHarResponse;
|
|
|
time?: number;
|
|
|
};
|
|
|
|
|
|
export type FlockHarFile = {
|
|
|
log?: { entries?: FlockHarEntry[] };
|
|
|
};
|
|
|
|
|
|
/** 旁路网络日志（捕获脚本额外落盘） */
|
|
|
export type FlockNetworkLogEntry = {
|
|
|
method: string;
|
|
|
url: string;
|
|
|
status: number;
|
|
|
mimeType?: string;
|
|
|
requestBodyPreview?: string;
|
|
|
responseBodyPreview?: string;
|
|
|
resourceType?: string;
|
|
|
};
|
|
|
|
|
|
const STATIC_OR_TELEMETRY =
|
|
|
/\.(js|css|png|jpe?g|gif|svg|woff2?|ttf|map|ico)(\?|$)|google-analytics|googletagmanager|segment\.|mixpanel|sentry|hotjar|fullstory|datadog|newrelic|intercom|facebook\.com|doubleclick|amplitude|heap\.io|clarity\.ms|cdn\.|static\./i;
|
|
|
|
|
|
const API_HOST_HINT =
|
|
|
/flockfreight|flock\.|api\.|graphql|gql|supabase|firebase|azure|cloudfront|amazonaws/i;
|
|
|
|
|
|
const QUOTE_URL_HINT =
|
|
|
/quote|rate|pricing|estimate|shipment|freight|pallet|zip|offer|price|cart|booking|land|inquiry/i;
|
|
|
|
|
|
const QUOTE_BODY_HINTS = [
|
|
|
"flockdirect",
|
|
|
"flock_direct",
|
|
|
"prices from",
|
|
|
"transit",
|
|
|
"reference",
|
|
|
"quotes generated",
|
|
|
"rate",
|
|
|
"price",
|
|
|
"total",
|
|
|
"quote",
|
|
|
"pallet",
|
|
|
"zip",
|
|
|
];
|
|
|
|
|
|
export function stripUrlQuery(url: string): string {
|
|
|
try {
|
|
|
const u = new URL(url);
|
|
|
return `${u.origin}${u.pathname}`;
|
|
|
} catch {
|
|
|
return url.split("?")[0] ?? url;
|
|
|
}
|
|
|
}
|
|
|
|
|
|
export function isLikelyStaticOrTelemetry(url: string): boolean {
|
|
|
return STATIC_OR_TELEMETRY.test(url);
|
|
|
}
|
|
|
|
|
|
export function scoreFlockQuoteCandidate(input: {
|
|
|
url: string;
|
|
|
method: string;
|
|
|
status: number;
|
|
|
mimeType?: string;
|
|
|
responseText?: string;
|
|
|
requestText?: string;
|
|
|
}): number {
|
|
|
const { url, method, status, mimeType, responseText, requestText } = input;
|
|
|
if (status < 200 || status >= 400) return 0;
|
|
|
if (isLikelyStaticOrTelemetry(url)) return 0;
|
|
|
|
|
|
let score = 0;
|
|
|
const urlLower = url.toLowerCase();
|
|
|
const methodUpper = method.toUpperCase();
|
|
|
|
|
|
if (API_HOST_HINT.test(urlLower)) score += 2;
|
|
|
if (QUOTE_URL_HINT.test(urlLower)) score += 3;
|
|
|
if (methodUpper === "POST" || methodUpper === "PUT" || methodUpper === "PATCH") {
|
|
|
score += 2;
|
|
|
}
|
|
|
if (/json|graphql/i.test(mimeType ?? "")) score += 2;
|
|
|
if (responseText?.trim().startsWith("{") || responseText?.trim().startsWith("[")) {
|
|
|
score += 1;
|
|
|
}
|
|
|
|
|
|
const blob = `${responseText ?? ""}\n${requestText ?? ""}`.toLowerCase();
|
|
|
for (const hint of QUOTE_BODY_HINTS) {
|
|
|
if (blob.includes(hint)) score += 2;
|
|
|
}
|
|
|
if (/\$\s*\d/.test(blob) || /"amount"|"totalUsd"|"finalPrice"/.test(blob)) {
|
|
|
score += 3;
|
|
|
}
|
|
|
if (/flockdirect|standard line|prices from/i.test(blob)) {
|
|
|
score += 4;
|
|
|
}
|
|
|
|
|
|
return score;
|
|
|
}
|
|
|
|
|
|
export type FlockQuoteApiCandidate = {
|
|
|
method: string;
|
|
|
url: string;
|
|
|
urlWithoutQuery: string;
|
|
|
status: number;
|
|
|
score: number;
|
|
|
mimeType?: string;
|
|
|
requestPreview?: string;
|
|
|
responsePreview?: string;
|
|
|
source: "har" | "network-log";
|
|
|
};
|
|
|
|
|
|
function preview(text: string | undefined, max = 600): string | undefined {
|
|
|
if (!text?.trim()) return undefined;
|
|
|
const t = text.trim();
|
|
|
return t.length <= max ? t : `${t.slice(0, max)}…`;
|
|
|
}
|
|
|
|
|
|
export function analyzeFlockHarEntries(
|
|
|
entries: FlockHarEntry[],
|
|
|
): FlockQuoteApiCandidate[] {
|
|
|
const out: FlockQuoteApiCandidate[] = [];
|
|
|
for (const entry of entries) {
|
|
|
const url = entry.request.url;
|
|
|
const text = entry.response.content?.text;
|
|
|
const score = scoreFlockQuoteCandidate({
|
|
|
url,
|
|
|
method: entry.request.method,
|
|
|
status: entry.response.status,
|
|
|
mimeType: entry.response.content?.mimeType,
|
|
|
responseText: text,
|
|
|
});
|
|
|
if (score < 4) continue;
|
|
|
out.push({
|
|
|
method: entry.request.method,
|
|
|
url,
|
|
|
urlWithoutQuery: stripUrlQuery(url),
|
|
|
status: entry.response.status,
|
|
|
score,
|
|
|
mimeType: entry.response.content?.mimeType,
|
|
|
responsePreview: preview(text),
|
|
|
source: "har",
|
|
|
});
|
|
|
}
|
|
|
return out.sort((a, b) => b.score - a.score);
|
|
|
}
|
|
|
|
|
|
export function analyzeFlockNetworkLog(
|
|
|
logs: FlockNetworkLogEntry[],
|
|
|
): FlockQuoteApiCandidate[] {
|
|
|
const out: FlockQuoteApiCandidate[] = [];
|
|
|
for (const entry of logs) {
|
|
|
const score = scoreFlockQuoteCandidate({
|
|
|
url: entry.url,
|
|
|
method: entry.method,
|
|
|
status: entry.status,
|
|
|
mimeType: entry.mimeType,
|
|
|
responseText: entry.responseBodyPreview,
|
|
|
requestText: entry.requestBodyPreview,
|
|
|
});
|
|
|
if (score < 4) continue;
|
|
|
out.push({
|
|
|
method: entry.method,
|
|
|
url: entry.url,
|
|
|
urlWithoutQuery: stripUrlQuery(entry.url),
|
|
|
status: entry.status,
|
|
|
score,
|
|
|
mimeType: entry.mimeType,
|
|
|
requestPreview: preview(entry.requestBodyPreview),
|
|
|
responsePreview: preview(entry.responseBodyPreview),
|
|
|
source: "network-log",
|
|
|
});
|
|
|
}
|
|
|
return out.sort((a, b) => b.score - a.score);
|
|
|
}
|
|
|
|
|
|
export function mergeAndDedupeCandidates(
|
|
|
lists: FlockQuoteApiCandidate[][],
|
|
|
): FlockQuoteApiCandidate[] {
|
|
|
const map = new Map<string, FlockQuoteApiCandidate>();
|
|
|
for (const list of lists) {
|
|
|
for (const c of list) {
|
|
|
const key = `${c.method}|${c.urlWithoutQuery}`;
|
|
|
const prev = map.get(key);
|
|
|
if (!prev || c.score > prev.score) {
|
|
|
map.set(key, c);
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
return [...map.values()].sort((a, b) => b.score - a.score);
|
|
|
}
|
|
|
|
|
|
export function summarizeFlockHosts(entries: FlockHarEntry[]): Array<{
|
|
|
host: string;
|
|
|
count: number;
|
|
|
methods: string[];
|
|
|
}> {
|
|
|
const hosts = new Map<string, { count: number; methods: Set<string> }>();
|
|
|
for (const e of entries) {
|
|
|
if (isLikelyStaticOrTelemetry(e.request.url)) continue;
|
|
|
let host = "invalid";
|
|
|
try {
|
|
|
host = new URL(e.request.url).host;
|
|
|
} catch {
|
|
|
/* keep invalid */
|
|
|
}
|
|
|
const cur = hosts.get(host) ?? { count: 0, methods: new Set<string>() };
|
|
|
cur.count += 1;
|
|
|
cur.methods.add(e.request.method.toUpperCase());
|
|
|
hosts.set(host, cur);
|
|
|
}
|
|
|
return [...hosts.entries()]
|
|
|
.map(([host, v]) => ({
|
|
|
host,
|
|
|
count: v.count,
|
|
|
methods: [...v.methods].sort(),
|
|
|
}))
|
|
|
.sort((a, b) => b.count - a.count);
|
|
|
}
|
|
|
|
|
|
export type FlockNetworkPocVerdict =
|
|
|
| "stable_json_quote_api_found"
|
|
|
| "json_api_candidates_unclear"
|
|
|
| "no_quote_json_api_dom_only";
|
|
|
|
|
|
export function verdictFromCandidates(
|
|
|
candidates: FlockQuoteApiCandidate[],
|
|
|
): FlockNetworkPocVerdict {
|
|
|
const strong = candidates.filter((c) => c.score >= 10);
|
|
|
if (strong.length > 0) return "stable_json_quote_api_found";
|
|
|
if (candidates.length > 0) return "json_api_candidates_unclear";
|
|
|
return "no_quote_json_api_dom_only";
|
|
|
}
|
