chajia/scripts/verify-rpa-stack.ts

/**
 * RPA 全栈验收（真实环境，非 mock）
 *
 * 通过标准（全部 exit 0）：
 * 1. WSL mysql:3307 + redis:6379 可达
 * 2. smoke:connect 通过
 * 3. worker:rpa 启动后持续监控 30s：≥2 次心跳更新，且全程无 Redis 致命错误
 * 4. probe:rpa 返回 4 档报价
 *
 * 用法：npm run verify:rpa-stack
 */
import { spawn, execSync } from "node:child_process";
import { loadDevEnv, readInfraEnvOverrides } from "@/lib/dev-env";

loadDevEnv();

import { ensureNativeInfraIfNeeded } from "@/lib/infra/ensure-native-infra";
import {
  ensureRedisReady,
  getRedis,
  probeFreshRedisConnection,
  resetRedisClient,
} from "@/lib/redis";
import { executeProbeQuote } from "@/lib/rpa/run-probe";
import { validateProbeItems } from "@/lib/rpa/probe-validation";
import { prisma } from "@/lib/prisma";

const WORKER_WARMUP_MS = 30_000;
const WORKER_HEALTH_POLL_MS = 2_000;
/** 首次心跳后至少再观察此时长，避免「刚连上就断」误报 PASS */
const MIN_MONITOR_AFTER_FIRST_HB_MS = 15_000;

const WORKER_FATAL_PATTERNS: RegExp[] = [
  /ECONNREFUSED.*6379/i,
  /MaxRetriesPerRequestError/i,
  /\[rpa-worker\] 心跳失败/i,
  /\[rpa-worker\] 熔断\/暂停检查失败/i,
  /\[rpa-worker\] BullMQ 连接错误/i,
];

function fail(step: string, detail: string): never {
  console.error(`[FAIL] ${step}: ${detail}`);
  process.exit(1);
}

function pass(step: string, detail: string): void {
  console.log(`[PASS] ${step}: ${detail}`);
}

function findWorkerFatalError(text: string): string | null {
  for (const pattern of WORKER_FATAL_PATTERNS) {
    if (pattern.test(text)) {
      return pattern.source;
    }
  }
  return null;
}

function reloadDevEnvAfterInfra(): void {
  loadDevEnv();
}

async function stepInfra(): Promise<void> {
  if (process.env.RPA_MOCK_MODE === "true") {
    fail("infra", "RPA_MOCK_MODE=true，请设 false 做真实验收");
  }
  try {
    if (process.platform === "win32") {
      execSync("npm run dev:infra:start", {
        stdio: "inherit",
        cwd: process.cwd(),
        timeout: 120_000,
      });
      reloadDevEnvAfterInfra();
    } else {
      ensureNativeInfraIfNeeded();
    }
  } catch (error) {
    fail(
      "infra",
      error instanceof Error ? error.message : "dev:infra:start 失败",
    );
  }
  resetRedisClient();
  await ensureRedisReady();
  pass(
    "infra",
    `mysql + redis 就绪（REDIS_URL=${process.env.REDIS_URL ?? "未配置"}）`,
  );
}

async function stepSmokeConnect(): Promise<void> {
  await prisma.$connect();
  const userCount = await prisma.sysUser.count();
  await prisma.$disconnect();
  const pong = await getRedis().ping();
  if (pong !== "PONG") {
    fail("smoke-connect", `redis ping=${pong}`);
  }
  pass("smoke-connect", `prisma sys_user=${userCount}, redis PONG`);
}

async function stepWorkerHealth(): Promise<void> {
  const workerId = `verify-${process.pid}`;
  const logChunks: string[] = [];

  reloadDevEnvAfterInfra();
  resetRedisClient();
  await probeFreshRedisConnection();
  await ensureRedisReady();

  const workerEnv = {
    ...process.env,
    ...readInfraEnvOverrides(),
    RPA_WORKER_ID: workerId,
  };

  const workerProc = spawn("npx", ["tsx", "workers/rpa/index.ts"], {
    cwd: process.cwd(),
    env: workerEnv,
    stdio: ["ignore", "pipe", "pipe"],
    shell: true,
  });

  workerProc.stdout?.on("data", (buf) => {
    logChunks.push(buf.toString());
  });
  workerProc.stderr?.on("data", (buf) => {
    logChunks.push(buf.toString());
  });

  const deadline = Date.now() + WORKER_WARMUP_MS;
  let sawWorkerStart = false;
  let firstHbAt: number | null = null;
  let lastHbTs: string | null = null;
  let hbUpdates = 0;

  const killWorker = (): void => {
    if (!workerProc.killed) {
      workerProc.kill("SIGTERM");
    }
  };

  while (Date.now() < deadline) {
    const text = logChunks.join("");
    const fatal = findWorkerFatalError(text);
    if (fatal) {
      killWorker();
      fail(
        "worker-health",
        `worker 出现 Redis 致命错误（${fatal}）\n${text.slice(-1200)}`,
      );
    }

    if (text.includes("[rpa-worker] 启动 worker_id=")) {
      sawWorkerStart = true;
    }

    if (sawWorkerStart) {
      try {
        resetRedisClient();
        await ensureRedisReady(3, 500);
        const key = `worker:heartbeat:${workerId}`;
        const raw = await getRedis().get(key);
        if (raw && raw !== lastHbTs) {
          if (lastHbTs !== null) {
            hbUpdates += 1;
          }
          lastHbTs = raw;
          if (firstHbAt === null) {
            firstHbAt = Date.now();
          }
        }
      } catch {
        /* 继续轮询 */
      }
    }

    const monitoredEnough =
      firstHbAt !== null &&
      Date.now() - firstHbAt >= MIN_MONITOR_AFTER_FIRST_HB_MS &&
      hbUpdates >= 1;

    if (monitoredEnough) {
      killWorker();
      pass(
        "worker-health",
        `稳定观察 ${MIN_MONITOR_AFTER_FIRST_HB_MS / 1000}s，心跳更新 ${hbUpdates + 1} 次（worker_id=${workerId}）`,
      );
      return;
    }

    await new Promise((r) => setTimeout(r, WORKER_HEALTH_POLL_MS));
  }

  killWorker();
  const text = logChunks.join("");
  if (!sawWorkerStart) {
    fail("worker-health", `worker 未启动\n${text.slice(-1200)}`);
  }
  if (firstHbAt === null) {
    fail("worker-health", `30s 内未写入心跳\n${text.slice(-1200)}`);
  }
  fail(
    "worker-health",
    `首次心跳后未满 ${MIN_MONITOR_AFTER_FIRST_HB_MS / 1000}s 或心跳未续期（updates=${hbUpdates}）\n${text.slice(-1200)}`,
  );
}

async function stepProbe(): Promise<void> {
  resetRedisClient();
  await ensureRedisReady();
  const result = await executeProbeQuote();
  const validation = validateProbeItems(result.items);
  if (!validation.ok) {
    fail("probe-rpa", validation.message);
  }
  pass(
    "probe-rpa",
    `${result.items.length} 档报价 source=${result.sourceType}`,
  );
}

async function main(): Promise<void> {
  console.log("=== RPA 全栈验收开始 ===\n");
  console.log(
    "目标：infra 就绪 → DB/Redis 连通 → worker 30s 无 Redis 致命错误 → probe 4 档报价\n",
  );
  await stepInfra();
  await stepSmokeConnect();
  await stepWorkerHealth();
  await stepProbe();
  console.log("\n=== RPA 全栈验收通过 ===");
}

main()
  .catch((error) => {
    fail("unexpected", error instanceof Error ? error.message : String(error));
  })
  .finally(async () => {
    resetRedisClient();
    await prisma.$disconnect().catch(() => undefined);
  });