mail-yubao/scripts/ingest-recognize-gold.ts

/**
 * 金样入库：docx/邮件/模板/附件下载_邮件识别
 * 四类识别：新增预<E5A29E>?+ 上传 DO（同封）
 * 用法: pnpm sample:recognize
 */
import { createHash } from "crypto";
import fs from "fs/promises";
import path from "path";
import { prisma } from "@/services/db";
import { ParsePipeline } from "@/services/parse/pipeline";
import { extractMailInstructions } from "@/services/parse/split-instructions";
import { classifyMail } from "@/services/parse/classify";
import { sha256 } from "@/utils/hash";

const GOLD_DIR = path.join(
  process.cwd(),
  "docx",
  "邮件",
  "模板",
  "附件下载_邮件识别",
);

const SUBJECT =
  "智鸿2+WHLC027G597465+WHSU8127240+洛杉<E6B49B>?40HQ+EDT2026.04-25 ETA2026.05-15船名航次HMM EMERALD 013E+提拆<E68F90>?;
const BODY = [
  "请查收新增预报：智鸿2+WHLC027G597465+WHSU8127240+洛杉<EFBFBD>?40HQ+EDT2026.04-25 ETA2026.05-15船名航次HMM EMERALD 013E+提拆<EFBFBD>?,
  "DO也同步上传至附件请注意查收！",
].join("\n");
const MESSAGE_ID = "<sample-recognize-gold-whsu8127240@local.test>";

const CONTENT_TYPES: Record<string, string> = {
  ".xlsx":
    "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
  ".xls": "application/vnd.ms-excel",
  ".pdf": "application/pdf",
  ".csv": "text/csv",
};

async function collectGoldFiles(dir: string): Promise<string[]> {
  const out: string[] = [];
  const entries = await fs.readdir(dir, { withFileTypes: true });
  for (const e of entries) {
    const p = path.join(dir, e.name);
    if (e.isDirectory()) {
      out.push(...(await collectGoldFiles(p)));
    } else if (!/\.docx$/i.test(e.name)) {
      out.push(p);
    }
  }
  return out;
}

async function attachAll(mailId: bigint, files: string[]): Promise<string[]> {
  const dir = path.join(process.cwd(), "data", "mails", String(mailId));
  await fs.mkdir(dir, { recursive: true });
  const names: string[] = [];
  for (const src of files) {
    const buf = await fs.readFile(src);
    const filename = path.basename(src);
    const dest = path.join(dir, filename);
    await fs.writeFile(dest, buf);
    const rel = path.relative(process.cwd(), dest).replace(/\\/g, "/");
    const ext = path.extname(filename).toLowerCase();
    await prisma.mailAttachment.create({
      data: {
        mailId,
        filename,
        contentType: CONTENT_TYPES[ext] || "application/octet-stream",
        sha256: sha256(buf),
        path: rel,
        size: buf.length,
        rejected: false,
      },
    });
    names.push(filename);
  }
  return names;
}

async function resetMail(mailId: bigint): Promise<void> {
  await prisma.importCompensation.deleteMany({
    where: { import: { mailId } },
  });
  await prisma.containerImport.deleteMany({ where: { mailId } });
  await prisma.parseResult.deleteMany({ where: { mailId } });
  await prisma.mailAttachment.deleteMany({ where: { mailId } });
}

async function main() {
  await fs.access(GOLD_DIR);
  const goldFiles = await collectGoldFiles(GOLD_DIR);
  if (!goldFiles.length) {
    throw new Error(`no attachments under ${GOLD_DIR}`);
  }

  const rawHash = createHash("sha256")
    .update(
      `recognize-gold|${MESSAGE_ID}|${SUBJECT}|${goldFiles.map((f) => path.basename(f)).sort().join(",")}|v1`,
    )
    .digest("hex");

  const existing = await prisma.mailMessage.findFirst({
    where: { OR: [{ messageId: MESSAGE_ID }, { rawHash }] },
  });

  let mailId: bigint;
  if (existing) {
    await resetMail(existing.id);
    await prisma.mailMessage.update({
      where: { id: existing.id },
      data: {
        subject: SUBJECT.slice(0, 512),
        fromAddr: "recognize-gold@local.test",
        bodyText: BODY,
        status: "FETCHED",
        mailType: "UNKNOWN",
        lastError: null,
        rawHash,
        receivedAt: new Date("2026-04-20T10:00:00.000Z"),
        typeEvidence: { note: "recognize_gold_附件下载_邮件识别" },
        version: { increment: 1 },
      },
    });
    mailId = existing.id;
  } else {
    const created = await prisma.mailMessage.create({
      data: {
        messageId: MESSAGE_ID,
        subject: SUBJECT.slice(0, 512),
        fromAddr: "recognize-gold@local.test",
        bodyText: BODY,
        status: "FETCHED",
        mailType: "UNKNOWN",
        rawHash,
        folder: "INBOX",
        receivedAt: new Date("2026-04-20T10:00:00.000Z"),
        isThreadRoot: true,
        typeEvidence: { note: "recognize_gold_附件下载_邮件识别" },
      },
    });
    mailId = created.id;
  }

  const attached = await attachAll(mailId, goldFiles);
  await ParsePipeline.run(mailId);

  const updated = await prisma.mailMessage.findUniqueOrThrow({
    where: { id: mailId },
    include: { parseResult: true, attachments: true },
  });

  const filenames = updated.attachments.map((a) => a.filename);
  const evidence = classifyMail({
    subject: updated.subject,
    body: updated.bodyText || "",
    filenames,
  });
  const units = extractMailInstructions({
    subject: updated.subject,
    body: updated.bodyText || "",
    filenames,
  });

  console.log(
    JSON.stringify(
      {
        id: String(updated.id),
        mail_type: updated.mailType,
        status: updated.status,
        attached,
        classify: {
          mail_type: evidence.mail_type,
          scores: {
            DO_UPLOAD: evidence.scores.DO_UPLOAD,
            NEW_CONTAINER: evidence.scores.NEW_CONTAINER,
          },
        },
        ui_kinds: units.map((u) => ({
          kind: u.uiKind,
          current: u.isCurrent,
          source: u.source,
        })),
        url: `http://localhost:3100/mails/${updated.id}`,
      },
      null,
      2,
    ),
  );
}

main()
  .catch((e) => {
    console.error(e);
    process.exit(1);
  })
  .finally(async () => {
    await prisma.$disconnect();
  });