mail-yubao/scripts/ingest-sample-mails.ts

/**
 * <20><>?docx/邮件/邮件1~4（及可选模板预报）注入本地库并<E5BA93><E5B9B6>?ParsePipeline<6E><65>? * 用法: pnpm sample:mails
 *
 * 目录约定（平铺）<E993BA><EFBC89>? *   docx/邮件/邮件N/
 *     *邮箱.pdf / QQ邮箱.pdf   <20><>?QQ 截图（默认不入库<E585A5><E5BA93>? *     *.xlsx / 业务 *.pdf      <20><>?业务附件（按 SampleDef.attachPolicy 入库<E585A5><E5BA93>? *   docx/邮件/模板/            <20><>?共享预报模板（M_BL / forecast-ui<75><69>? *
 * M1 主题-only（UNKNOWN<57><4E>? * M2 主题+卡派 xlsx（WORK_ORDER/转仓<E8BDAC><E4BB93>? * M3 主题 + 卡转<E58DA1><E8BDAC>?PDF 展示（WORK_ORDER 拆柜清单；解析仍主题短路<E79FAD><E8B7AF>? * M4 主题+换标正文 + 换标指令 xlsx/PDF（WORK_ORDER / 贴标<E8B4B4><E6A087>? */
import { createHash } from "crypto";
import fs from "fs/promises";
import path from "path";
import { prisma } from "@/services/db";
import { ParsePipeline } from "@/services/parse/pipeline";
import {
  selectAttachments,
  type AttachPolicy,
} from "@/services/sample/mail-attachment-select";
import { sha256 } from "@/utils/hash";
import { loadMailboxPdfBody } from "./lib/load-mailbox-pdf-body";

const DOC_MAIL_ROOT = path.join(process.cwd(), "docx", "邮件");

type SampleDef = {
  key: string;
  messageId: string;
  fromAddr: string;
  subject: string;
  bodyText: string;
  receivedAt: string;
  /** 相对 docx/邮件/ 的目录名，如 邮件2 */
  sampleDir: string;
  attachPolicy: AttachPolicy;
  /** 可选：文件名须包含的关键词（全部命中） */
  nameIncludes?: string[];
};

const SAMPLES: SampleDef[] = [
  {
    key: "mail1",
    messageId: "<sample-mail1-tiiu8073522@local.test>",
    fromAddr: "cs3@xinfenginc.com",
    subject: "Fw: 转发：TIIU8073522-90022",
    bodyText: [
      "（样例）主题为柜号线索，完整正文<E6ADA3><E69687>?QQ 邮箱截图附件<E99984><E4BBB6>?,
      "柜号线索：TIIU8073522",
    ].join("\n"),
    receivedAt: "2026-07-14T08:02:00.000Z",
    sampleDir: "邮件1",
    attachPolicy: "none",
  },
  {
    key: "mail2",
    messageId: "<sample-mail2-matu2745683@local.test>",
    fromAddr: "op7@xinfenginc.com",
    subject:
      "Fw: 转发：LINK EVER INC + MATS4583030000+ 柜号：MATU2745683+ ETA : 7/13",
    bodyText: "新增转仓，请留意\n柜号：MATU2745683 更新派送单，请查收",
    receivedAt: "2026-07-14T08:02:00.000Z",
    sampleDir: "邮件2",
    attachPolicy: "packing_xlsx",
    nameIncludes: ["MATU2745683"],
  },
  {
    key: "mail3",
    messageId: "<sample-mail3-whsu5574991@local.test>",
    fromAddr: "clx@cnwally.com.cn",
    subject:
      "Fw: 转发：派送要求更<EFBFBD><EFBFBD>? 拆柜清单更新<EFBFBD><EFBFBD>?DO请查<EFBFBD><EFBFBD>?拆柜清单更新: WHSU5574991+WHL063G550810+船名航次：OOCL SINGAPORE / 065W+ETA<EFBFBD><EFBFBD>?/28+FedEx-29<EFBFBD><EFBFBD>?UPS-102<EFBFBD><EFBFBD>?亚马逊卡<EFBFBD><EFBFBD>?289<EFBFBD><EFBFBD>?私人地址-166<EFBFBD><EFBFBD>?拦截-209<EFBFBD><EFBFBD>?拆柜清单",
    bodyText: [
      "派送要求更新，请查<E8AFB7><E69FA5>?DO 与拆柜清单<E6B885><E58D95>?,
      "柜号：WHSU5574991",
      "提单：WHL063G550810",
      "船名航次：OOCL SINGAPORE / 065W",
      "ETA<EFBFBD><EFBFBD>?026-06-28",
      "FedEx-29<EFBFBD><EFBFBD>?UPS-102<EFBFBD><EFBFBD>?亚马逊卡<EFBFBD><EFBFBD>?289<EFBFBD><EFBFBD>?私人地址-166<EFBFBD><EFBFBD>?拦截-209<EFBFBD><EFBFBD>?,
    ].join("\n"),
    receivedAt: "2026-07-14T08:02:00.000Z",
    sampleDir: "邮件3",
    // 卡转<E58DA1><E8BDAC>?PDF 入库供详情展示；解析仍走主题短路，不吃表<E59083><E8A1A8>?    attachPolicy: "business",
  },
  {
    key: "mail4",
    messageId: "<sample-mail4-label-yt2604021091@local.test>",
    fromAddr: "xinchenze002@126.com",
    subject:
      "Fw: 转发：新增预<E5A29E><E9A284>?36：新辰泽+WHLC027G597465+WHSU8127240+洛杉<E6B49B><E69D89>?40HQ+EDT2026.04-25 ETA2026.05-15船名航次HMM EMERALD 013E+提拆派组合柜-不带托架",
    bodyText: [
      "Dear,",
      "原箱号YT2604021091=FBA199R49LD6-MDW2=76件操作指<E4BD9C><E68C87>?,
      "换标后单号：FBA19HW52S0L-2G1MCB6X-HIA1=76<EFBFBD><EFBFBD>?,
      "1：覆盖贴FBA标签，一箱贴两张",
      "2：覆盖贴SKU标签，一<EFBC8C><E4B880>?张（FNSKU：X003UHDF7B<37><42>?04PCS））",
      "3：贴好拍照回传等国内客户确认后，再约仓卡派交付！",
      "注：回传过来的照片需要清晰拍照SKU上面的字迹，客户确认后再安排卡派交付，不确认不予安排！！<EFBC81><EFBC81>?,
    ].join("\n"),
    receivedAt: "2026-07-14T08:00:00.000Z",
    sampleDir: "邮件4",
    attachPolicy: "label_instruction",
  },
];

async function resolveSampleDir(sampleDir: string): Promise<string | null> {
  const direct = path.join(DOC_MAIL_ROOT, sampleDir);
  try {
    const st = await fs.stat(direct);
    if (st.isDirectory()) return direct;
  } catch {
    /* fall through */
  }
  // 兜底：在 docx/邮件 下按目录名包含匹<E590AB><E58CB9>?  try {
    const entries = await fs.readdir(DOC_MAIL_ROOT, { withFileTypes: true });
    for (const e of entries) {
      if (e.isDirectory() && (e.name === sampleDir || e.name.includes(sampleDir))) {
        return path.join(DOC_MAIL_ROOT, e.name);
      }
    }
  } catch {
    return null;
  }
  return null;
}

/** 递归收集目录内文件（含一层子目录，兼容未平铺旧结构） */
async function listFilesRecursive(
  root: string,
): Promise<Array<{ abs: string; filename: string; rel: string }>> {
  const out: Array<{ abs: string; filename: string; rel: string }> = [];
  const stack = [root];
  while (stack.length) {
    const cur = stack.pop()!;
    let entries: Awaited<ReturnType<typeof fs.readdir>>;
    try {
      entries = await fs.readdir(cur, { withFileTypes: true });
    } catch {
      continue;
    }
    for (const e of entries) {
      const full = path.join(cur, e.name);
      if (e.isDirectory()) {
        stack.push(full);
        continue;
      }
      out.push({
        abs: full,
        filename: e.name,
        rel: path.relative(root, full).split(path.sep).join("/"),
      });
    }
  }
  return out;
}

async function upsertMail(sample: SampleDef): Promise<bigint> {
  const rawHash = createHash("sha256")
    .update(`sample|${sample.key}|${sample.subject}|${sample.bodyText}|fullbody-v1`)
    .digest("hex");

  const existing = await prisma.mailMessage.findFirst({
    where: {
      OR: [{ messageId: sample.messageId }, { rawHash }],
    },
  });

  if (existing) {
    await prisma.importCompensation.deleteMany({
      where: { import: { mailId: existing.id } },
    });
    await prisma.containerImport.deleteMany({ where: { mailId: existing.id } });
    await prisma.parseResult.deleteMany({ where: { mailId: existing.id } });
    await prisma.mailAttachment.deleteMany({ where: { mailId: existing.id } });
    await prisma.mailMessage.update({
      where: { id: existing.id },
      data: {
        subject: sample.subject.slice(0, 512),
        fromAddr: sample.fromAddr,
        bodyText: sample.bodyText,
        ocrText: null,
        status: "FETCHED",
        mailType: "UNKNOWN",
        lastError: null,
        rawHash,
        receivedAt: new Date(sample.receivedAt),
        typeEvidence: {
          sample_dir: `docx/邮件/${sample.sampleDir}`,
          note: "sample_ingest",
          sample_key: sample.key,
          attach_policy: sample.attachPolicy,
        },
        version: { increment: 1 },
      },
    });
    return existing.id;
  }

  const created = await prisma.mailMessage.create({
    data: {
      messageId: sample.messageId,
      subject: sample.subject.slice(0, 512),
      fromAddr: sample.fromAddr,
      bodyText: sample.bodyText,
      status: "FETCHED",
      mailType: "UNKNOWN",
      rawHash,
      folder: "INBOX",
      receivedAt: new Date(sample.receivedAt),
      isThreadRoot: true,
      typeEvidence: {
        sample_dir: `docx/邮件/${sample.sampleDir}`,
        note: "sample_ingest",
        sample_key: sample.key,
        attach_policy: sample.attachPolicy,
      },
    },
  });
  return created.id;
}

function contentTypeFor(filename: string): string {
  if (/\.xlsx?$/i.test(filename)) {
    return "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet";
  }
  if (/\.csv$/i.test(filename)) return "text/csv";
  if (/\.pdf$/i.test(filename)) return "application/pdf";
  return "application/octet-stream";
}

async function attachFiles(
  mailId: bigint,
  files: Array<{ abs: string; filename: string }>,
): Promise<void> {
  if (!files.length) return;
  const dir = path.join(process.cwd(), "data", "mails", String(mailId));
  await fs.mkdir(dir, { recursive: true });
  for (const f of files) {
    const buf = await fs.readFile(f.abs);
    const hash = sha256(buf);
    const safe = f.filename.replace(/[<>:"|?*\x00-\x1f]/g, "_");
    const dest = path.join(dir, safe);
    await fs.writeFile(dest, buf);
    const rel = path.relative(process.cwd(), dest).replace(/\\/g, "/");
    await prisma.mailAttachment.create({
      data: {
        mailId,
        filename: f.filename.slice(0, 512),
        contentType: contentTypeFor(f.filename),
        sha256: hash,
        path: rel,
        size: buf.length,
        rejected: false,
      },
    });
  }
}

async function main() {
  const results: Array<Record<string, unknown>> = [];

  for (const sample of SAMPLES) {
    const sampleDir = await resolveSampleDir(sample.sampleDir);
    let bodyText = sample.bodyText;
    if (sampleDir) {
      const pdfBody = await loadMailboxPdfBody(sampleDir);
      if (pdfBody && pdfBody.length > bodyText.length) {
        bodyText = pdfBody;
        console.log(
          `${sample.key}: body from mailbox PDF (${pdfBody.length} chars)`,
        );
      }
    }
    const mailId = await upsertMail({ ...sample, bodyText });

    let attached: string[] = [];
    if (!sampleDir) {
      console.warn(`${sample.key}: sample dir not found: ${sample.sampleDir}`);
    } else if (sample.attachPolicy !== "none") {
      const files = await listFilesRecursive(sampleDir);
      const selected = selectAttachments(
        files,
        sample.attachPolicy,
        sample.nameIncludes,
      );
      await attachFiles(mailId, selected);
      attached = selected.map((f) => f.filename);
      console.log(
        `${sample.key}: dir=${sample.sampleDir} policy=${sample.attachPolicy} attached=${attached.length ? attached.join(" | ") : "(none)"}`,
      );
    } else {
      console.log(`${sample.key}: attachPolicy=none`);
    }

    await ParsePipeline.run(mailId);

    const mail = await prisma.mailMessage.findUnique({
      where: { id: mailId },
      include: { parseResult: true, attachments: true },
    });
    const header = mail?.parseResult?.containerHeader as {
      F_ContainerNo?: string;
      F_BLCopyCode?: string;
      F_ETA?: string;
    } | null;
    const lineage = mail?.parseResult?.lineage as {
      mail_record?: { kind?: string; summary?: string };
      note?: string;
    } | null;
    const shipments = (mail?.parseResult?.shipments as unknown[]) || [];

    const row = {
      key: sample.key,
      id: String(mailId),
      status: mail?.status,
      mail_type: mail?.mailType,
      container_no: header?.F_ContainerNo,
      bl: header?.F_BLCopyCode,
      eta: header?.F_ETA,
      shipments: shipments.length,
      attachments: mail?.attachments.length ?? 0,
      attached_names: attached,
      record_kind: lineage?.mail_record?.kind,
      parse_note: lineage?.note,
      summary: lineage?.mail_record?.summary?.slice(0, 160),
      url: `http://localhost:3100/mails/${mailId}`,
    };
    results.push(row);
    console.log(JSON.stringify(row, null, 2));
  }

  console.log("\n=== summary ===");
  for (const r of results) {
    console.log(
      `${r.key}\t#${r.id}\t${r.mail_type}\t${r.status}\tatt=${r.attachments}\t${r.url}`,
    );
  }
}

main()
  .catch((err) => {
    console.error(err);
    process.exitCode = 1;
  })
  .finally(async () => {
    await prisma.$disconnect();
  });