You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
72 lines
2.4 KiB
72 lines
2.4 KiB
/**
|
|
* Wire sample ingest to use full mailbox PDF text as bodyText.
|
|
*/
|
|
import fs from "fs";
|
|
|
|
const path = "scripts/ingest-sample-mails.ts";
|
|
let src = fs.readFileSync(path, "utf8");
|
|
|
|
if (!src.includes("loadMailboxPdfBody")) {
|
|
src = src.replace(
|
|
`import { sha256 } from "@/utils/hash";`,
|
|
`import { sha256 } from "@/utils/hash";
|
|
import { loadMailboxPdfBody } from "./lib/load-mailbox-pdf-body";`,
|
|
);
|
|
}
|
|
|
|
// bump hash version so re-ingest refreshes body
|
|
src = src.replace(
|
|
'`.update(`sample|${sample.key}|${sample.subject}|${sample.bodyText}`)',
|
|
'`.update(`sample|${sample.key}|${sample.subject}|${sample.bodyText}|fullbody-v1`)',
|
|
);
|
|
// the above might be wrong quoting - fix:
|
|
src = src.replace(
|
|
"sample|${sample.key}|${sample.subject}|${sample.bodyText}`",
|
|
"sample|${sample.key}|${sample.subject}|${sample.bodyText}|fullbody-v1`",
|
|
);
|
|
|
|
const oldLoop = ` for (const sample of SAMPLES) {
|
|
const mailId = await upsertMail(sample);
|
|
const sampleDir = await resolveSampleDir(sample.sampleDir);`;
|
|
|
|
const neuLoop = ` for (const sample of SAMPLES) {
|
|
const sampleDir = await resolveSampleDir(sample.sampleDir);
|
|
let bodyText = sample.bodyText;
|
|
if (sampleDir) {
|
|
const pdfBody = await loadMailboxPdfBody(sampleDir);
|
|
if (pdfBody && pdfBody.length > bodyText.length) {
|
|
bodyText = pdfBody;
|
|
console.log(
|
|
\`\${sample.key}: body from mailbox PDF (\${pdfBody.length} chars)\`,
|
|
);
|
|
}
|
|
}
|
|
const mailId = await upsertMail({ ...sample, bodyText });`;
|
|
|
|
if (!src.includes(oldLoop)) {
|
|
console.error("loop missing");
|
|
process.exit(1);
|
|
}
|
|
src = src.replace(oldLoop, neuLoop);
|
|
|
|
// remove duplicate sampleDir resolve
|
|
src = src.replace(
|
|
` const mailId = await upsertMail({ ...sample, bodyText });
|
|
|
|
let attached: string[] = [];
|
|
if (!sampleDir) {
|
|
console.warn(\`\${sample.key}: sample dir not found: \${sample.sampleDir}\`);
|
|
} else if (sample.attachPolicy !== "none") {
|
|
const files = await listFilesRecursive(sampleDir);`,
|
|
` const mailId = await upsertMail({ ...sample, bodyText });
|
|
|
|
let attached: string[] = [];
|
|
if (!sampleDir) {
|
|
console.warn(\`\${sample.key}: sample dir not found: \${sample.sampleDir}\`);
|
|
} else if (sample.attachPolicy !== "none") {
|
|
const files = await listFilesRecursive(sampleDir);`,
|
|
);
|
|
|
|
fs.writeFileSync(path, src);
|
|
console.log("ingest wired", src.includes("loadMailboxPdfBody"), src.includes("fullbody-v1"));
|