chore: add guarded reference image backfill progress
This commit is contained in:
@@ -11,6 +11,7 @@ const dryRun = args.has('--dry-run');
|
||||
const referenceImageStorage = await import('../src/lib/reference-image-storage.ts');
|
||||
const persistReferenceImages = referenceImageStorage.persistReferenceImages
|
||||
|| referenceImageStorage.default?.persistReferenceImages;
|
||||
const verbose = args.has('--verbose');
|
||||
|
||||
if (args.has('--check-import')) {
|
||||
if (typeof persistReferenceImages !== 'function') {
|
||||
@@ -21,6 +22,8 @@ if (args.has('--check-import')) {
|
||||
}
|
||||
const limitArg = [...args].find(arg => arg.startsWith('--limit='));
|
||||
const limit = Math.max(1, Math.min(5000, Number(limitArg?.split('=')[1] || 500)));
|
||||
const timeoutArg = [...args].find(arg => arg.startsWith('--item-timeout-ms='));
|
||||
const itemTimeoutMs = Math.max(5_000, Math.min(300_000, Number(timeoutArg?.split('=')[1] || 90_000)));
|
||||
|
||||
function loadEnvFile(filePath) {
|
||||
if (!fs.existsSync(filePath)) return;
|
||||
@@ -62,6 +65,14 @@ function shouldBackfill(params) {
|
||||
&& (!Array.isArray(params.referenceImages) || params.referenceImages.length === 0 || !hasThumbnails || references.some(value => value.startsWith('data:image/')));
|
||||
}
|
||||
|
||||
function withTimeout(promise, timeoutMs, label) {
|
||||
let timeoutId;
|
||||
const timeout = new Promise((_, reject) => {
|
||||
timeoutId = setTimeout(() => reject(new Error(`${label} timed out after ${timeoutMs}ms`)), timeoutMs);
|
||||
});
|
||||
return Promise.race([promise, timeout]).finally(() => clearTimeout(timeoutId));
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const connectionString = process.env.LOCAL_DB_URL || process.env.DATABASE_URL;
|
||||
if (!connectionString) throw new Error('LOCAL_DB_URL or DATABASE_URL is required');
|
||||
@@ -95,7 +106,27 @@ async function main() {
|
||||
}
|
||||
candidates += 1;
|
||||
if (dryRun) continue;
|
||||
const persisted = await persistReferenceImages(getReferenceInputs(params));
|
||||
const references = getReferenceInputs(params);
|
||||
if (verbose) {
|
||||
console.log(JSON.stringify({
|
||||
event: 'backfill-reference-images:start',
|
||||
id: row.id,
|
||||
index: candidates,
|
||||
references: references.length,
|
||||
}));
|
||||
}
|
||||
let persisted;
|
||||
try {
|
||||
persisted = await withTimeout(
|
||||
persistReferenceImages(references),
|
||||
itemTimeoutMs,
|
||||
`work ${row.id}`,
|
||||
);
|
||||
} catch (error) {
|
||||
skipped += 1;
|
||||
console.warn('[backfill-work-reference-images] skipped row:', row.id, error instanceof Error ? error.message : error);
|
||||
continue;
|
||||
}
|
||||
if (persisted.length === 0) {
|
||||
skipped += 1;
|
||||
continue;
|
||||
@@ -118,6 +149,14 @@ async function main() {
|
||||
],
|
||||
);
|
||||
updated += 1;
|
||||
if (verbose) {
|
||||
console.log(JSON.stringify({
|
||||
event: 'backfill-reference-images:updated',
|
||||
id: row.id,
|
||||
index: candidates,
|
||||
persisted: persisted.length,
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
console.log(JSON.stringify({
|
||||
|
||||
Reference in New Issue
Block a user