diff --git a/worker/src/db/queries.ts b/worker/src/db/queries.ts index 1070109..7461d5e 100644 --- a/worker/src/db/queries.ts +++ b/worker/src/db/queries.ts @@ -189,6 +189,36 @@ export async function packageExistsBySourceMessage( return pkg !== null; } +/** + * Detect a likely repost: same source channel + same fileName + same total + * fileSize already exists with destMessageId set. Used to skip downloads + * when the channel admin re-posts the same file under a new message ID + * (which `packageExistsBySourceMessage` cannot catch because the message ID + * is different). + * + * Returns the existing package's destMessageId for logging/observability, + * or null if no match. Approximate: same name + same total size is an + * extremely strong signal that it's the same content, but theoretically + * two unrelated files could collide. If that ever happens, the new file + * gets treated as a duplicate and is lost; the user can manually re-link + * via the UI by removing the existing Package. + */ +export async function findRepostedPackage( + sourceChannelId: string, + fileName: string, + fileSize: bigint +): Promise<{ id: string; destMessageId: bigint | null } | null> { + return db.package.findFirst({ + where: { + sourceChannelId, + fileName, + fileSize, + destMessageId: { not: null }, + }, + select: { id: true, destMessageId: true }, + }); +} + /** * Delete orphaned Package rows that have the same content hash but never * completed the upload (destMessageId is null). Called before creating a diff --git a/worker/src/worker.ts b/worker/src/worker.ts index 46244ac..3c06cc4 100644 --- a/worker/src/worker.ts +++ b/worker/src/worker.ts @@ -31,6 +31,7 @@ import { upsertSkippedPackage, deleteSkippedPackage, getCappedSkippedMessageIds, + findRepostedPackage, } from "./db/queries.js"; import type { ActivityUpdate } from "./db/queries.js"; import { createTdlibClient, closeTdlibClient } from "./tdlib/client.js"; @@ -1160,8 +1161,51 @@ async function processOneArchiveSet( return null; } - // ── Size guard: skip archives that exceed WORKER_MAX_ZIP_SIZE_MB ── + // Compute the total size across all parts (used by the repost check below + // AND by the size guard further down). const totalArchiveSize = archiveSet.parts.reduce((sum, p) => sum + p.fileSize, 0n); + + // ── Pre-download repost detection ── + // The source channel admin frequently reposts the same file at new message + // IDs. packageExistsBySourceMessage misses these (different msgId), so we + // historically downloaded the file just to discover via hash that it's a + // duplicate — wasting hours of bandwidth per run. + // + // Match by (sourceChannelId, fileName, totalSize). The totalSize comparison + // makes this very strong — name-and-size collision between unrelated files + // is rare in practice. If it ever happens, the new file is treated as a + // duplicate; the user can remove the existing Package via the UI to force + // a re-ingestion. + const reposted = await findRepostedPackage( + channel.id, + archiveName, + totalArchiveSize + ); + if (reposted) { + counters.zipsDuplicate++; + accountLog.info( + { + fileName: archiveName, + sourceMessageId: Number(archiveSet.parts[0].id), + existingPackageId: reposted.id, + existingDestMessageId: reposted.destMessageId ? Number(reposted.destMessageId) : null, + totalSize: Number(totalArchiveSize), + }, + "Skipping repost — same fileName + size already uploaded in this channel" + ); + await updateRunActivity(runId, { + currentActivity: `Skipped ${archiveName} (repost of already-uploaded file)`, + currentStep: "deduplicating", + currentChannel: channelTitle, + currentFile: archiveName, + currentFileNum: setIdx + 1, + totalFiles: totalSets, + zipsDuplicate: counters.zipsDuplicate, + }); + return null; + } + + // ── Size guard: skip archives that exceed WORKER_MAX_ZIP_SIZE_MB ── const maxSizeBytes = BigInt(config.maxZipSizeMB) * 1024n * 1024n; if (totalArchiveSize > maxSizeBytes) { accountLog.warn(