diff --git a/worker/src/archive/forward-repost-check.test.ts b/worker/src/archive/forward-repost-check.test.ts new file mode 100644 index 0000000..2e221ae --- /dev/null +++ b/worker/src/archive/forward-repost-check.test.ts @@ -0,0 +1,35 @@ +import { describe, it, expect, vi } from "vitest"; + +const candidate = { + id: "pkg-1", archiveType: "ZIP", fileName: "a.zip", fileCount: 3, fileSize: 100n, + destMessageId: 1n, destMessageIds: [1n], destChannel: { telegramId: 999n }, +}; + +vi.mock("../db/queries.js", () => ({ + findFingerprintDedupCandidates: vi.fn(async () => [candidate]), +})); +const resolveMock = vi.fn(async (..._args: unknown[]) => [{ path: "x", fileName: "x", extension: null, compressedSize: 1n, uncompressedSize: 1n, crc32: "AAAA" }]); +const compareMock = vi.fn(); +vi.mock("../provenance-backfill.js", () => ({ + resolveCandidateFingerprintEntries: (...args: unknown[]) => resolveMock(...args), + compareFingerprints: (...args: unknown[]) => compareMock(...args), +})); + +import { checkFingerprintRepost } from "./forward-repost-check.js"; +import type { FileEntry } from "./zip-reader.js"; + +const newEntries: FileEntry[] = [{ path: "x", fileName: "x", extension: null, compressedSize: 1n, uncompressedSize: 1n, crc32: "AAAA" }]; + +describe("checkFingerprintRepost", () => { + it("reports a duplicate when a candidate's fingerprint matches", async () => { + compareMock.mockReturnValueOnce("match"); + const result = await checkFingerprintRepost({} as never, newEntries, "a.zip", 100n); + expect(result).toEqual({ isDuplicate: true, matchedPackageId: "pkg-1" }); + }); + + it("reports no duplicate when no candidate matches", async () => { + compareMock.mockReturnValueOnce("mismatch"); + const result = await checkFingerprintRepost({} as never, newEntries, "a.zip", 100n); + expect(result).toEqual({ isDuplicate: false, matchedPackageId: null }); + }); +}); diff --git a/worker/src/archive/forward-repost-check.ts b/worker/src/archive/forward-repost-check.ts new file mode 100644 index 0000000..9609c4f --- /dev/null +++ b/worker/src/archive/forward-repost-check.ts @@ -0,0 +1,33 @@ +import type { Client } from "tdl"; +import type { FileEntry } from "./zip-reader.js"; +import { compareFingerprints, resolveCandidateFingerprintEntries } from "../provenance-backfill.js"; +import { findFingerprintDedupCandidates } from "../db/queries.js"; + +export interface FingerprintRepostResult { + isDuplicate: boolean; + matchedPackageId: string | null; +} + +/** + * Cross-channel duplicate check for the forward-priority path: compare the + * new archive's CRC fingerprint against every existing Package sharing its + * name+size, regardless of which channel or ingestion path produced them. + * This is what lets a forwarded copy dedupe against a previously + * fully-downloaded copy of the same archive, despite never sharing a + * byte-hash-derived contentHash. + */ +export async function checkFingerprintRepost( + client: Client, + entries: FileEntry[], + fileName: string, + fileSize: bigint, +): Promise { + const candidates = await findFingerprintDedupCandidates(fileName, fileSize); + for (const candidate of candidates) { + const candidateEntries = await resolveCandidateFingerprintEntries(client, candidate); + if (compareFingerprints(entries, candidateEntries) === "match") { + return { isDuplicate: true, matchedPackageId: candidate.id }; + } + } + return { isDuplicate: false, matchedPackageId: null }; +} diff --git a/worker/src/db/queries.ts b/worker/src/db/queries.ts index d48b3f2..36bba1d 100644 --- a/worker/src/db/queries.ts +++ b/worker/src/db/queries.ts @@ -1030,6 +1030,30 @@ export interface PlaceholderCandidate { destChannel: { telegramId: bigint } | null; } +type PlaceholderRow = { + id: string; archiveType: string; fileName: string; fileCount: number; fileSize: bigint; + destMessageId: bigint | null; destMessageIds: bigint[]; destChannelId: string | null; +}; + +async function enrichWithDestChannel(rows: PlaceholderRow[]): Promise { + if (rows.length === 0) return []; + const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))]; + const channels = destChannelIds.length + ? await db.telegramChannel.findMany({ + where: { id: { in: destChannelIds } }, + select: { id: true, telegramId: true }, + }) + : []; + const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId])); + return rows.map((row) => ({ + id: row.id, archiveType: row.archiveType, fileName: row.fileName, fileCount: row.fileCount, fileSize: row.fileSize, + destMessageId: row.destMessageId, destMessageIds: row.destMessageIds, + destChannel: row.destChannelId && telegramIdById.has(row.destChannelId) + ? { telegramId: telegramIdById.get(row.destChannelId)! } + : null, + })); +} + /** * Find every placeholder Package matching name+size (oldest first). Package * has no direct `destChannel` relation (only the scalar `destChannelId`), so @@ -1059,29 +1083,30 @@ export async function findPlaceholderCandidates( }, orderBy: { indexedAt: "asc" }, }); - if (rows.length === 0) return []; + return enrichWithDestChannel(rows); +} - const destChannelIds = [...new Set(rows.map((r) => r.destChannelId).filter((id): id is string => !!id))]; - const channels = destChannelIds.length - ? await db.telegramChannel.findMany({ - where: { id: { in: destChannelIds } }, - select: { id: true, telegramId: true }, - }) - : []; - const telegramIdById = new Map(channels.map((c) => [c.id, c.telegramId])); - - return rows.map((row) => ({ - id: row.id, - archiveType: row.archiveType, - fileName: row.fileName, - fileCount: row.fileCount, - fileSize: row.fileSize, - destMessageId: row.destMessageId, - destMessageIds: row.destMessageIds, - destChannel: row.destChannelId && telegramIdById.has(row.destChannelId) - ? { telegramId: telegramIdById.get(row.destChannelId)! } - : null, - })); +/** + * Find every uploaded Package (any provenance, any channel) matching + * name+size, for the forward-priority path's cross-channel CRC-fingerprint + * dedup check. Unlike findPlaceholderCandidates, this is NOT restricted to + * placeholder rows — it exists to catch the case where the exact same + * archive was independently uploaded (not reposted/forwarded) to two + * different source channels. + */ +export async function findFingerprintDedupCandidates( + fileName: string, + fileSize: bigint, +): Promise { + const rows = await db.package.findMany({ + where: { fileName, fileSize, destMessageId: { not: null } }, + select: { + id: true, archiveType: true, fileName: true, fileCount: true, fileSize: true, + destMessageId: true, destMessageIds: true, destChannelId: true, + }, + orderBy: { indexedAt: "asc" }, + }); + return enrichWithDestChannel(rows); } export async function findPlaceholderCandidate( diff --git a/worker/src/provenance-backfill.ts b/worker/src/provenance-backfill.ts index 987dee0..0fceb26 100644 --- a/worker/src/provenance-backfill.ts +++ b/worker/src/provenance-backfill.ts @@ -79,7 +79,7 @@ async function resolveDestParts( * candidate with fileCount === 0), fall back to a fresh ranged read of the * candidate's own copy in the destination channel (Task 9). */ -async function resolveCandidateFingerprintEntries( +export async function resolveCandidateFingerprintEntries( client: Client, candidate: PlaceholderCandidate, ): Promise { @@ -119,7 +119,7 @@ async function resolveCandidateFingerprintEntries( * refute a match — callers must fall back to name+size confidence rather * than treating this as a mismatch. */ -function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" { +export function compareFingerprints(a: FileEntry[], b: FileEntry[]): "match" | "mismatch" | "incomplete" { const fa = crcFingerprint(a); const fb = crcFingerprint(b); if (!fa.complete || !fb.complete) return "incomplete";