From 0bce1168a9ff91a0b8da5ff69397589695c59137 Mon Sep 17 00:00:00 2001 From: xCyanGrizzly Date: Thu, 23 Jul 2026 13:05:34 +0200 Subject: [PATCH] docs: correct provenance-backfill candidate predicate for rebuild records Rebuild records use sourceMessageId=0 + synthetic 'rebuild:' contentHash and an arbitrary fallback sourceChannelId, so the original sourceChannelId==destChannelId candidate definition missed them. Predicate is now (sourceChannelId==destChannelId OR sourceMessageId==0), verified against 59,893 live rebuilt records. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-07-23-provenance-backfill.md | 17 ++++++++--- .../2026-07-23-provenance-backfill-design.md | 28 +++++++++++++++---- 2 files changed, 36 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/plans/2026-07-23-provenance-backfill.md b/docs/superpowers/plans/2026-07-23-provenance-backfill.md index 037df0a..a05cf60 100644 --- a/docs/superpowers/plans/2026-07-23-provenance-backfill.md +++ b/docs/superpowers/plans/2026-07-23-provenance-backfill.md @@ -493,7 +493,7 @@ git commit -m "feat(worker): ranged TDLib file download (downloadFileRange)" **Interfaces:** - Produces: - - `findPlaceholderCandidate(destChannelId: string, fileName: string, fileSize: bigint): Promise<{ id: string; archiveType: string; fileCount: number } | null>` — a package where `sourceChannelId === destChannelId` (placeholder) AND `fileName` AND `fileSize` match, with a real destination (`destMessageId != null`). + - `findPlaceholderCandidate(destChannelId: string, fileName: string, fileSize: bigint): Promise<{ id: string; archiveType: string; fileCount: number } | null>` — a **placeholder** package (`sourceChannelId === destChannelId` OR `sourceMessageId === 0` — see spec §1) AND `fileName` AND `fileSize` match, with a real destination (`destMessageId != null`). - `getPackageFileCrcs(packageId: string): Promise<(string | null)[]>` — `PackageFile.crc32` values for the candidate. - `backfillProvenance(input: BackfillProvenanceInput): Promise` — transactionally overwrite placeholder fields; returns `false` (no-op) if the row is no longer a placeholder. Type: ```ts @@ -522,10 +522,15 @@ export async function findPlaceholderCandidate( ): Promise<{ id: string; archiveType: string; fileCount: number } | null> { return db.package.findFirst({ where: { - sourceChannelId: destChannelId, // placeholder: source == destination fileName, fileSize, destMessageId: { not: null }, + // Placeholder provenance (spec §1): manual-upload (source == destination) + // OR rebuild record (sourceMessageId == 0 "unknown" sentinel). + OR: [ + { sourceChannelId: destChannelId }, + { sourceMessageId: 0n }, + ], }, select: { id: true, archiveType: true, fileCount: true }, orderBy: { indexedAt: "asc" }, @@ -544,10 +549,14 @@ export async function backfillProvenance(input: BackfillProvenanceInput): Promis return db.$transaction(async (tx) => { const current = await tx.package.findUnique({ where: { id: input.packageId }, - select: { sourceChannelId: true, previewData: true, fileCount: true }, + select: { sourceChannelId: true, sourceMessageId: true, previewData: true, fileCount: true }, }); // Re-check placeholder status inside the txn (another worker may have won). - if (!current || current.sourceChannelId !== input.destChannelId) return false; + // Placeholder = manual-upload (source==dest) OR rebuild (sourceMessageId==0). + const stillPlaceholder = + !!current && + (current.sourceChannelId === input.destChannelId || current.sourceMessageId === 0n); + if (!stillPlaceholder) return false; // eslint-disable-next-line @typescript-eslint/no-explicit-any const data: any = { diff --git a/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md b/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md index acd9d88..60aedd4 100644 --- a/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md +++ b/docs/superpowers/specs/2026-07-23-provenance-backfill-design.md @@ -88,10 +88,27 @@ Current state (as of this design): ### 1. Candidate definition -> A package is a backfill candidate iff `sourceChannelId == destChannelId`. +> A package is a backfill candidate iff **`sourceChannelId == destChannelId` +> OR `sourceMessageId == 0`**. -These are exactly the manual-upload and rebuild-created records. Packages with a -real, non-placeholder source are **never** candidates and are never overwritten. +Two placeholder shapes exist (verified against live data 2026-07-23): +- **Manual uploads** (`manual-upload.ts`): `sourceChannelId == destChannelId`, + real `contentHash`, real `sourceMessageId`, has a `PackageFile` listing. +- **Rebuild records** (`rebuild.ts`): `sourceMessageId == 0n` (deliberate + "unknown" sentinel), synthetic `contentHash = "rebuild::"`, + `fileCount == 0`, and `sourceChannelId` set to an **arbitrary fallback source + channel** (`sourceChannels[0]`) — NOT the destination. (This is the common + case: e.g. 59,893 records after a destination rebuild.) + +Normal ingestion always sets a real `sourceMessageId` (> 0) and a real source +channel, so neither marker matches a genuinely-sourced package. Both markers are +overwritten on backfill (source channel + message become real), so a record +stops being a candidate once fixed — this is what makes re-scans idempotent. + +**Known limitation:** backfill does NOT rewrite a rebuild record's synthetic +`"rebuild:"` `contentHash` (the true content hash would require a full download, +which this feature avoids). That is acceptable — dedup after backfill relies on +`remoteUniqueId` + name/size within the source channel, not on `contentHash`. ### 2. Where it hooks @@ -103,8 +120,9 @@ existing fast paths). Flow for the scanned archive set: 1. Stage A — **candidate lookup (zero download).** Query for a package where - `sourceChannelId == destChannelId` AND `fileName == archiveName` AND - `fileSize == totalArchiveSize`. (`Package` has `@@index([fileName])`.) + `(sourceChannelId == destChannelId OR sourceMessageId == 0)` AND + `fileName == archiveName` AND `fileSize == totalArchiveSize`. + (`Package` has `@@index([fileName])`.) - No candidate → fall through to normal ingestion unchanged. 2. Stage B — **fingerprint confirmation (tiny download).** See §3. 3. On confirmation → **backfill** (see §4) and return `null` (treated as a