mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-07-24 07:42:47 +00:00
feat(worker): use TDLib remote.unique_id as zero-false-positive dedup signal
The fileName + size repost detection from ff4e150 works but has a
theoretical false-positive: two unrelated files in the same channel
with identical names and identical total sizes get treated as duplicates.
TDLib's document.remote.unique_id is a stable identifier per file
content — every repost of the exact same file across messages keeps
the same unique_id. Using it as the first dedup check eliminates the
false-positive risk entirely.
Schema:
- Package.remoteUniqueId (nullable, since existing rows lack it)
- Index on (sourceChannelId, remoteUniqueId)
Pipeline:
1. Capture remoteUniqueId in getChannelMessages + getTopicMessages
2. Pass through TelegramMessage type
3. processOneArchiveSet checks findPackageByRemoteUniqueId FIRST
(before packageExistsBySourceMessage / findRepostedPackage)
4. createPackageStub stores it on the new Package row
Existing 19,952 Packages have remoteUniqueId = NULL — they fall through
to the existing checks (source-msg-id, name+size, content-hash). New
ingestions populate it and benefit from the strong signal immediately.
Old Packages get backfilled organically when their content is
re-encountered and a new Package would otherwise be created.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -32,6 +32,7 @@ import {
|
||||
deleteSkippedPackage,
|
||||
getCappedSkippedMessageIds,
|
||||
findRepostedPackage,
|
||||
findPackageByRemoteUniqueId,
|
||||
getRetryableSkippedMessageIds,
|
||||
updatePackageTopicContext,
|
||||
} from "./db/queries.js";
|
||||
@@ -1240,6 +1241,38 @@ async function processOneArchiveSet(
|
||||
|
||||
const archiveName = archiveSet.parts[0].fileName;
|
||||
|
||||
// ── Earliest skip: remote.unique_id match ──
|
||||
// TDLib reports a stable unique_id per file content. If we already have a
|
||||
// Package in this channel with the same unique_id, it's the exact same
|
||||
// file content reposted at a new message ID — zero false positives.
|
||||
const firstRemoteUniqueId = archiveSet.parts[0].remoteUniqueId;
|
||||
if (firstRemoteUniqueId) {
|
||||
const match = await findPackageByRemoteUniqueId(channel.id, firstRemoteUniqueId);
|
||||
if (match) {
|
||||
counters.zipsDuplicate++;
|
||||
accountLog.info(
|
||||
{
|
||||
fileName: archiveSet.parts[0].fileName,
|
||||
sourceMessageId: Number(archiveSet.parts[0].id),
|
||||
remoteUniqueId: firstRemoteUniqueId,
|
||||
existingPackageId: match.id,
|
||||
existingDestMessageId: match.destMessageId ? Number(match.destMessageId) : null,
|
||||
},
|
||||
"Skipping — remote.unique_id matches an existing Package in this channel"
|
||||
);
|
||||
await updateRunActivity(runId, {
|
||||
currentActivity: `Skipped ${archiveSet.parts[0].fileName} (already ingested by unique_id)`,
|
||||
currentStep: "deduplicating",
|
||||
currentChannel: channelTitle,
|
||||
currentFile: archiveSet.parts[0].fileName,
|
||||
currentFileNum: setIdx + 1,
|
||||
totalFiles: totalSets,
|
||||
zipsDuplicate: counters.zipsDuplicate,
|
||||
});
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// ── Early skip: check if this archive set was already ingested ──
|
||||
// This avoids re-downloading large archives that were processed in a prior run.
|
||||
const alreadyIngested = await packageExistsBySourceMessage(
|
||||
@@ -1705,6 +1738,7 @@ async function processOneArchiveSet(
|
||||
sourceChannelId: channel.id,
|
||||
sourceMessageId: archiveSet.parts[0].id,
|
||||
sourceTopicId,
|
||||
remoteUniqueId: archiveSet.parts[0].remoteUniqueId ?? null,
|
||||
destChannelId,
|
||||
destMessageId: destResult.messageId,
|
||||
destMessageIds: destResult.messageIds,
|
||||
|
||||
Reference in New Issue
Block a user