admin/dragonsstash
1
0
Fork 0
mirror of https://github.com/xCyanGrizzly/DragonsStash.git synced 2026-06-13 12:41:16 +00:00
Code Issues Packages Projects Releases Wiki Activity

perf: set watermarks even when no archives found to prevent re-scanning
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source 
Previously, channels/topics with no new archives never had their
watermark updated. This meant every cycle re-scanned all messages from
scratch just to discover nothing new — especially costly for the 1079-
topic Model Printing Emporium forum.

- Add maxScannedMessageId to ChannelScanResult (highest msg ID seen)
- Set channel watermark to scan boundary when no archives are found
- Set topic watermark to scan boundary when no archives are found
- Fall back to scan watermark when archive processing doesn't advance it

After one full cycle, subsequent cycles will skip already-scanned
messages via the early-exit boundary check, dramatically reducing
TDLib API calls on channels with mostly non-archive content.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
xCyanGrizzly
2026-05-04 20:37:42 +02:00
parent 35cce3151c
commit 77c26adb31
3 changed files with 40 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
worker/src/worker.ts
View File

@@ -538,6 +538,15 @@ export async function runWorkerForAccount(
{ channelId: channel.id, topic: topic.name, totalScanned: scanResult.totalScanned },
"No new archives in topic"
);
// Still advance topic watermark so we don't re-scan these messages next cycle
if (scanResult.maxScannedMessageId) {
await upsertTopicProgress(
mapping.id,
topic.topicId,
topic.name,
scanResult.maxScannedMessageId
);
}
continue;
}
@@ -555,13 +564,14 @@ export async function runWorkerForAccount(
// Sync client back in case it was recreated during upload stall recovery
client = pipelineCtx.client;
// Only advance progress to the highest successfully processed message
if (maxProcessedId) {
// Advance progress: use archive watermark if available, fall back to scan watermark
const topicWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
if (topicWatermark) {
await upsertTopicProgress(
mapping.id,
topic.topicId,
topic.name,
maxProcessedId
topicWatermark
);
}
} catch (topicErr) {
@@ -611,6 +621,11 @@ export async function runWorkerForAccount(
if (scanResult.archives.length === 0) {
accountLog.info({ channelId: channel.id, title: channel.title, totalScanned: scanResult.totalScanned }, "No new archives in channel");
// Still advance watermark to highest scanned message so we don't
// re-scan these messages next cycle
if (scanResult.maxScannedMessageId) {
await updateLastProcessedMessage(mapping.id, scanResult.maxScannedMessageId);
}
continue;
}
@@ -628,9 +643,10 @@ export async function runWorkerForAccount(
// Sync client back in case it was recreated during upload stall recovery
client = pipelineCtx.client;
// Only advance progress to the highest successfully processed message
if (maxProcessedId) {
await updateLastProcessedMessage(mapping.id, maxProcessedId);
// Advance progress: use archive watermark if available, fall back to scan watermark
const channelWatermark = maxProcessedId ?? scanResult.maxScannedMessageId;
if (channelWatermark) {
await updateLastProcessedMessage(mapping.id, channelWatermark);
}
}
} catch (channelErr) {