mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-06-13 12:41:16 +00:00
perf: set watermarks even when no archives found to prevent re-scanning
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
Previously, channels/topics with no new archives never had their watermark updated. This meant every cycle re-scanned all messages from scratch just to discover nothing new — especially costly for the 1079- topic Model Printing Emporium forum. - Add maxScannedMessageId to ChannelScanResult (highest msg ID seen) - Set channel watermark to scan boundary when no archives are found - Set topic watermark to scan boundary when no archives are found - Fall back to scan watermark when archive processing doesn't advance it After one full cycle, subsequent cycles will skip already-scanned messages via the early-exit boundary check, dramatically reducing TDLib API calls on channels with mostly non-archive content. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -79,6 +79,8 @@ export interface ChannelScanResult {
|
||||
archives: TelegramMessage[];
|
||||
photos: TelegramPhoto[];
|
||||
totalScanned: number;
|
||||
/** Highest message ID seen during scan (for watermark, even when no archives found). */
|
||||
maxScannedMessageId: bigint | null;
|
||||
}
|
||||
|
||||
export type ScanProgressCallback = (messagesScanned: number) => void;
|
||||
@@ -158,6 +160,7 @@ export async function getChannelMessages(
|
||||
const archives: TelegramMessage[] = [];
|
||||
const photos: TelegramPhoto[] = [];
|
||||
const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
|
||||
let maxScannedMessageId: bigint | null = null;
|
||||
|
||||
// Open the chat so TDLib can access it
|
||||
try {
|
||||
@@ -204,6 +207,12 @@ export async function getChannelMessages(
|
||||
|
||||
totalScanned += result.messages.length;
|
||||
|
||||
// Track highest message ID (first message in batch = newest, since results are newest-first)
|
||||
const batchMaxId = BigInt(result.messages[0].id);
|
||||
if (maxScannedMessageId === null || batchMaxId > maxScannedMessageId) {
|
||||
maxScannedMessageId = batchMaxId;
|
||||
}
|
||||
|
||||
for (const msg of result.messages) {
|
||||
// Check for archive documents
|
||||
const doc = msg.content?.document;
|
||||
@@ -271,6 +280,7 @@ export async function getChannelMessages(
|
||||
archives: archives.reverse(),
|
||||
photos: photos.reverse(),
|
||||
totalScanned,
|
||||
maxScannedMessageId,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -178,6 +178,7 @@ export async function getTopicMessages(
|
||||
const archives: TelegramMessage[] = [];
|
||||
const photos: TelegramPhoto[] = [];
|
||||
const boundary = lastProcessedMessageId ? Number(lastProcessedMessageId) : null;
|
||||
let maxScannedMessageId: bigint | null = null;
|
||||
|
||||
let currentFromId = 0;
|
||||
let totalScanned = 0;
|
||||
@@ -239,6 +240,12 @@ export async function getTopicMessages(
|
||||
|
||||
totalScanned += result.messages.length;
|
||||
|
||||
// Track highest message ID (first message = newest, since results are newest-first)
|
||||
const batchMaxId = BigInt(result.messages[0].id);
|
||||
if (maxScannedMessageId === null || batchMaxId > maxScannedMessageId) {
|
||||
maxScannedMessageId = batchMaxId;
|
||||
}
|
||||
|
||||
for (const msg of result.messages) {
|
||||
// Check for archive documents
|
||||
const doc = msg.content?.document;
|
||||
@@ -302,6 +309,7 @@ export async function getTopicMessages(
|
||||
archives: archives.reverse(),
|
||||
photos: photos.reverse(),
|
||||
totalScanned,
|
||||
maxScannedMessageId,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user