From 0cf5fcd3a78e2cf990eddd96f2379ef9cad95f45 Mon Sep 17 00:00:00 2001 From: xCyanGrizzly Date: Mon, 22 Jun 2026 19:05:34 +0200 Subject: [PATCH] fix(stls): stop loading preview blobs in list queries (OOM fix) The STLs list queries selected the full previewData JPEG bytes (avg ~700KB, up to 2MB, and every group member's preview) only to compute a hasPreview boolean. Under concurrent page loads this piled blobs into the Node heap, exceeded the 512MB container limit, crashed the process, and surfaced as repeated connection errors while browsing. Replace the byte-loading select with a fetchPreviewFlags() helper that checks `previewData IS NOT NULL` in SQL (IDs only, no bytes). Applied to listPackages, listDisplayItems, searchPackages, and listUngroupedPackages. Verified: same concurrent load that drove memory to 508/512MB and forced a restart now peaks at 156MB (30%) with zero failed requests and no restart. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/lib/telegram/queries.ts | 68 ++++++++++++++++++++++++++++++------- 1 file changed, 56 insertions(+), 12 deletions(-) diff --git a/src/lib/telegram/queries.ts b/src/lib/telegram/queries.ts index fbf6c3a..1c4a23b 100644 --- a/src/lib/telegram/queries.ts +++ b/src/lib/telegram/queries.ts @@ -9,6 +9,28 @@ import type { PackageGroupRow, } from "./types"; +/** + * Returns the subset of the given IDs whose `previewData` column is non-null, + * WITHOUT transferring the (large) image bytes. + * + * List views only need a `hasPreview` boolean per row. Selecting `previewData` + * directly pulls every JPEG blob (avg ~700 KB, up to 2 MB) into the Node heap + * just to compare it against null — under concurrent requests this exhausts the + * container memory limit and crashes the process. This keeps the check in SQL. + */ +async function fetchPreviewFlags( + table: "packages" | "package_groups", + ids: string[] +): Promise> { + if (ids.length === 0) return new Set(); + const placeholders = ids.map((_, i) => `$${i + 1}`).join(", "); + const rows = await prisma.$queryRawUnsafe<{ id: string }[]>( + `SELECT id FROM ${table} WHERE "previewData" IS NOT NULL AND id IN (${placeholders})`, + ...ids + ); + return new Set(rows.map((r) => r.id)); +} + export async function listPackages(options: { page: number; limit: number; @@ -40,13 +62,17 @@ export async function listPackages(options: { indexedAt: true, creator: true, tags: true, - previewData: true, // check actual image data, not previewMsgId proxy sourceChannel: { select: { id: true, title: true } }, }, }), prisma.package.count({ where }), ]); + const previewIds = await fetchPreviewFlags( + "packages", + items.map((p) => p.id) + ); + const mapped: PackageListItem[] = items.map((pkg) => ({ id: pkg.id, fileName: pkg.fileName, @@ -55,7 +81,7 @@ export async function listPackages(options: { archiveType: pkg.archiveType, fileCount: pkg.fileCount, isMultipart: pkg.isMultipart, - hasPreview: pkg.previewData !== null, + hasPreview: previewIds.has(pkg.id), creator: pkg.creator, tags: pkg.tags, indexedAt: pkg.indexedAt.toISOString(), @@ -149,7 +175,7 @@ export async function listDisplayItems(options: { select: { id: true, fileName: true, fileSize: true, contentHash: true, archiveType: true, fileCount: true, isMultipart: true, - indexedAt: true, creator: true, tags: true, previewData: true, + indexedAt: true, creator: true, tags: true, sourceChannel: { select: { id: true, title: true } }, }, }) @@ -159,13 +185,13 @@ export async function listDisplayItems(options: { ? await prisma.packageGroup.findMany({ where: { id: { in: groupIds } }, select: { - id: true, name: true, previewData: true, + id: true, name: true, sourceChannel: { select: { id: true, title: true } }, packages: { select: { id: true, fileName: true, fileSize: true, contentHash: true, archiveType: true, fileCount: true, isMultipart: true, - indexedAt: true, creator: true, tags: true, previewData: true, + indexedAt: true, creator: true, tags: true, sourceChannel: { select: { id: true, title: true } }, }, orderBy: { indexedAt: "desc" }, @@ -174,6 +200,16 @@ export async function listDisplayItems(options: { }) : []; + // Compute hasPreview flags without transferring image bytes (see fetchPreviewFlags) + const allPackageIds = [ + ...standalonePackages.map((p) => p.id), + ...groups.flatMap((g) => g.packages.map((p) => p.id)), + ]; + const [packagePreviewIds, groupPreviewIds] = await Promise.all([ + fetchPreviewFlags("packages", allPackageIds), + fetchPreviewFlags("package_groups", groups.map((g) => g.id)), + ]); + // Build DisplayItem array in the original sort order const packageMap = new Map(standalonePackages.map((p) => [p.id, p])); const groupMap = new Map(groups.map((g) => [g.id, g])); @@ -191,7 +227,7 @@ export async function listDisplayItems(options: { archiveType: pkg.archiveType, fileCount: pkg.fileCount, isMultipart: pkg.isMultipart, - hasPreview: pkg.previewData !== null, + hasPreview: packagePreviewIds.has(pkg.id), creator: pkg.creator, tags: pkg.tags, indexedAt: pkg.indexedAt.toISOString(), @@ -209,7 +245,7 @@ export async function listDisplayItems(options: { data: { id: grp.id, name: grp.name, - hasPreview: grp.previewData !== null, + hasPreview: groupPreviewIds.has(grp.id), totalFileSize: grp.packages.reduce((sum, p) => sum + p.fileSize, BigInt(0)).toString(), totalFileCount: grp.packages.reduce((sum, p) => sum + p.fileCount, 0), packageCount: grp.packages.length, @@ -227,7 +263,7 @@ export async function listDisplayItems(options: { archiveType: pkg.archiveType, fileCount: pkg.fileCount, isMultipart: pkg.isMultipart, - hasPreview: pkg.previewData !== null, + hasPreview: packagePreviewIds.has(pkg.id), creator: pkg.creator, tags: pkg.tags, indexedAt: pkg.indexedAt.toISOString(), @@ -440,13 +476,17 @@ export async function searchPackages(options: { indexedAt: true, creator: true, tags: true, - previewData: true, sourceChannel: { select: { id: true, title: true } }, }, }), Promise.resolve(allIds.length), ]); + const previewIds = await fetchPreviewFlags( + "packages", + items.map((p) => p.id) + ); + const mapped: PackageListItem[] = items.map((pkg) => ({ id: pkg.id, fileName: pkg.fileName, @@ -455,7 +495,7 @@ export async function searchPackages(options: { archiveType: pkg.archiveType, fileCount: pkg.fileCount, isMultipart: pkg.isMultipart, - hasPreview: pkg.previewData !== null, + hasPreview: previewIds.has(pkg.id), creator: pkg.creator, tags: pkg.tags, indexedAt: pkg.indexedAt.toISOString(), @@ -636,13 +676,17 @@ export async function listUngroupedPackages(options: { partCount: true, tags: true, indexedAt: true, - previewData: true, sourceChannel: { select: { id: true, title: true } }, }, }), prisma.package.count({ where }), ]); + const previewIds = await fetchPreviewFlags( + "packages", + items.map((p) => p.id) + ); + return { items: items.map((p) => ({ id: p.id, @@ -656,7 +700,7 @@ export async function listUngroupedPackages(options: { partCount: p.partCount, tags: p.tags, indexedAt: p.indexedAt.toISOString(), - hasPreview: !!p.previewData, + hasPreview: previewIds.has(p.id), sourceChannel: p.sourceChannel, matchedFileCount: 0, matchedByContent: false,