fix(stls): stop loading preview blobs in list queries (OOM fix)
All checks were successful
continuous-integration/drone/push Build is passing

The STLs list queries selected the full previewData JPEG bytes (avg
~700KB, up to 2MB, and every group member's preview) only to compute a
hasPreview boolean. Under concurrent page loads this piled blobs into
the Node heap, exceeded the 512MB container limit, crashed the process,
and surfaced as repeated connection errors while browsing.

Replace the byte-loading select with a fetchPreviewFlags() helper that
checks `previewData IS NOT NULL` in SQL (IDs only, no bytes). Applied to
listPackages, listDisplayItems, searchPackages, and listUngroupedPackages.

Verified: same concurrent load that drove memory to 508/512MB and forced
a restart now peaks at 156MB (30%) with zero failed requests and no restart.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-06-22 19:05:34 +02:00
parent 3b7202a662
commit 0cf5fcd3a7

View File

@@ -9,6 +9,28 @@ import type {
PackageGroupRow,
} from "./types";
/**
* Returns the subset of the given IDs whose `previewData` column is non-null,
* WITHOUT transferring the (large) image bytes.
*
* List views only need a `hasPreview` boolean per row. Selecting `previewData`
* directly pulls every JPEG blob (avg ~700 KB, up to 2 MB) into the Node heap
* just to compare it against null — under concurrent requests this exhausts the
* container memory limit and crashes the process. This keeps the check in SQL.
*/
async function fetchPreviewFlags(
table: "packages" | "package_groups",
ids: string[]
): Promise<Set<string>> {
if (ids.length === 0) return new Set();
const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ");
const rows = await prisma.$queryRawUnsafe<{ id: string }[]>(
`SELECT id FROM ${table} WHERE "previewData" IS NOT NULL AND id IN (${placeholders})`,
...ids
);
return new Set(rows.map((r) => r.id));
}
export async function listPackages(options: {
page: number;
limit: number;
@@ -40,13 +62,17 @@ export async function listPackages(options: {
indexedAt: true,
creator: true,
tags: true,
previewData: true, // check actual image data, not previewMsgId proxy
sourceChannel: { select: { id: true, title: true } },
},
}),
prisma.package.count({ where }),
]);
const previewIds = await fetchPreviewFlags(
"packages",
items.map((p) => p.id)
);
const mapped: PackageListItem[] = items.map((pkg) => ({
id: pkg.id,
fileName: pkg.fileName,
@@ -55,7 +81,7 @@ export async function listPackages(options: {
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewData !== null,
hasPreview: previewIds.has(pkg.id),
creator: pkg.creator,
tags: pkg.tags,
indexedAt: pkg.indexedAt.toISOString(),
@@ -149,7 +175,7 @@ export async function listDisplayItems(options: {
select: {
id: true, fileName: true, fileSize: true, contentHash: true,
archiveType: true, fileCount: true, isMultipart: true,
indexedAt: true, creator: true, tags: true, previewData: true,
indexedAt: true, creator: true, tags: true,
sourceChannel: { select: { id: true, title: true } },
},
})
@@ -159,13 +185,13 @@ export async function listDisplayItems(options: {
? await prisma.packageGroup.findMany({
where: { id: { in: groupIds } },
select: {
id: true, name: true, previewData: true,
id: true, name: true,
sourceChannel: { select: { id: true, title: true } },
packages: {
select: {
id: true, fileName: true, fileSize: true, contentHash: true,
archiveType: true, fileCount: true, isMultipart: true,
indexedAt: true, creator: true, tags: true, previewData: true,
indexedAt: true, creator: true, tags: true,
sourceChannel: { select: { id: true, title: true } },
},
orderBy: { indexedAt: "desc" },
@@ -174,6 +200,16 @@ export async function listDisplayItems(options: {
})
: [];
// Compute hasPreview flags without transferring image bytes (see fetchPreviewFlags)
const allPackageIds = [
...standalonePackages.map((p) => p.id),
...groups.flatMap((g) => g.packages.map((p) => p.id)),
];
const [packagePreviewIds, groupPreviewIds] = await Promise.all([
fetchPreviewFlags("packages", allPackageIds),
fetchPreviewFlags("package_groups", groups.map((g) => g.id)),
]);
// Build DisplayItem array in the original sort order
const packageMap = new Map(standalonePackages.map((p) => [p.id, p]));
const groupMap = new Map(groups.map((g) => [g.id, g]));
@@ -191,7 +227,7 @@ export async function listDisplayItems(options: {
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewData !== null,
hasPreview: packagePreviewIds.has(pkg.id),
creator: pkg.creator,
tags: pkg.tags,
indexedAt: pkg.indexedAt.toISOString(),
@@ -209,7 +245,7 @@ export async function listDisplayItems(options: {
data: {
id: grp.id,
name: grp.name,
hasPreview: grp.previewData !== null,
hasPreview: groupPreviewIds.has(grp.id),
totalFileSize: grp.packages.reduce((sum, p) => sum + p.fileSize, BigInt(0)).toString(),
totalFileCount: grp.packages.reduce((sum, p) => sum + p.fileCount, 0),
packageCount: grp.packages.length,
@@ -227,7 +263,7 @@ export async function listDisplayItems(options: {
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewData !== null,
hasPreview: packagePreviewIds.has(pkg.id),
creator: pkg.creator,
tags: pkg.tags,
indexedAt: pkg.indexedAt.toISOString(),
@@ -440,13 +476,17 @@ export async function searchPackages(options: {
indexedAt: true,
creator: true,
tags: true,
previewData: true,
sourceChannel: { select: { id: true, title: true } },
},
}),
Promise.resolve(allIds.length),
]);
const previewIds = await fetchPreviewFlags(
"packages",
items.map((p) => p.id)
);
const mapped: PackageListItem[] = items.map((pkg) => ({
id: pkg.id,
fileName: pkg.fileName,
@@ -455,7 +495,7 @@ export async function searchPackages(options: {
archiveType: pkg.archiveType,
fileCount: pkg.fileCount,
isMultipart: pkg.isMultipart,
hasPreview: pkg.previewData !== null,
hasPreview: previewIds.has(pkg.id),
creator: pkg.creator,
tags: pkg.tags,
indexedAt: pkg.indexedAt.toISOString(),
@@ -636,13 +676,17 @@ export async function listUngroupedPackages(options: {
partCount: true,
tags: true,
indexedAt: true,
previewData: true,
sourceChannel: { select: { id: true, title: true } },
},
}),
prisma.package.count({ where }),
]);
const previewIds = await fetchPreviewFlags(
"packages",
items.map((p) => p.id)
);
return {
items: items.map((p) => ({
id: p.id,
@@ -656,7 +700,7 @@ export async function listUngroupedPackages(options: {
partCount: p.partCount,
tags: p.tags,
indexedAt: p.indexedAt.toISOString(),
hasPreview: !!p.previewData,
hasPreview: previewIds.has(p.id),
sourceChannel: p.sourceChannel,
matchedFileCount: 0,
matchedByContent: false,