mirror of
https://github.com/xCyanGrizzly/DragonsStash.git
synced 2026-07-22 23:12:02 +00:00
fix(stls): stop loading preview blobs in list queries (OOM fix)
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
The STLs list queries selected the full previewData JPEG bytes (avg ~700KB, up to 2MB, and every group member's preview) only to compute a hasPreview boolean. Under concurrent page loads this piled blobs into the Node heap, exceeded the 512MB container limit, crashed the process, and surfaced as repeated connection errors while browsing. Replace the byte-loading select with a fetchPreviewFlags() helper that checks `previewData IS NOT NULL` in SQL (IDs only, no bytes). Applied to listPackages, listDisplayItems, searchPackages, and listUngroupedPackages. Verified: same concurrent load that drove memory to 508/512MB and forced a restart now peaks at 156MB (30%) with zero failed requests and no restart. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -9,6 +9,28 @@ import type {
|
|||||||
PackageGroupRow,
|
PackageGroupRow,
|
||||||
} from "./types";
|
} from "./types";
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the subset of the given IDs whose `previewData` column is non-null,
|
||||||
|
* WITHOUT transferring the (large) image bytes.
|
||||||
|
*
|
||||||
|
* List views only need a `hasPreview` boolean per row. Selecting `previewData`
|
||||||
|
* directly pulls every JPEG blob (avg ~700 KB, up to 2 MB) into the Node heap
|
||||||
|
* just to compare it against null — under concurrent requests this exhausts the
|
||||||
|
* container memory limit and crashes the process. This keeps the check in SQL.
|
||||||
|
*/
|
||||||
|
async function fetchPreviewFlags(
|
||||||
|
table: "packages" | "package_groups",
|
||||||
|
ids: string[]
|
||||||
|
): Promise<Set<string>> {
|
||||||
|
if (ids.length === 0) return new Set();
|
||||||
|
const placeholders = ids.map((_, i) => `$${i + 1}`).join(", ");
|
||||||
|
const rows = await prisma.$queryRawUnsafe<{ id: string }[]>(
|
||||||
|
`SELECT id FROM ${table} WHERE "previewData" IS NOT NULL AND id IN (${placeholders})`,
|
||||||
|
...ids
|
||||||
|
);
|
||||||
|
return new Set(rows.map((r) => r.id));
|
||||||
|
}
|
||||||
|
|
||||||
export async function listPackages(options: {
|
export async function listPackages(options: {
|
||||||
page: number;
|
page: number;
|
||||||
limit: number;
|
limit: number;
|
||||||
@@ -40,13 +62,17 @@ export async function listPackages(options: {
|
|||||||
indexedAt: true,
|
indexedAt: true,
|
||||||
creator: true,
|
creator: true,
|
||||||
tags: true,
|
tags: true,
|
||||||
previewData: true, // check actual image data, not previewMsgId proxy
|
|
||||||
sourceChannel: { select: { id: true, title: true } },
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
prisma.package.count({ where }),
|
prisma.package.count({ where }),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const previewIds = await fetchPreviewFlags(
|
||||||
|
"packages",
|
||||||
|
items.map((p) => p.id)
|
||||||
|
);
|
||||||
|
|
||||||
const mapped: PackageListItem[] = items.map((pkg) => ({
|
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||||
id: pkg.id,
|
id: pkg.id,
|
||||||
fileName: pkg.fileName,
|
fileName: pkg.fileName,
|
||||||
@@ -55,7 +81,7 @@ export async function listPackages(options: {
|
|||||||
archiveType: pkg.archiveType,
|
archiveType: pkg.archiveType,
|
||||||
fileCount: pkg.fileCount,
|
fileCount: pkg.fileCount,
|
||||||
isMultipart: pkg.isMultipart,
|
isMultipart: pkg.isMultipart,
|
||||||
hasPreview: pkg.previewData !== null,
|
hasPreview: previewIds.has(pkg.id),
|
||||||
creator: pkg.creator,
|
creator: pkg.creator,
|
||||||
tags: pkg.tags,
|
tags: pkg.tags,
|
||||||
indexedAt: pkg.indexedAt.toISOString(),
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
@@ -149,7 +175,7 @@ export async function listDisplayItems(options: {
|
|||||||
select: {
|
select: {
|
||||||
id: true, fileName: true, fileSize: true, contentHash: true,
|
id: true, fileName: true, fileSize: true, contentHash: true,
|
||||||
archiveType: true, fileCount: true, isMultipart: true,
|
archiveType: true, fileCount: true, isMultipart: true,
|
||||||
indexedAt: true, creator: true, tags: true, previewData: true,
|
indexedAt: true, creator: true, tags: true,
|
||||||
sourceChannel: { select: { id: true, title: true } },
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
@@ -159,13 +185,13 @@ export async function listDisplayItems(options: {
|
|||||||
? await prisma.packageGroup.findMany({
|
? await prisma.packageGroup.findMany({
|
||||||
where: { id: { in: groupIds } },
|
where: { id: { in: groupIds } },
|
||||||
select: {
|
select: {
|
||||||
id: true, name: true, previewData: true,
|
id: true, name: true,
|
||||||
sourceChannel: { select: { id: true, title: true } },
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
packages: {
|
packages: {
|
||||||
select: {
|
select: {
|
||||||
id: true, fileName: true, fileSize: true, contentHash: true,
|
id: true, fileName: true, fileSize: true, contentHash: true,
|
||||||
archiveType: true, fileCount: true, isMultipart: true,
|
archiveType: true, fileCount: true, isMultipart: true,
|
||||||
indexedAt: true, creator: true, tags: true, previewData: true,
|
indexedAt: true, creator: true, tags: true,
|
||||||
sourceChannel: { select: { id: true, title: true } },
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
},
|
},
|
||||||
orderBy: { indexedAt: "desc" },
|
orderBy: { indexedAt: "desc" },
|
||||||
@@ -174,6 +200,16 @@ export async function listDisplayItems(options: {
|
|||||||
})
|
})
|
||||||
: [];
|
: [];
|
||||||
|
|
||||||
|
// Compute hasPreview flags without transferring image bytes (see fetchPreviewFlags)
|
||||||
|
const allPackageIds = [
|
||||||
|
...standalonePackages.map((p) => p.id),
|
||||||
|
...groups.flatMap((g) => g.packages.map((p) => p.id)),
|
||||||
|
];
|
||||||
|
const [packagePreviewIds, groupPreviewIds] = await Promise.all([
|
||||||
|
fetchPreviewFlags("packages", allPackageIds),
|
||||||
|
fetchPreviewFlags("package_groups", groups.map((g) => g.id)),
|
||||||
|
]);
|
||||||
|
|
||||||
// Build DisplayItem array in the original sort order
|
// Build DisplayItem array in the original sort order
|
||||||
const packageMap = new Map(standalonePackages.map((p) => [p.id, p]));
|
const packageMap = new Map(standalonePackages.map((p) => [p.id, p]));
|
||||||
const groupMap = new Map(groups.map((g) => [g.id, g]));
|
const groupMap = new Map(groups.map((g) => [g.id, g]));
|
||||||
@@ -191,7 +227,7 @@ export async function listDisplayItems(options: {
|
|||||||
archiveType: pkg.archiveType,
|
archiveType: pkg.archiveType,
|
||||||
fileCount: pkg.fileCount,
|
fileCount: pkg.fileCount,
|
||||||
isMultipart: pkg.isMultipart,
|
isMultipart: pkg.isMultipart,
|
||||||
hasPreview: pkg.previewData !== null,
|
hasPreview: packagePreviewIds.has(pkg.id),
|
||||||
creator: pkg.creator,
|
creator: pkg.creator,
|
||||||
tags: pkg.tags,
|
tags: pkg.tags,
|
||||||
indexedAt: pkg.indexedAt.toISOString(),
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
@@ -209,7 +245,7 @@ export async function listDisplayItems(options: {
|
|||||||
data: {
|
data: {
|
||||||
id: grp.id,
|
id: grp.id,
|
||||||
name: grp.name,
|
name: grp.name,
|
||||||
hasPreview: grp.previewData !== null,
|
hasPreview: groupPreviewIds.has(grp.id),
|
||||||
totalFileSize: grp.packages.reduce((sum, p) => sum + p.fileSize, BigInt(0)).toString(),
|
totalFileSize: grp.packages.reduce((sum, p) => sum + p.fileSize, BigInt(0)).toString(),
|
||||||
totalFileCount: grp.packages.reduce((sum, p) => sum + p.fileCount, 0),
|
totalFileCount: grp.packages.reduce((sum, p) => sum + p.fileCount, 0),
|
||||||
packageCount: grp.packages.length,
|
packageCount: grp.packages.length,
|
||||||
@@ -227,7 +263,7 @@ export async function listDisplayItems(options: {
|
|||||||
archiveType: pkg.archiveType,
|
archiveType: pkg.archiveType,
|
||||||
fileCount: pkg.fileCount,
|
fileCount: pkg.fileCount,
|
||||||
isMultipart: pkg.isMultipart,
|
isMultipart: pkg.isMultipart,
|
||||||
hasPreview: pkg.previewData !== null,
|
hasPreview: packagePreviewIds.has(pkg.id),
|
||||||
creator: pkg.creator,
|
creator: pkg.creator,
|
||||||
tags: pkg.tags,
|
tags: pkg.tags,
|
||||||
indexedAt: pkg.indexedAt.toISOString(),
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
@@ -440,13 +476,17 @@ export async function searchPackages(options: {
|
|||||||
indexedAt: true,
|
indexedAt: true,
|
||||||
creator: true,
|
creator: true,
|
||||||
tags: true,
|
tags: true,
|
||||||
previewData: true,
|
|
||||||
sourceChannel: { select: { id: true, title: true } },
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
Promise.resolve(allIds.length),
|
Promise.resolve(allIds.length),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const previewIds = await fetchPreviewFlags(
|
||||||
|
"packages",
|
||||||
|
items.map((p) => p.id)
|
||||||
|
);
|
||||||
|
|
||||||
const mapped: PackageListItem[] = items.map((pkg) => ({
|
const mapped: PackageListItem[] = items.map((pkg) => ({
|
||||||
id: pkg.id,
|
id: pkg.id,
|
||||||
fileName: pkg.fileName,
|
fileName: pkg.fileName,
|
||||||
@@ -455,7 +495,7 @@ export async function searchPackages(options: {
|
|||||||
archiveType: pkg.archiveType,
|
archiveType: pkg.archiveType,
|
||||||
fileCount: pkg.fileCount,
|
fileCount: pkg.fileCount,
|
||||||
isMultipart: pkg.isMultipart,
|
isMultipart: pkg.isMultipart,
|
||||||
hasPreview: pkg.previewData !== null,
|
hasPreview: previewIds.has(pkg.id),
|
||||||
creator: pkg.creator,
|
creator: pkg.creator,
|
||||||
tags: pkg.tags,
|
tags: pkg.tags,
|
||||||
indexedAt: pkg.indexedAt.toISOString(),
|
indexedAt: pkg.indexedAt.toISOString(),
|
||||||
@@ -636,13 +676,17 @@ export async function listUngroupedPackages(options: {
|
|||||||
partCount: true,
|
partCount: true,
|
||||||
tags: true,
|
tags: true,
|
||||||
indexedAt: true,
|
indexedAt: true,
|
||||||
previewData: true,
|
|
||||||
sourceChannel: { select: { id: true, title: true } },
|
sourceChannel: { select: { id: true, title: true } },
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
prisma.package.count({ where }),
|
prisma.package.count({ where }),
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
const previewIds = await fetchPreviewFlags(
|
||||||
|
"packages",
|
||||||
|
items.map((p) => p.id)
|
||||||
|
);
|
||||||
|
|
||||||
return {
|
return {
|
||||||
items: items.map((p) => ({
|
items: items.map((p) => ({
|
||||||
id: p.id,
|
id: p.id,
|
||||||
@@ -656,7 +700,7 @@ export async function listUngroupedPackages(options: {
|
|||||||
partCount: p.partCount,
|
partCount: p.partCount,
|
||||||
tags: p.tags,
|
tags: p.tags,
|
||||||
indexedAt: p.indexedAt.toISOString(),
|
indexedAt: p.indexedAt.toISOString(),
|
||||||
hasPreview: !!p.previewData,
|
hasPreview: previewIds.has(p.id),
|
||||||
sourceChannel: p.sourceChannel,
|
sourceChannel: p.sourceChannel,
|
||||||
matchedFileCount: 0,
|
matchedFileCount: 0,
|
||||||
matchedByContent: false,
|
matchedByContent: false,
|
||||||
|
|||||||
Reference in New Issue
Block a user