fix(server): thumbnail queueing (#26077)

* fix thumbnail queueing

* add bmp

* other isEdited column

server/src/queries/asset.job.repository.sql

@@ -78,43 +78,13 @@ limit
 -- AssetJobRepository.streamForThumbnailJob
 select
   "asset"."id",
-  "asset"."thumbhash",
+  "asset"."isEdited"
-  (
-    select
-      coalesce(json_agg(agg), '[]')
-    from
-      (
-        select
-          "asset_file"."id",
-          "asset_file"."path",
-          "asset_file"."type",
-          "asset_file"."isEdited"
-        from
-          "asset_file"
-        where
-          "asset_file"."assetId" = "asset"."id"
-      ) as agg
-  ) as "files",
-  (
-    select
-      coalesce(json_agg(agg), '[]')
-    from
-      (
-        select
-          "asset_edit"."action",
-          "asset_edit"."parameters"
-        from
-          "asset_edit"
-        where
-          "asset_edit"."assetId" = "asset"."id"
-      ) as agg
-  ) as "edits"
 from
   "asset"
   inner join "asset_job_status" on "asset_job_status"."assetId" = "asset"."id"
 where
   "asset"."deletedAt" is null
-  and "asset"."visibility" != $1
+  and "asset"."visibility" != 'hidden'
   and (
     not exists (
       select
@@ -122,7 +92,7 @@ where
         "asset_file"
       where
         "assetId" = "asset"."id"
-        and "asset_file"."type" = $2
+        and "type" = 'thumbnail'
     )
     or not exists (
       select
@@ -130,17 +100,75 @@ where
         "asset_file"
       where
         "assetId" = "asset"."id"
-        and "asset_file"."type" = $3
+        and "type" = 'preview'
     )
-    or not exists (
+    or (
-      select
+      "asset"."isEdited" = true
-      from
+      and not exists (
-        "asset_file"
+        select
-      where
+        from
-        "assetId" = "asset"."id"
+          "asset_file"
-        and "asset_file"."type" = $4
+        where
+          "assetId" = "asset"."id"
+          and "type" = 'fullsize'
+          and "asset_file"."isEdited" = true
+      )
     )
     or "asset"."thumbhash" is null
+    or (
+      not exists (
+        select
+        from
+          "asset_file"
+        where
+          "assetId" = "asset"."id"
+          and "type" = 'fullsize'
+      )
+      and f_unaccent (asset."originalFileName") like any (
+        array[
+          '%.3fr',
+          '%.ari',
+          '%.arw',
+          '%.cap',
+          '%.cin',
+          '%.cr2',
+          '%.cr3',
+          '%.crw',
+          '%.dcr',
+          '%.dng',
+          '%.erf',
+          '%.fff',
+          '%.iiq',
+          '%.k25',
+          '%.kdc',
+          '%.mrw',
+          '%.nef',
+          '%.nrw',
+          '%.orf',
+          '%.ori',
+          '%.pef',
+          '%.psd',
+          '%.raf',
+          '%.raw',
+          '%.rw2',
+          '%.rwl',
+          '%.sr2',
+          '%.srf',
+          '%.srw',
+          '%.x3f',
+          '%.heic',
+          '%.heif',
+          '%.hif',
+          '%.insp',
+          '%.jp2',
+          '%.jpe',
+          '%.jxl',
+          '%.svg',
+          '%.tif',
+          '%.tiff'
+        ]::text[]
+      )
+    )
   )
 
 -- AssetJobRepository.getForMigrationJob

server/src/repositories/asset-job.repository.ts

@@ -18,6 +18,7 @@ import {
   withFilePath,
   withFiles,
 } from 'src/utils/database';
+import { mimeTypes } from 'src/utils/mime-types';
 
 @Injectable()
 export class AssetJobRepository {
@@ -61,51 +62,40 @@ export class AssetJobRepository {
   streamForThumbnailJob(options: { force: boolean | undefined; fullsizeEnabled: boolean }) {
     return this.db
       .selectFrom('asset')
-      .select(['asset.id', 'asset.thumbhash'])
+      .select(['asset.id', 'asset.isEdited'])
-      .select(withFiles)
-      .select(withEdits)
       .where('asset.deletedAt', 'is', null)
-      .where('asset.visibility', '!=', AssetVisibility.Hidden)
+      .where('asset.visibility', '!=', sql.lit(AssetVisibility.Hidden))
       .$if(!options.force, (qb) =>
         qb
           // If there aren't any entries, metadata extraction hasn't run yet which is required for thumbnails
           .innerJoin('asset_job_status', 'asset_job_status.assetId', 'asset.id')
-          .where((eb) => {
+          .where(({ and, eb, exists, not, or, selectFrom }) => {
+            const file = (type: AssetFileType) =>
+              selectFrom('asset_file').whereRef('assetId', '=', 'asset.id').where('type', '=', sql.lit(type));
+
             const conditions = [
-              eb.not((eb) =>
+              not(exists(file(AssetFileType.Thumbnail))),
-                eb.exists((qb) =>
+              not(exists(file(AssetFileType.Preview))),
-                  qb
+              and([
-                    .selectFrom('asset_file')
+                eb('asset.isEdited', '=', sql.lit(true)),
-                    .whereRef('assetId', '=', 'asset.id')
+                not(exists(file(AssetFileType.FullSize).where('asset_file.isEdited', '=', sql.lit(true)))),
-                    .where('asset_file.type', '=', AssetFileType.Preview),
+              ]),
-                ),
+              eb('asset.thumbhash', 'is', null),
-              ),
-              eb.not((eb) =>
-                eb.exists((qb) =>
-                  qb
-                    .selectFrom('asset_file')
-                    .whereRef('assetId', '=', 'asset.id')
-                    .where('asset_file.type', '=', AssetFileType.Thumbnail),
-                ),
-              ),
             ];
 
             if (options.fullsizeEnabled) {
+              const isWebUnsupported = sql.join(
+                Object.keys(mimeTypes.webUnsupportedImage).map((ext) => sql.lit(`%${ext}`)),
+              );
               conditions.push(
-                eb.not((eb) =>
+                and([
-                  eb.exists((qb) =>
+                  not(exists(file(AssetFileType.FullSize))),
-                    qb
+                  eb(sql`f_unaccent(asset."originalFileName")`, 'like', sql`any(array[${isWebUnsupported}]::text[])`),
-                      .selectFrom('asset_file')
+                ]),
-                      .whereRef('assetId', '=', 'asset.id')
-                      .where('asset_file.type', '=', AssetFileType.FullSize),
-                  ),
-                ),
               );
             }
 
-            conditions.push(eb('asset.thumbhash', 'is', null));
+            return or(conditions);
-
-            return eb.or(conditions);
           }),
       )
       .stream();

server/src/services/media.service.spec.ts

@@ -27,11 +27,6 @@ import { systemConfigStub } from 'test/fixtures/system-config.stub';
 import { factory } from 'test/small.factory';
 import { makeStream, newTestService, ServiceMocks } from 'test/utils';
 
-const filesNoFullsize = [
-  factory.assetFile({ type: AssetFileType.Preview }),
-  factory.assetFile({ type: AssetFileType.Thumbnail }),
-];
-
 const fullsizeBuffer = Buffer.from('embedded image data');
 const rawBuffer = Buffer.from('raw image data');
 const extractedBuffer = Buffer.from('embedded image file');
@@ -171,7 +166,7 @@ describe(MediaService.name, () => {
 
     it('should queue all assets with missing fullsize when feature is enabled', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: true } } });
-      const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize };
+      const asset = { id: factory.uuid(), isEdited: false };
       mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));
       mocks.person.getAll.mockReturnValue(makeStream());
       await sut.handleQueueGenerateThumbnails({ force: false });
@@ -189,7 +184,7 @@ describe(MediaService.name, () => {
 
     it('should not queue assets with missing fullsize when feature is disabled', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
-      const asset = { id: factory.uuid(), thumbhash: factory.buffer(), edits: [], files: filesNoFullsize };
+      const asset = { id: factory.uuid(), isEdited: false };
       mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));
       mocks.person.getAll.mockReturnValue(makeStream());
       await sut.handleQueueGenerateThumbnails({ force: false });
@@ -230,7 +225,7 @@ describe(MediaService.name, () => {
 
     it('should queue assets with missing fullsize when force is true, regardless of setting', async () => {
       mocks.systemMetadata.get.mockResolvedValue({ image: { fullsize: { enabled: false } } });
-      const asset = { id: factory.uuid(), thumbhash: Buffer.from('thumbhash'), edits: [], files: filesNoFullsize };
+      const asset = { id: factory.uuid(), isEdited: false };
       mocks.assetJob.streamForThumbnailJob.mockReturnValue(makeStream([asset]));
       mocks.person.getAll.mockReturnValue(makeStream());
       await sut.handleQueueGenerateThumbnails({ force: true });

server/src/services/media.service.ts

@@ -39,7 +39,7 @@ import {
   VideoInterfaces,
   VideoStreamInfo,
 } from 'src/types';
-import { getAssetFiles, getDimensions } from 'src/utils/asset.util';
+import { getDimensions } from 'src/utils/asset.util';
 import { checkFaceVisibility, checkOcrVisibility } from 'src/utils/editor';
 import { BaseConfig, ThumbnailConfig } from 'src/utils/media';
 import { mimeTypes } from 'src/utils/mime-types';
@@ -78,17 +78,11 @@ export class MediaService extends BaseService {
 
     const fullsizeEnabled = config.image.fullsize.enabled;
     for await (const asset of this.assetJobRepository.streamForThumbnailJob({ force, fullsizeEnabled })) {
-      const { previewFile, thumbnailFile, fullsizeFile, editedPreviewFile, editedThumbnailFile, editedFullsizeFile } =
+      if (force || !asset.isEdited) {
-        getAssetFiles(asset.files);
-
-      if (force || !previewFile || !thumbnailFile || !asset.thumbhash || (fullsizeEnabled && !fullsizeFile)) {
         jobs.push({ name: JobName.AssetGenerateThumbnails, data: { id: asset.id } });
       }
 
-      if (
+      if (asset.isEdited) {
-        asset.edits.length > 0 &&
-        (force || !editedPreviewFile || !editedThumbnailFile || (fullsizeEnabled && !editedFullsizeFile))
-      ) {
         jobs.push({ name: JobName.AssetEditThumbnailGeneration, data: { id: asset.id } });
       }
 

server/src/utils/mime-types.ts

@@ -1,7 +1,7 @@
 import { extname } from 'node:path';
 import { AssetType } from 'src/enum';
 
-const raw: Record<string, string[]> = {
+const raw = {
   '.3fr': ['image/3fr', 'image/x-hasselblad-3fr'],
   '.ari': ['image/ari', 'image/x-arriflex-ari'],
   '.arw': ['image/arw', 'image/x-sony-arw'],
@@ -41,6 +41,7 @@ const raw: Record<string, string[]> = {
  **/
 const webSupportedImage = {
   '.avif': ['image/avif'],
+  '.bmp': ['image/bmp'],
   '.gif': ['image/gif'],
   '.jpeg': ['image/jpeg'],
   '.jpg': ['image/jpeg'],
@@ -48,10 +49,8 @@ const webSupportedImage = {
   '.webp': ['image/webp'],
 };
 
-const image: Record<string, string[]> = {
+const webUnsupportedImage = {
   ...raw,
-  ...webSupportedImage,
-  '.bmp': ['image/bmp'],
   '.heic': ['image/heic'],
   '.heif': ['image/heif'],
   '.hif': ['image/hif'],
@@ -64,6 +63,11 @@ const image: Record<string, string[]> = {
   '.tiff': ['image/tiff'],
 };
 
+const image: Record<string, string[]> = {
+  ...webSupportedImage,
+  ...webUnsupportedImage,
+};
+
 const possiblyAnimatedImageExtensions = new Set(['.avif', '.gif', '.heic', '.heif', '.jxl', '.png', '.webp']);
 const possiblyAnimatedImage: Record<string, string[]> = Object.fromEntries(
   Object.entries(image).filter(([key]) => possiblyAnimatedImageExtensions.has(key)),
@@ -120,6 +124,7 @@ export const mimeTypes = {
   sidecar,
   video,
   raw,
+  webUnsupportedImage,
 
   isAsset: (filename: string) => isType(filename, image) || isType(filename, video),
   isImage: (filename: string) => isType(filename, image),

web/src/lib/utils/asset-utils.ts

@@ -295,6 +295,7 @@ export function getAssetRatio(asset: AssetResponseDto) {
 const supportedImageMimeTypes = new Set([
   'image/apng',
   'image/avif',
+  'image/bmp',
   'image/gif',
   'image/jpeg',
   'image/png',