From 2069293cc1ccc175cd48901a57be869161029ccb Mon Sep 17 00:00:00 2001 From: Mert <101130780+mertalev@users.noreply.github.com> Date: Sun, 3 Sep 2023 02:21:51 -0400 Subject: [PATCH] feat(server): wide gamut thumbnails (#3658) --- cli/src/api/open-api/api.ts | 28 +++++++ mobile/openapi/.openapi-generator/FILES | 3 + mobile/openapi/README.md | Bin 19176 -> 19212 bytes mobile/openapi/doc/Colorspace.md | Bin 0 -> 376 bytes .../openapi/doc/SystemConfigThumbnailDto.md | Bin 448 -> 531 bytes mobile/openapi/lib/api.dart | Bin 6394 -> 6424 bytes mobile/openapi/lib/api_client.dart | Bin 19696 -> 19783 bytes mobile/openapi/lib/api_helper.dart | Bin 4944 -> 5042 bytes mobile/openapi/lib/model/colorspace.dart | Bin 0 -> 2524 bytes .../model/system_config_thumbnail_dto.dart | Bin 3194 -> 3697 bytes mobile/openapi/test/colorspace_test.dart | Bin 0 -> 417 bytes .../system_config_thumbnail_dto_test.dart | Bin 691 -> 896 bytes server/immich-openapi-specs.json | 17 +++- .../facial-recognition.service.spec.ts | 8 +- .../facial-recognition.services.ts | 9 +- server/src/domain/media/media.repository.ts | 5 +- server/src/domain/media/media.service.spec.ts | 26 +++--- server/src/domain/media/media.service.ts | 78 +++++++++++------- server/src/domain/media/media.util.ts | 11 ++- .../dto/system-config-thumbnail.dto.ts | 16 +++- .../system-config/system-config.core.ts | 3 + .../system-config.service.spec.ts | 3 + .../infra/entities/system-config.entity.ts | 9 ++ .../infra/repositories/media.repository.ts | 29 ++++++- web/src/api/open-api/api.ts | 28 +++++++ .../admin-page/settings/setting-switch.svelte | 12 ++- .../thumbnail/thumbnail-settings.svelte | 24 +++++- 27 files changed, 249 insertions(+), 60 deletions(-) create mode 100644 mobile/openapi/doc/Colorspace.md create mode 100644 mobile/openapi/lib/model/colorspace.dart create mode 100644 mobile/openapi/test/colorspace_test.dart diff --git a/cli/src/api/open-api/api.ts b/cli/src/api/open-api/api.ts index b185f4d0c..56167aaf3 100644 --- a/cli/src/api/open-api/api.ts +++ b/cli/src/api/open-api/api.ts @@ -1046,6 +1046,20 @@ export interface ClassificationConfig { } +/** + * + * @export + * @enum {string} + */ + +export const Colorspace = { + Srgb: 'srgb', + P3: 'p3' +} as const; + +export type Colorspace = typeof Colorspace[keyof typeof Colorspace]; + + /** * * @export @@ -3184,12 +3198,24 @@ export interface SystemConfigTemplateStorageOptionDto { * @interface SystemConfigThumbnailDto */ export interface SystemConfigThumbnailDto { + /** + * + * @type {Colorspace} + * @memberof SystemConfigThumbnailDto + */ + 'colorspace': Colorspace; /** * * @type {number} * @memberof SystemConfigThumbnailDto */ 'jpegSize': number; + /** + * + * @type {number} + * @memberof SystemConfigThumbnailDto + */ + 'quality': number; /** * * @type {number} @@ -3197,6 +3223,8 @@ export interface SystemConfigThumbnailDto { */ 'webpSize': number; } + + /** * * @export diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 7f2f8e0ff..5cb7fa651 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -44,6 +44,7 @@ doc/CheckDuplicateAssetResponseDto.md doc/CheckExistingAssetsDto.md doc/CheckExistingAssetsResponseDto.md doc/ClassificationConfig.md +doc/Colorspace.md doc/CreateAlbumDto.md doc/CreateProfileImageResponseDto.md doc/CreateTagDto.md @@ -199,6 +200,7 @@ lib/model/check_existing_assets_response_dto.dart lib/model/classification_config.dart lib/model/clip_config.dart lib/model/clip_mode.dart +lib/model/colorspace.dart lib/model/cq_mode.dart lib/model/create_album_dto.dart lib/model/create_profile_image_response_dto.dart @@ -326,6 +328,7 @@ test/check_existing_assets_response_dto_test.dart test/classification_config_test.dart test/clip_config_test.dart test/clip_mode_test.dart +test/colorspace_test.dart test/cq_mode_test.dart test/create_album_dto_test.dart test/create_profile_image_response_dto_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index f5ffe531982bec6bf2444703bbc3d92e16e100ad..fc0e4fd335bd070f7f23fe6ac3bb3d584f3d73d1 100644 GIT binary patch delta 41 pcmaDcm9b|U!4BY({mfipi2|JxAFjOiCRoxKMi(U+k`VvQp87lGdBoCz0E|4X<^ZCx# zAV&%&I_=p~uNT~N%F%QL&@}kY#-e-?4pS`rw>c8DZEH;ua6WPp9NqZY&1+iys$ld| zrGwKpm0=P`88*ffp7HZ1)^ACAzP|${@DAe66*jO*G7dzWRu zA8S4RSk0H~mG-u=+?_lHF*R2mIH{pwp5X8HeEas~Zt1Hi*vKx(AB$_?zwikFTmaAn BbMOEF literal 0 HcmV?d00001 diff --git a/mobile/openapi/doc/SystemConfigThumbnailDto.md b/mobile/openapi/doc/SystemConfigThumbnailDto.md index 892b863b3ab4b239b90c5f2e2ed065b1fca83351..491bf9f120066ea34e142eec685d5eb5b6853976 100644 GIT binary patch delta 73 zcmX@WJeg&}W0U0kocyBVg2d!hEiHu_g=j4;XC#qW4FpdwH$@Yu0*EF)P+%`CP0YzG Jshs%wDgf4F8*Kmp delta 14 WcmbQta)5clplP5pa66JJ;^B(9MOrEdKwt2Q%s2Tvb<`PB# delta 14 WcmX>;i}Axu#tqZdHosMiQUd@s9tNKP diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index c64bacac8e6fea78671f744e3b8f937f6e61918f..b3205a0c9f6528b6f679e82e3b905d6d521caccb 100644 GIT binary patch delta 52 ycmcbhwn=>hCx^0geolT-aY15os-{9UmjVzJrIwTyPWYZm>98iY_Tu=fhwa0xeNCSDDmEDuz+yfA2d| zk`*V*0>qN6$NTZ_?w!Zu!FWuU_w&0Sf1lmX{<@scuIT#a(`-ancXWFHddyEv1y@7c|oC)2ZHDG=tbo$Fr zX|@s84F+)M$rNNNsYRs{{J$Fv23aYbqu)$vtgA#O>Dz#4jNEUcVUv8% zlX8J@QI=F0=hUi{F)T+)$7C1G>2h(PKhXHZPhiKv z*RKcE`vrU;9z-?@chxyxm%(>)`hcENVa+yVpYPFo$Tt>*c{;%6`%2EO&<^dsm3DZR zlUgee;I+IZ^J`_}RAd`_>xl8B4?1Ht@+*GxYGj^%&dkNx3_oa4=My zok#Z(eKH|kmu24=Q4Bp1dEQYK&k;R_@Y`#AveMU9x2L?}goZ6W7adW-|Ks41g=kD& zVbHGNAxeQrj1~@Pbw%`^xD4 z>ExME9JNa*W+5_f?EbZ;;+{*p7^l<8GtoFQ3=N4hbu2Fbg=^+LQZhXBH*s=78~7$_ zQa7wuq17FfZqafPD!yDWc*%5#URtb7C`M98_;yQLh#VW)=dZq#nTQS?jw|#BMgp<^4kc*~#y)Y6M@ThPs7!4XrjqKg`_;_^J(TSw^WK{wv9EPe+FB|PZ> literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/system_config_thumbnail_dto.dart b/mobile/openapi/lib/model/system_config_thumbnail_dto.dart index 54360074e381e57866dc66a330233ce7338cafb2..46d5fe7c1b02e2bb4f3822e985be3f4295e1d19e 100644 GIT binary patch delta 442 zcmew*@lj^ODn`!a{G9xv;)2BF$r~9(C$D0ZVJ|FA%*iaNoV%o2md;m}XVnK{JJqsGBS(x5Mq;WFk47Kl_TV#0-G6GHZYnY6k91c!^2B2ttda&t2jSTBMV4G7pWtZ osmE$6>P%k2>H_!3PF6V>d-HwP3N{`j#X6G%dBk9{ljC`$0S^eC@Bjb+ delta 125 zcmew;^GjmGD#pnt7-c5^XB6hi%qvmIDo9Na&a6tE?7`GG`7e|5WJl)I$%mMwCVyoP z5`szEs=!#2n_2QE%du*1&SPE3IN6Wgdh;RnY&K<>w7L$L0uZQ|rzRDEOa=36t+{Hs FxB!iAD$xJ{ diff --git a/mobile/openapi/test/colorspace_test.dart b/mobile/openapi/test/colorspace_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..f689d519edb0d580920dce323d0550ba591807e1 GIT binary patch literal 417 zcmZvY(M!WH5XRs0SDa7Xpf0*6*$^DIlYy=v-GfgdwAU_}Ch?LgBKzM>l!0I$?v4+B z-+h;G&Nzq7p{_0;%6<8~smlVkyT>wzqJn)@!&6o4wl5bEOXOWEDcmj>*Grb9>Ww87 zoh8vJjVl<<=&=>3aX=IDvsQz%6eb-~f5=ZS+!zGm+o&kojT~ZklHv?VJ}xIeZEvjc zL7E^`8v|9RkbdlbGE6IjUTACjh=rpx;^s;;M@gQs-y0!wvU!y~uB<~No#L?;>DAf# z9}>_9p4u5mp-Fp)Ujp}5Tk>%qvUjUm7^E_@Ieak&(32hAEKBeoF3aHzoaSrBPV5^} C5se!F literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/system_config_thumbnail_dto_test.dart b/mobile/openapi/test/system_config_thumbnail_dto_test.dart index 3dd82cff7ca5b20ef0d3ad93b0aa8974e8accc38..3cc66f46778435aea63fee7ee443f88428309cdc 100644 GIT binary patch delta 74 zcmdnY+Q7cy4Wo#2eolT-aY15oszNf1IXRL^i4(z7U^1Hgfia1_urx6zv!rrz6q6#H It;jS900xE|1^@s6 delta 15 WcmZo*-^{w<4ddiUrnJfO%!vRlHU&oj diff --git a/server/immich-openapi-specs.json b/server/immich-openapi-specs.json index a61a16a76..c38db8cbf 100644 --- a/server/immich-openapi-specs.json +++ b/server/immich-openapi-specs.json @@ -5523,6 +5523,13 @@ ], "type": "object" }, + "Colorspace": { + "enum": [ + "srgb", + "p3" + ], + "type": "string" + }, "CreateAlbumDto": { "properties": { "albumName": { @@ -7284,16 +7291,24 @@ }, "SystemConfigThumbnailDto": { "properties": { + "colorspace": { + "$ref": "#/components/schemas/Colorspace" + }, "jpegSize": { "type": "integer" }, + "quality": { + "type": "integer" + }, "webpSize": { "type": "integer" } }, "required": [ "webpSize", - "jpegSize" + "jpegSize", + "quality", + "colorspace" ], "type": "object" }, diff --git a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts index 0b117d6a2..1ca3ed89b 100644 --- a/server/src/domain/facial-recognition/facial-recognition.service.spec.ts +++ b/server/src/domain/facial-recognition/facial-recognition.service.spec.ts @@ -1,3 +1,4 @@ +import { Colorspace } from '@app/infra/entities'; import { assetStub, faceStub, @@ -115,7 +116,6 @@ describe(FacialRecognitionService.name, () => { personMock = newPersonRepositoryMock(); searchMock = newSearchRepositoryMock(); storageMock = newStorageRepositoryMock(); - configMock = newSystemConfigRepositoryMock(); mediaMock.crop.mockResolvedValue(croppedFace); @@ -292,6 +292,8 @@ describe(FacialRecognitionService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { format: 'jpeg', size: 250, + quality: 80, + colorspace: Colorspace.P3, }); expect(personMock.update).toHaveBeenCalledWith({ id: 'person-1', @@ -313,6 +315,8 @@ describe(FacialRecognitionService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { format: 'jpeg', size: 250, + quality: 80, + colorspace: Colorspace.P3, }); }); @@ -330,6 +334,8 @@ describe(FacialRecognitionService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { format: 'jpeg', size: 250, + quality: 80, + colorspace: Colorspace.P3, }); }); }); diff --git a/server/src/domain/facial-recognition/facial-recognition.services.ts b/server/src/domain/facial-recognition/facial-recognition.services.ts index ad90dca24..471d21ba4 100644 --- a/server/src/domain/facial-recognition/facial-recognition.services.ts +++ b/server/src/domain/facial-recognition/facial-recognition.services.ts @@ -162,8 +162,15 @@ export class FacialRecognitionService { height: newHalfSize * 2, }; + const { thumbnail } = await this.configCore.getConfig(); const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions); - await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' }); + const thumbnailOptions = { + format: 'jpeg', + size: FACE_THUMBNAIL_SIZE, + colorspace: thumbnail.colorspace, + quality: thumbnail.quality, + } as const; + await this.mediaRepository.resize(croppedOutput, output, thumbnailOptions); await this.personRepository.update({ id: personId, thumbnailPath: output }); return true; diff --git a/server/src/domain/media/media.repository.ts b/server/src/domain/media/media.repository.ts index 25486cc4e..cf2b8fa81 100644 --- a/server/src/domain/media/media.repository.ts +++ b/server/src/domain/media/media.repository.ts @@ -1,10 +1,13 @@ import { VideoCodec } from '@app/infra/entities'; +import { Writable } from 'stream'; export const IMediaRepository = 'IMediaRepository'; export interface ResizeOptions { size: number; format: 'webp' | 'jpeg'; + colorspace: string; + quality: number; } export interface VideoStreamInfo { @@ -73,5 +76,5 @@ export interface IMediaRepository { // video probe(input: string): Promise; - transcode(input: string, output: string, options: TranscodeOptions): Promise; + transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise; } diff --git a/server/src/domain/media/media.service.spec.ts b/server/src/domain/media/media.service.spec.ts index 355ba7873..dc9359aa4 100644 --- a/server/src/domain/media/media.service.spec.ts +++ b/server/src/domain/media/media.service.spec.ts @@ -1,5 +1,6 @@ import { AssetType, + Colorspace, SystemConfigKey, ToneMapping, TranscodeHWAccel, @@ -134,6 +135,8 @@ describe(MediaService.name, () => { expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', { size: 1440, format: 'jpeg', + quality: 80, + colorspace: Colorspace.P3, }); expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', @@ -148,12 +151,11 @@ describe(MediaService.name, () => { expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { - inputOptions: [], + inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], outputOptions: [ - '-ss 00:00:00.000', '-frames:v 1', '-v verbose', - '-vf scale=-2:1440:out_color_matrix=bt601:out_range=pc,format=yuv420p', + '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p', ], twoPass: false, }); @@ -170,12 +172,11 @@ describe(MediaService.name, () => { expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { - inputOptions: [], + inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'], outputOptions: [ - '-ss 00:00:00.000', '-frames:v 1', '-v verbose', - '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt470bg:t=601:m=bt470bg:range=pc,format=yuv420p', + '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p', ], twoPass: false, }); @@ -209,12 +210,13 @@ describe(MediaService.name, () => { assetMock.getByIds.mockResolvedValue([assetStub.image]); await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id }); - expect(mediaMock.resize).toHaveBeenCalledWith( - '/uploads/user-id/thumbs/path.jpg', - '/uploads/user-id/thumbs/path.webp', - { format: 'webp', size: 250 }, - ); - expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: '/uploads/user-id/thumbs/path.webp' }); + expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', { + format: 'webp', + size: 250, + quality: 80, + colorspace: Colorspace.P3, + }); + expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' }); }); }); diff --git a/server/src/domain/media/media.service.ts b/server/src/domain/media/media.service.ts index 732aadde7..a09b77cbc 100644 --- a/server/src/domain/media/media.service.ts +++ b/server/src/domain/media/media.service.ts @@ -9,7 +9,6 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config import { SystemConfigCore } from '../system-config/system-config.core'; import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository'; import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util'; - @Injectable() export class MediaService { private logger = new Logger(MediaService.name); @@ -21,9 +20,9 @@ export class MediaService { @Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository, - @Inject(ISystemConfigRepository) systemConfig: ISystemConfigRepository, + @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository, ) { - this.configCore = new SystemConfigCore(systemConfig); + this.configCore = new SystemConfigCore(configRepository); } async handleQueueGenerateThumbnails(job: IBaseJob) { @@ -59,38 +58,53 @@ export class MediaService { return false; } - const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); - this.storageRepository.mkdirSync(resizePath); - const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`); - const { thumbnail } = await this.configCore.getConfig(); + const resizePath = await this.generateThumbnail(asset, 'jpeg'); + await this.assetRepository.save({ id: asset.id, resizePath }); + return true; + } + async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + let path; switch (asset.type) { case AssetType.IMAGE: - await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, { - size: thumbnail.jpegSize, - format: 'jpeg', - }); - this.logger.log(`Successfully generated image thumbnail ${asset.id}`); + path = await this.generateImageThumbnail(asset, format); break; case AssetType.VIDEO: - const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); - const mainVideoStream = this.getMainStream(videoStreams); - if (!mainVideoStream) { - this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`); - return false; - } - const mainAudioStream = this.getMainStream(audioStreams); - const { ffmpeg } = await this.configCore.getConfig(); - const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false }; - const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); - await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options); - this.logger.log(`Successfully generated video thumbnail ${asset.id}`); + path = await this.generateVideoThumbnail(asset, format); break; + default: + throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`); } + this.logger.log( + `Successfully generated ${format.toUpperCase()} ${asset.type.toLowerCase()} thumbnail for asset ${asset.id}`, + ); + return path; + } - await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath }); + async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + const { thumbnail } = await this.configCore.getConfig(); + const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; + const thumbnailOptions = { format, size, colorspace: thumbnail.colorspace, quality: thumbnail.quality }; + const path = this.ensureThumbnailPath(asset, format); + await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions); + return path; + } - return true; + async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') { + const { ffmpeg, thumbnail } = await this.configCore.getConfig(); + const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize; + const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); + const mainVideoStream = this.getMainStream(videoStreams); + if (!mainVideoStream) { + this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`); + return; + } + const mainAudioStream = this.getMainStream(audioStreams); + const path = this.ensureThumbnailPath(asset, format); + const config = { ...ffmpeg, targetResolution: size.toString() }; + const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); + await this.mediaRepository.transcode(asset.originalPath, path, options); + return path; } async handleGenerateWebpThumbnail({ id }: IEntityJob) { @@ -99,12 +113,8 @@ export class MediaService { return false; } - const webpPath = asset.resizePath.replace('jpeg', 'webp').replace('jpg', 'webp'); - - const { thumbnail } = await this.configCore.getConfig(); - await this.mediaRepository.resize(asset.resizePath, webpPath, { size: thumbnail.webpSize, format: 'webp' }); + const webpPath = await this.generateThumbnail(asset, 'webp'); await this.assetRepository.save({ id: asset.id, webpPath }); - return true; } @@ -289,4 +299,10 @@ export class MediaService { return handler; } + + ensureThumbnailPath(asset: AssetEntity, extension: string): string { + const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); + this.storageRepository.mkdirSync(folderPath); + return join(folderPath, `${asset.id}.${extension}`); + } } diff --git a/server/src/domain/media/media.util.ts b/server/src/domain/media/media.util.ts index b276f20d0..75d202094 100644 --- a/server/src/domain/media/media.util.ts +++ b/server/src/domain/media/media.util.ts @@ -263,8 +263,11 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig { } export class ThumbnailConfig extends BaseConfig { + getBaseInputOptions(): string[] { + return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int']; + } getBaseOutputOptions() { - return ['-ss 00:00:00.000', '-frames:v 1']; + return ['-frames:v 1']; } getPresetOptions() { @@ -277,16 +280,16 @@ export class ThumbnailConfig extends BaseConfig { getScaling(videoStream: VideoStreamInfo) { let options = super.getScaling(videoStream); + options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int'; if (!this.shouldToneMap(videoStream)) { - options += ':out_color_matrix=bt601:out_range=pc'; + options += ':out_color_matrix=601:out_range=pc'; } return options; } getColors() { return { - // jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts - primaries: 'bt470bg', + primaries: 'bt709', transfer: '601', matrix: 'bt470bg', }; diff --git a/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts b/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts index 53d9d64a5..c389ef77a 100644 --- a/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts +++ b/server/src/domain/system-config/dto/system-config-thumbnail.dto.ts @@ -1,15 +1,29 @@ +import { Colorspace } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; import { Type } from 'class-transformer'; -import { IsInt } from 'class-validator'; +import { IsEnum, IsInt, Max, Min } from 'class-validator'; export class SystemConfigThumbnailDto { @IsInt() + @Min(1) @Type(() => Number) @ApiProperty({ type: 'integer' }) webpSize!: number; @IsInt() + @Min(1) @Type(() => Number) @ApiProperty({ type: 'integer' }) jpegSize!: number; + + @IsInt() + @Min(1) + @Max(100) + @Type(() => Number) + @ApiProperty({ type: 'integer' }) + quality!: number; + + @IsEnum(Colorspace) + @ApiProperty({ enumName: 'Colorspace', enum: Colorspace }) + colorspace!: Colorspace; } diff --git a/server/src/domain/system-config/system-config.core.ts b/server/src/domain/system-config/system-config.core.ts index 1236c15e8..cfe64b0a7 100644 --- a/server/src/domain/system-config/system-config.core.ts +++ b/server/src/domain/system-config/system-config.core.ts @@ -1,6 +1,7 @@ import { AudioCodec, CQMode, + Colorspace, SystemConfig, SystemConfigEntity, SystemConfigKey, @@ -98,6 +99,8 @@ export const defaults = Object.freeze({ thumbnail: { webpSize: 250, jpegSize: 1440, + quality: 80, + colorspace: Colorspace.P3, }, }); diff --git a/server/src/domain/system-config/system-config.service.spec.ts b/server/src/domain/system-config/system-config.service.spec.ts index 28016c20d..6b9ef2d9d 100644 --- a/server/src/domain/system-config/system-config.service.spec.ts +++ b/server/src/domain/system-config/system-config.service.spec.ts @@ -1,6 +1,7 @@ import { AudioCodec, CQMode, + Colorspace, SystemConfig, SystemConfigEntity, SystemConfigKey, @@ -94,6 +95,8 @@ const updatedConfig = Object.freeze({ thumbnail: { webpSize: 250, jpegSize: 1440, + quality: 80, + colorspace: Colorspace.P3, }, }); diff --git a/server/src/infra/entities/system-config.entity.ts b/server/src/infra/entities/system-config.entity.ts index 941058959..2da905d10 100644 --- a/server/src/infra/entities/system-config.entity.ts +++ b/server/src/infra/entities/system-config.entity.ts @@ -76,6 +76,8 @@ export enum SystemConfigKey { THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize', THUMBNAIL_JPEG_SIZE = 'thumbnail.jpegSize', + THUMBNAIL_QUALITY = 'thumbnail.quality', + THUMBNAIL_COLORSPACE = 'thumbnail.colorspace', } export enum TranscodePolicy { @@ -117,6 +119,11 @@ export enum CQMode { ICQ = 'icq', } +export enum Colorspace { + SRGB = 'srgb', + P3 = 'p3', +} + export interface SystemConfig { ffmpeg: { crf: number; @@ -179,5 +186,7 @@ export interface SystemConfig { thumbnail: { webpSize: number; jpegSize: number; + quality: number; + colorspace: Colorspace; }; } diff --git a/server/src/infra/repositories/media.repository.ts b/server/src/infra/repositories/media.repository.ts index 4012cbf13..2d711a530 100644 --- a/server/src/infra/repositories/media.repository.ts +++ b/server/src/infra/repositories/media.repository.ts @@ -1,8 +1,10 @@ import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain'; +import { Colorspace } from '@app/infra/entities'; import { Logger } from '@nestjs/common'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import fs from 'fs/promises'; import sharp from 'sharp'; +import { Writable } from 'stream'; import { promisify } from 'util'; const probe = promisify(ffmpeg.ffprobe); @@ -11,7 +13,7 @@ sharp.concurrency(0); export class MediaRepository implements IMediaRepository { private logger = new Logger(MediaRepository.name); - crop(input: string, options: CropOptions): Promise { + crop(input: string | Buffer, options: CropOptions): Promise { return sharp(input, { failOn: 'none' }) .extract({ left: options.left, @@ -23,10 +25,25 @@ export class MediaRepository implements IMediaRepository { } async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise { - await sharp(input, { failOn: 'none' }) + let colorProfile = options.colorspace; + if (options.colorspace !== Colorspace.SRGB) { + try { + const { space } = await sharp(input).metadata(); + // if the image is already in srgb, keep it that way + if (space === 'srgb') { + colorProfile = Colorspace.SRGB; + } + } catch (err) { + this.logger.warn(`Could not determine colorspace of image, defaulting to ${colorProfile} profile`); + } + } + const chromaSubsampling = options.quality >= 80 ? '4:4:4' : '4:2:0'; // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp + sharp(input, { failOn: 'none' }) + .pipelineColorspace('rgb16') .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }) .rotate() - .toFormat(options.format) + .withMetadata({ icc: colorProfile }) + .toFormat(options.format, { quality: options.quality, chromaSubsampling }) .toFile(output); } @@ -61,7 +78,7 @@ export class MediaRepository implements IMediaRepository { }; } - transcode(input: string, output: string, options: TranscodeOptions): Promise { + transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise { if (!options.twoPass) { return new Promise((resolve, reject) => { ffmpeg(input, { niceness: 10 }) @@ -77,6 +94,10 @@ export class MediaRepository implements IMediaRepository { }); } + if (typeof output !== 'string') { + throw new Error('Two-pass transcoding does not support writing to a stream'); + } + // two-pass allows for precise control of bitrate at the cost of running twice // recommended for vp9 for better quality and compression return new Promise((resolve, reject) => { diff --git a/web/src/api/open-api/api.ts b/web/src/api/open-api/api.ts index b185f4d0c..56167aaf3 100644 --- a/web/src/api/open-api/api.ts +++ b/web/src/api/open-api/api.ts @@ -1046,6 +1046,20 @@ export interface ClassificationConfig { } +/** + * + * @export + * @enum {string} + */ + +export const Colorspace = { + Srgb: 'srgb', + P3: 'p3' +} as const; + +export type Colorspace = typeof Colorspace[keyof typeof Colorspace]; + + /** * * @export @@ -3184,12 +3198,24 @@ export interface SystemConfigTemplateStorageOptionDto { * @interface SystemConfigThumbnailDto */ export interface SystemConfigThumbnailDto { + /** + * + * @type {Colorspace} + * @memberof SystemConfigThumbnailDto + */ + 'colorspace': Colorspace; /** * * @type {number} * @memberof SystemConfigThumbnailDto */ 'jpegSize': number; + /** + * + * @type {number} + * @memberof SystemConfigThumbnailDto + */ + 'quality': number; /** * * @type {number} @@ -3197,6 +3223,8 @@ export interface SystemConfigThumbnailDto { */ 'webpSize': number; } + + /** * * @export diff --git a/web/src/lib/components/admin-page/settings/setting-switch.svelte b/web/src/lib/components/admin-page/settings/setting-switch.svelte index 6cc4166e9..8cd1f8483 100644 --- a/web/src/lib/components/admin-page/settings/setting-switch.svelte +++ b/web/src/lib/components/admin-page/settings/setting-switch.svelte @@ -1,12 +1,16 @@
@@ -29,7 +33,13 @@