feat(server): wide gamut thumbnails (#3658)

This commit is contained in:
Mert 2023-09-03 02:21:51 -04:00 committed by GitHub
parent 4bd77d5899
commit 2069293cc1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 249 additions and 60 deletions

View file

@ -1046,6 +1046,20 @@ export interface ClassificationConfig {
} }
/**
*
* @export
* @enum {string}
*/
export const Colorspace = {
Srgb: 'srgb',
P3: 'p3'
} as const;
export type Colorspace = typeof Colorspace[keyof typeof Colorspace];
/** /**
* *
* @export * @export
@ -3184,12 +3198,24 @@ export interface SystemConfigTemplateStorageOptionDto {
* @interface SystemConfigThumbnailDto * @interface SystemConfigThumbnailDto
*/ */
export interface SystemConfigThumbnailDto { export interface SystemConfigThumbnailDto {
/**
*
* @type {Colorspace}
* @memberof SystemConfigThumbnailDto
*/
'colorspace': Colorspace;
/** /**
* *
* @type {number} * @type {number}
* @memberof SystemConfigThumbnailDto * @memberof SystemConfigThumbnailDto
*/ */
'jpegSize': number; 'jpegSize': number;
/**
*
* @type {number}
* @memberof SystemConfigThumbnailDto
*/
'quality': number;
/** /**
* *
* @type {number} * @type {number}
@ -3197,6 +3223,8 @@ export interface SystemConfigThumbnailDto {
*/ */
'webpSize': number; 'webpSize': number;
} }
/** /**
* *
* @export * @export

View file

@ -44,6 +44,7 @@ doc/CheckDuplicateAssetResponseDto.md
doc/CheckExistingAssetsDto.md doc/CheckExistingAssetsDto.md
doc/CheckExistingAssetsResponseDto.md doc/CheckExistingAssetsResponseDto.md
doc/ClassificationConfig.md doc/ClassificationConfig.md
doc/Colorspace.md
doc/CreateAlbumDto.md doc/CreateAlbumDto.md
doc/CreateProfileImageResponseDto.md doc/CreateProfileImageResponseDto.md
doc/CreateTagDto.md doc/CreateTagDto.md
@ -199,6 +200,7 @@ lib/model/check_existing_assets_response_dto.dart
lib/model/classification_config.dart lib/model/classification_config.dart
lib/model/clip_config.dart lib/model/clip_config.dart
lib/model/clip_mode.dart lib/model/clip_mode.dart
lib/model/colorspace.dart
lib/model/cq_mode.dart lib/model/cq_mode.dart
lib/model/create_album_dto.dart lib/model/create_album_dto.dart
lib/model/create_profile_image_response_dto.dart lib/model/create_profile_image_response_dto.dart
@ -326,6 +328,7 @@ test/check_existing_assets_response_dto_test.dart
test/classification_config_test.dart test/classification_config_test.dart
test/clip_config_test.dart test/clip_config_test.dart
test/clip_mode_test.dart test/clip_mode_test.dart
test/colorspace_test.dart
test/cq_mode_test.dart test/cq_mode_test.dart
test/create_album_dto_test.dart test/create_album_dto_test.dart
test/create_profile_image_response_dto_test.dart test/create_profile_image_response_dto_test.dart

BIN
mobile/openapi/README.md generated

Binary file not shown.

BIN
mobile/openapi/doc/Colorspace.md generated Normal file

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

Binary file not shown.

BIN
mobile/openapi/lib/model/colorspace.dart generated Normal file

Binary file not shown.

BIN
mobile/openapi/test/colorspace_test.dart generated Normal file

Binary file not shown.

View file

@ -5523,6 +5523,13 @@
], ],
"type": "object" "type": "object"
}, },
"Colorspace": {
"enum": [
"srgb",
"p3"
],
"type": "string"
},
"CreateAlbumDto": { "CreateAlbumDto": {
"properties": { "properties": {
"albumName": { "albumName": {
@ -7284,16 +7291,24 @@
}, },
"SystemConfigThumbnailDto": { "SystemConfigThumbnailDto": {
"properties": { "properties": {
"colorspace": {
"$ref": "#/components/schemas/Colorspace"
},
"jpegSize": { "jpegSize": {
"type": "integer" "type": "integer"
}, },
"quality": {
"type": "integer"
},
"webpSize": { "webpSize": {
"type": "integer" "type": "integer"
} }
}, },
"required": [ "required": [
"webpSize", "webpSize",
"jpegSize" "jpegSize",
"quality",
"colorspace"
], ],
"type": "object" "type": "object"
}, },

View file

@ -1,3 +1,4 @@
import { Colorspace } from '@app/infra/entities';
import { import {
assetStub, assetStub,
faceStub, faceStub,
@ -115,7 +116,6 @@ describe(FacialRecognitionService.name, () => {
personMock = newPersonRepositoryMock(); personMock = newPersonRepositoryMock();
searchMock = newSearchRepositoryMock(); searchMock = newSearchRepositoryMock();
storageMock = newStorageRepositoryMock(); storageMock = newStorageRepositoryMock();
configMock = newSystemConfigRepositoryMock();
mediaMock.crop.mockResolvedValue(croppedFace); mediaMock.crop.mockResolvedValue(croppedFace);
@ -292,6 +292,8 @@ describe(FacialRecognitionService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg', format: 'jpeg',
size: 250, size: 250,
quality: 80,
colorspace: Colorspace.P3,
}); });
expect(personMock.update).toHaveBeenCalledWith({ expect(personMock.update).toHaveBeenCalledWith({
id: 'person-1', id: 'person-1',
@ -313,6 +315,8 @@ describe(FacialRecognitionService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg', format: 'jpeg',
size: 250, size: 250,
quality: 80,
colorspace: Colorspace.P3,
}); });
}); });
@ -330,6 +334,8 @@ describe(FacialRecognitionService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', { expect(mediaMock.resize).toHaveBeenCalledWith(croppedFace, 'upload/thumbs/user-id/person-1.jpeg', {
format: 'jpeg', format: 'jpeg',
size: 250, size: 250,
quality: 80,
colorspace: Colorspace.P3,
}); });
}); });
}); });

View file

@ -162,8 +162,15 @@ export class FacialRecognitionService {
height: newHalfSize * 2, height: newHalfSize * 2,
}; };
const { thumbnail } = await this.configCore.getConfig();
const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions); const croppedOutput = await this.mediaRepository.crop(asset.resizePath, cropOptions);
await this.mediaRepository.resize(croppedOutput, output, { size: FACE_THUMBNAIL_SIZE, format: 'jpeg' }); const thumbnailOptions = {
format: 'jpeg',
size: FACE_THUMBNAIL_SIZE,
colorspace: thumbnail.colorspace,
quality: thumbnail.quality,
} as const;
await this.mediaRepository.resize(croppedOutput, output, thumbnailOptions);
await this.personRepository.update({ id: personId, thumbnailPath: output }); await this.personRepository.update({ id: personId, thumbnailPath: output });
return true; return true;

View file

@ -1,10 +1,13 @@
import { VideoCodec } from '@app/infra/entities'; import { VideoCodec } from '@app/infra/entities';
import { Writable } from 'stream';
export const IMediaRepository = 'IMediaRepository'; export const IMediaRepository = 'IMediaRepository';
export interface ResizeOptions { export interface ResizeOptions {
size: number; size: number;
format: 'webp' | 'jpeg'; format: 'webp' | 'jpeg';
colorspace: string;
quality: number;
} }
export interface VideoStreamInfo { export interface VideoStreamInfo {
@ -73,5 +76,5 @@ export interface IMediaRepository {
// video // video
probe(input: string): Promise<VideoInfo>; probe(input: string): Promise<VideoInfo>;
transcode(input: string, output: string, options: TranscodeOptions): Promise<void>; transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void>;
} }

View file

@ -1,5 +1,6 @@
import { import {
AssetType, AssetType,
Colorspace,
SystemConfigKey, SystemConfigKey,
ToneMapping, ToneMapping,
TranscodeHWAccel, TranscodeHWAccel,
@ -134,6 +135,8 @@ describe(MediaService.name, () => {
expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', { expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.jpeg', {
size: 1440, size: 1440,
format: 'jpeg', format: 'jpeg',
quality: 80,
colorspace: Colorspace.P3,
}); });
expect(assetMock.save).toHaveBeenCalledWith({ expect(assetMock.save).toHaveBeenCalledWith({
id: 'asset-id', id: 'asset-id',
@ -148,12 +151,11 @@ describe(MediaService.name, () => {
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
inputOptions: [], inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [ outputOptions: [
'-ss 00:00:00.000',
'-frames:v 1', '-frames:v 1',
'-v verbose', '-v verbose',
'-vf scale=-2:1440:out_color_matrix=bt601:out_range=pc,format=yuv420p', '-vf scale=-2:1440:flags=lanczos+accurate_rnd+bitexact+full_chroma_int:out_color_matrix=601:out_range=pc,format=yuv420p',
], ],
twoPass: false, twoPass: false,
}); });
@ -170,12 +172,11 @@ describe(MediaService.name, () => {
expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id'); expect(storageMock.mkdirSync).toHaveBeenCalledWith('upload/thumbs/user-id');
expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', { expect(mediaMock.transcode).toHaveBeenCalledWith('/original/path.ext', 'upload/thumbs/user-id/asset-id.jpeg', {
inputOptions: [], inputOptions: ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'],
outputOptions: [ outputOptions: [
'-ss 00:00:00.000',
'-frames:v 1', '-frames:v 1',
'-v verbose', '-v verbose',
'-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt470bg:t=601:m=bt470bg:range=pc,format=yuv420p', '-vf zscale=t=linear:npl=100,tonemap=hable:desat=0,zscale=p=bt709:t=601:m=bt470bg:range=pc,format=yuv420p',
], ],
twoPass: false, twoPass: false,
}); });
@ -209,12 +210,13 @@ describe(MediaService.name, () => {
assetMock.getByIds.mockResolvedValue([assetStub.image]); assetMock.getByIds.mockResolvedValue([assetStub.image]);
await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id }); await sut.handleGenerateWebpThumbnail({ id: assetStub.image.id });
expect(mediaMock.resize).toHaveBeenCalledWith( expect(mediaMock.resize).toHaveBeenCalledWith('/original/path.jpg', 'upload/thumbs/user-id/asset-id.webp', {
'/uploads/user-id/thumbs/path.jpg', format: 'webp',
'/uploads/user-id/thumbs/path.webp', size: 250,
{ format: 'webp', size: 250 }, quality: 80,
); colorspace: Colorspace.P3,
expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: '/uploads/user-id/thumbs/path.webp' }); });
expect(assetMock.save).toHaveBeenCalledWith({ id: 'asset-id', webpPath: 'upload/thumbs/user-id/asset-id.webp' });
}); });
}); });

View file

@ -9,7 +9,6 @@ import { ISystemConfigRepository, SystemConfigFFmpegDto } from '../system-config
import { SystemConfigCore } from '../system-config/system-config.core'; import { SystemConfigCore } from '../system-config/system-config.core';
import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository'; import { AudioStreamInfo, IMediaRepository, VideoCodecHWConfig, VideoStreamInfo } from './media.repository';
import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util'; import { H264Config, HEVCConfig, NVENCConfig, QSVConfig, ThumbnailConfig, VAAPIConfig, VP9Config } from './media.util';
@Injectable() @Injectable()
export class MediaService { export class MediaService {
private logger = new Logger(MediaService.name); private logger = new Logger(MediaService.name);
@ -21,9 +20,9 @@ export class MediaService {
@Inject(IJobRepository) private jobRepository: IJobRepository, @Inject(IJobRepository) private jobRepository: IJobRepository,
@Inject(IMediaRepository) private mediaRepository: IMediaRepository, @Inject(IMediaRepository) private mediaRepository: IMediaRepository,
@Inject(IStorageRepository) private storageRepository: IStorageRepository, @Inject(IStorageRepository) private storageRepository: IStorageRepository,
@Inject(ISystemConfigRepository) systemConfig: ISystemConfigRepository, @Inject(ISystemConfigRepository) configRepository: ISystemConfigRepository,
) { ) {
this.configCore = new SystemConfigCore(systemConfig); this.configCore = new SystemConfigCore(configRepository);
} }
async handleQueueGenerateThumbnails(job: IBaseJob) { async handleQueueGenerateThumbnails(job: IBaseJob) {
@ -59,38 +58,53 @@ export class MediaService {
return false; return false;
} }
const resizePath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId); const resizePath = await this.generateThumbnail(asset, 'jpeg');
this.storageRepository.mkdirSync(resizePath); await this.assetRepository.save({ id: asset.id, resizePath });
const jpegThumbnailPath = join(resizePath, `${asset.id}.jpeg`); return true;
const { thumbnail } = await this.configCore.getConfig(); }
async generateThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
let path;
switch (asset.type) { switch (asset.type) {
case AssetType.IMAGE: case AssetType.IMAGE:
await this.mediaRepository.resize(asset.originalPath, jpegThumbnailPath, { path = await this.generateImageThumbnail(asset, format);
size: thumbnail.jpegSize,
format: 'jpeg',
});
this.logger.log(`Successfully generated image thumbnail ${asset.id}`);
break; break;
case AssetType.VIDEO: case AssetType.VIDEO:
path = await this.generateVideoThumbnail(asset, format);
break;
default:
throw new UnsupportedMediaTypeException(`Unsupported asset type for thumbnail generation: ${asset.type}`);
}
this.logger.log(
`Successfully generated ${format.toUpperCase()} ${asset.type.toLowerCase()} thumbnail for asset ${asset.id}`,
);
return path;
}
async generateImageThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { thumbnail } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const thumbnailOptions = { format, size, colorspace: thumbnail.colorspace, quality: thumbnail.quality };
const path = this.ensureThumbnailPath(asset, format);
await this.mediaRepository.resize(asset.originalPath, path, thumbnailOptions);
return path;
}
async generateVideoThumbnail(asset: AssetEntity, format: 'jpeg' | 'webp') {
const { ffmpeg, thumbnail } = await this.configCore.getConfig();
const size = format === 'jpeg' ? thumbnail.jpegSize : thumbnail.webpSize;
const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath); const { audioStreams, videoStreams } = await this.mediaRepository.probe(asset.originalPath);
const mainVideoStream = this.getMainStream(videoStreams); const mainVideoStream = this.getMainStream(videoStreams);
if (!mainVideoStream) { if (!mainVideoStream) {
this.logger.error(`Could not extract thumbnail for asset ${asset.id}: no video streams found`); this.logger.warn(`Skipped thumbnail generation for asset ${asset.id}: no video streams found`);
return false; return;
} }
const mainAudioStream = this.getMainStream(audioStreams); const mainAudioStream = this.getMainStream(audioStreams);
const { ffmpeg } = await this.configCore.getConfig(); const path = this.ensureThumbnailPath(asset, format);
const config = { ...ffmpeg, targetResolution: thumbnail.jpegSize.toString(), twoPass: false }; const config = { ...ffmpeg, targetResolution: size.toString() };
const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream); const options = new ThumbnailConfig(config).getOptions(mainVideoStream, mainAudioStream);
await this.mediaRepository.transcode(asset.originalPath, jpegThumbnailPath, options); await this.mediaRepository.transcode(asset.originalPath, path, options);
this.logger.log(`Successfully generated video thumbnail ${asset.id}`); return path;
break;
}
await this.assetRepository.save({ id: asset.id, resizePath: jpegThumbnailPath });
return true;
} }
async handleGenerateWebpThumbnail({ id }: IEntityJob) { async handleGenerateWebpThumbnail({ id }: IEntityJob) {
@ -99,12 +113,8 @@ export class MediaService {
return false; return false;
} }
const webpPath = asset.resizePath.replace('jpeg', 'webp').replace('jpg', 'webp'); const webpPath = await this.generateThumbnail(asset, 'webp');
const { thumbnail } = await this.configCore.getConfig();
await this.mediaRepository.resize(asset.resizePath, webpPath, { size: thumbnail.webpSize, format: 'webp' });
await this.assetRepository.save({ id: asset.id, webpPath }); await this.assetRepository.save({ id: asset.id, webpPath });
return true; return true;
} }
@ -289,4 +299,10 @@ export class MediaService {
return handler; return handler;
} }
ensureThumbnailPath(asset: AssetEntity, extension: string): string {
const folderPath = this.storageCore.getFolderLocation(StorageFolder.THUMBNAILS, asset.ownerId);
this.storageRepository.mkdirSync(folderPath);
return join(folderPath, `${asset.id}.${extension}`);
}
} }

View file

@ -263,8 +263,11 @@ export class BaseHWConfig extends BaseConfig implements VideoCodecHWConfig {
} }
export class ThumbnailConfig extends BaseConfig { export class ThumbnailConfig extends BaseConfig {
getBaseInputOptions(): string[] {
return ['-ss 00:00:00', '-sws_flags accurate_rnd+bitexact+full_chroma_int'];
}
getBaseOutputOptions() { getBaseOutputOptions() {
return ['-ss 00:00:00.000', '-frames:v 1']; return ['-frames:v 1'];
} }
getPresetOptions() { getPresetOptions() {
@ -277,16 +280,16 @@ export class ThumbnailConfig extends BaseConfig {
getScaling(videoStream: VideoStreamInfo) { getScaling(videoStream: VideoStreamInfo) {
let options = super.getScaling(videoStream); let options = super.getScaling(videoStream);
options += ':flags=lanczos+accurate_rnd+bitexact+full_chroma_int';
if (!this.shouldToneMap(videoStream)) { if (!this.shouldToneMap(videoStream)) {
options += ':out_color_matrix=bt601:out_range=pc'; options += ':out_color_matrix=601:out_range=pc';
} }
return options; return options;
} }
getColors() { getColors() {
return { return {
// jpeg and webp only support bt.601, so we need to convert to that directly when tone-mapping to avoid color shifts primaries: 'bt709',
primaries: 'bt470bg',
transfer: '601', transfer: '601',
matrix: 'bt470bg', matrix: 'bt470bg',
}; };

View file

@ -1,15 +1,29 @@
import { Colorspace } from '@app/infra/entities';
import { ApiProperty } from '@nestjs/swagger'; import { ApiProperty } from '@nestjs/swagger';
import { Type } from 'class-transformer'; import { Type } from 'class-transformer';
import { IsInt } from 'class-validator'; import { IsEnum, IsInt, Max, Min } from 'class-validator';
export class SystemConfigThumbnailDto { export class SystemConfigThumbnailDto {
@IsInt() @IsInt()
@Min(1)
@Type(() => Number) @Type(() => Number)
@ApiProperty({ type: 'integer' }) @ApiProperty({ type: 'integer' })
webpSize!: number; webpSize!: number;
@IsInt() @IsInt()
@Min(1)
@Type(() => Number) @Type(() => Number)
@ApiProperty({ type: 'integer' }) @ApiProperty({ type: 'integer' })
jpegSize!: number; jpegSize!: number;
@IsInt()
@Min(1)
@Max(100)
@Type(() => Number)
@ApiProperty({ type: 'integer' })
quality!: number;
@IsEnum(Colorspace)
@ApiProperty({ enumName: 'Colorspace', enum: Colorspace })
colorspace!: Colorspace;
} }

View file

@ -1,6 +1,7 @@
import { import {
AudioCodec, AudioCodec,
CQMode, CQMode,
Colorspace,
SystemConfig, SystemConfig,
SystemConfigEntity, SystemConfigEntity,
SystemConfigKey, SystemConfigKey,
@ -98,6 +99,8 @@ export const defaults = Object.freeze<SystemConfig>({
thumbnail: { thumbnail: {
webpSize: 250, webpSize: 250,
jpegSize: 1440, jpegSize: 1440,
quality: 80,
colorspace: Colorspace.P3,
}, },
}); });

View file

@ -1,6 +1,7 @@
import { import {
AudioCodec, AudioCodec,
CQMode, CQMode,
Colorspace,
SystemConfig, SystemConfig,
SystemConfigEntity, SystemConfigEntity,
SystemConfigKey, SystemConfigKey,
@ -94,6 +95,8 @@ const updatedConfig = Object.freeze<SystemConfig>({
thumbnail: { thumbnail: {
webpSize: 250, webpSize: 250,
jpegSize: 1440, jpegSize: 1440,
quality: 80,
colorspace: Colorspace.P3,
}, },
}); });

View file

@ -76,6 +76,8 @@ export enum SystemConfigKey {
THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize', THUMBNAIL_WEBP_SIZE = 'thumbnail.webpSize',
THUMBNAIL_JPEG_SIZE = 'thumbnail.jpegSize', THUMBNAIL_JPEG_SIZE = 'thumbnail.jpegSize',
THUMBNAIL_QUALITY = 'thumbnail.quality',
THUMBNAIL_COLORSPACE = 'thumbnail.colorspace',
} }
export enum TranscodePolicy { export enum TranscodePolicy {
@ -117,6 +119,11 @@ export enum CQMode {
ICQ = 'icq', ICQ = 'icq',
} }
export enum Colorspace {
SRGB = 'srgb',
P3 = 'p3',
}
export interface SystemConfig { export interface SystemConfig {
ffmpeg: { ffmpeg: {
crf: number; crf: number;
@ -179,5 +186,7 @@ export interface SystemConfig {
thumbnail: { thumbnail: {
webpSize: number; webpSize: number;
jpegSize: number; jpegSize: number;
quality: number;
colorspace: Colorspace;
}; };
} }

View file

@ -1,8 +1,10 @@
import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain'; import { CropOptions, IMediaRepository, ResizeOptions, TranscodeOptions, VideoInfo } from '@app/domain';
import { Colorspace } from '@app/infra/entities';
import { Logger } from '@nestjs/common'; import { Logger } from '@nestjs/common';
import ffmpeg, { FfprobeData } from 'fluent-ffmpeg'; import ffmpeg, { FfprobeData } from 'fluent-ffmpeg';
import fs from 'fs/promises'; import fs from 'fs/promises';
import sharp from 'sharp'; import sharp from 'sharp';
import { Writable } from 'stream';
import { promisify } from 'util'; import { promisify } from 'util';
const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe); const probe = promisify<string, FfprobeData>(ffmpeg.ffprobe);
@ -11,7 +13,7 @@ sharp.concurrency(0);
export class MediaRepository implements IMediaRepository { export class MediaRepository implements IMediaRepository {
private logger = new Logger(MediaRepository.name); private logger = new Logger(MediaRepository.name);
crop(input: string, options: CropOptions): Promise<Buffer> { crop(input: string | Buffer, options: CropOptions): Promise<Buffer> {
return sharp(input, { failOn: 'none' }) return sharp(input, { failOn: 'none' })
.extract({ .extract({
left: options.left, left: options.left,
@ -23,10 +25,25 @@ export class MediaRepository implements IMediaRepository {
} }
async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void> { async resize(input: string | Buffer, output: string, options: ResizeOptions): Promise<void> {
await sharp(input, { failOn: 'none' }) let colorProfile = options.colorspace;
if (options.colorspace !== Colorspace.SRGB) {
try {
const { space } = await sharp(input).metadata();
// if the image is already in srgb, keep it that way
if (space === 'srgb') {
colorProfile = Colorspace.SRGB;
}
} catch (err) {
this.logger.warn(`Could not determine colorspace of image, defaulting to ${colorProfile} profile`);
}
}
const chromaSubsampling = options.quality >= 80 ? '4:4:4' : '4:2:0'; // this is default in libvips (except the threshold is 90), but we need to set it manually in sharp
sharp(input, { failOn: 'none' })
.pipelineColorspace('rgb16')
.resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true }) .resize(options.size, options.size, { fit: 'outside', withoutEnlargement: true })
.rotate() .rotate()
.toFormat(options.format) .withMetadata({ icc: colorProfile })
.toFormat(options.format, { quality: options.quality, chromaSubsampling })
.toFile(output); .toFile(output);
} }
@ -61,7 +78,7 @@ export class MediaRepository implements IMediaRepository {
}; };
} }
transcode(input: string, output: string, options: TranscodeOptions): Promise<void> { transcode(input: string, output: string | Writable, options: TranscodeOptions): Promise<void> {
if (!options.twoPass) { if (!options.twoPass) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
ffmpeg(input, { niceness: 10 }) ffmpeg(input, { niceness: 10 })
@ -77,6 +94,10 @@ export class MediaRepository implements IMediaRepository {
}); });
} }
if (typeof output !== 'string') {
throw new Error('Two-pass transcoding does not support writing to a stream');
}
// two-pass allows for precise control of bitrate at the cost of running twice // two-pass allows for precise control of bitrate at the cost of running twice
// recommended for vp9 for better quality and compression // recommended for vp9 for better quality and compression
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {

View file

@ -1046,6 +1046,20 @@ export interface ClassificationConfig {
} }
/**
*
* @export
* @enum {string}
*/
export const Colorspace = {
Srgb: 'srgb',
P3: 'p3'
} as const;
export type Colorspace = typeof Colorspace[keyof typeof Colorspace];
/** /**
* *
* @export * @export
@ -3184,12 +3198,24 @@ export interface SystemConfigTemplateStorageOptionDto {
* @interface SystemConfigThumbnailDto * @interface SystemConfigThumbnailDto
*/ */
export interface SystemConfigThumbnailDto { export interface SystemConfigThumbnailDto {
/**
*
* @type {Colorspace}
* @memberof SystemConfigThumbnailDto
*/
'colorspace': Colorspace;
/** /**
* *
* @type {number} * @type {number}
* @memberof SystemConfigThumbnailDto * @memberof SystemConfigThumbnailDto
*/ */
'jpegSize': number; 'jpegSize': number;
/**
*
* @type {number}
* @memberof SystemConfigThumbnailDto
*/
'quality': number;
/** /**
* *
* @type {number} * @type {number}
@ -3197,6 +3223,8 @@ export interface SystemConfigThumbnailDto {
*/ */
'webpSize': number; 'webpSize': number;
} }
/** /**
* *
* @export * @export

View file

@ -1,12 +1,16 @@
<script lang="ts"> <script lang="ts">
import { quintOut } from 'svelte/easing'; import { quintOut } from 'svelte/easing';
import { fly } from 'svelte/transition'; import { fly } from 'svelte/transition';
import { createEventDispatcher } from 'svelte';
export let title: string; export let title: string;
export let subtitle = ''; export let subtitle = '';
export let checked = false; export let checked = false;
export let disabled = false; export let disabled = false;
export let isEdited = false; export let isEdited = false;
const dispatch = createEventDispatcher<{ toggle: boolean }>();
const onToggle = (event: Event) => dispatch('toggle', (event.target as HTMLInputElement).checked);
</script> </script>
<div class="flex place-items-center justify-between"> <div class="flex place-items-center justify-between">
@ -29,7 +33,13 @@
</div> </div>
<label class="relative inline-block h-[10px] w-[36px] flex-none"> <label class="relative inline-block h-[10px] w-[36px] flex-none">
<input class="disabled::cursor-not-allowed h-0 w-0 opacity-0" type="checkbox" bind:checked on:click {disabled} /> <input
class="disabled::cursor-not-allowed h-0 w-0 opacity-0"
type="checkbox"
bind:checked
on:click={onToggle}
{disabled}
/>
{#if disabled} {#if disabled}
<span class="slider-disable cursor-not-allowed" /> <span class="slider-disable cursor-not-allowed" />

View file

@ -1,6 +1,6 @@
<script lang="ts"> <script lang="ts">
import SettingSelect from '$lib/components/admin-page/settings/setting-select.svelte'; import SettingSelect from '$lib/components/admin-page/settings/setting-select.svelte';
import { api, SystemConfigThumbnailDto } from '@api'; import { api, Colorspace, SystemConfigThumbnailDto } from '@api';
import { fade } from 'svelte/transition'; import { fade } from 'svelte/transition';
import { isEqual } from 'lodash-es'; import { isEqual } from 'lodash-es';
import SettingButtonsRow from '$lib/components/admin-page/settings/setting-buttons-row.svelte'; import SettingButtonsRow from '$lib/components/admin-page/settings/setting-buttons-row.svelte';
@ -8,6 +8,8 @@
notificationController, notificationController,
NotificationType, NotificationType,
} from '$lib/components/shared-components/notification/notification'; } from '$lib/components/shared-components/notification/notification';
import SettingInputField, { SettingInputFieldType } from '../setting-input-field.svelte';
import SettingSwitch from '../setting-switch.svelte';
export let thumbnailConfig: SystemConfigThumbnailDto; // this is the config that is being edited export let thumbnailConfig: SystemConfigThumbnailDto; // this is the config that is being edited
export let disabled = false; export let disabled = false;
@ -91,7 +93,7 @@
{ value: 250, text: '250p' }, { value: 250, text: '250p' },
]} ]}
name="resolution" name="resolution"
isEdited={!(thumbnailConfig.webpSize === savedConfig.webpSize)} isEdited={thumbnailConfig.webpSize !== savedConfig.webpSize}
{disabled} {disabled}
/> />
@ -105,9 +107,25 @@
{ value: 1440, text: '1440p' }, { value: 1440, text: '1440p' },
]} ]}
name="resolution" name="resolution"
isEdited={!(thumbnailConfig.jpegSize === savedConfig.jpegSize)} isEdited={thumbnailConfig.jpegSize !== savedConfig.jpegSize}
{disabled} {disabled}
/> />
<SettingInputField
inputType={SettingInputFieldType.NUMBER}
label="QUALITY"
desc="Thumbnail quality from 1-100. Higher is better for quality but produces larger files."
bind:value={thumbnailConfig.quality}
isEdited={thumbnailConfig.quality !== savedConfig.quality}
/>
<SettingSwitch
title="PREFER WIDE GAMUT"
subtitle="Use Display P3 for thumbnails. This better preserves the vibrance of images with wide colorspaces, but images may appear differently on old devices with an old browser version. sRGB images are kept as sRGB to avoid color shifts."
checked={thumbnailConfig.colorspace === Colorspace.P3}
on:toggle={(e) => (thumbnailConfig.colorspace = e.detail ? Colorspace.P3 : Colorspace.Srgb)}
isEdited={thumbnailConfig.colorspace !== savedConfig.colorspace}
/>
</div> </div>
<div class="ml-4"> <div class="ml-4">