diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a8a8f0e5c..9d89063f2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -504,6 +504,7 @@ jobs: run: | echo "ERROR: Generated migration files not up to date!" echo "Changed files: ${{ steps.verify-changed-files.outputs.changed_files }}" + cat ./src/migrations/*-TestMigration.ts exit 1 - name: Run SQL generation diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index d006ef38b..e86ac9335 100644 Binary files a/mobile/openapi/README.md and b/mobile/openapi/README.md differ diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 2a2b6d46a..e5794a269 100644 Binary files a/mobile/openapi/lib/api.dart and b/mobile/openapi/lib/api.dart differ diff --git a/mobile/openapi/lib/api/sync_api.dart b/mobile/openapi/lib/api/sync_api.dart index f94eb8808..49a4963bf 100644 Binary files a/mobile/openapi/lib/api/sync_api.dart and b/mobile/openapi/lib/api/sync_api.dart differ diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 49fbe9464..54a8959f6 100644 Binary files a/mobile/openapi/lib/api_client.dart and b/mobile/openapi/lib/api_client.dart differ diff --git a/mobile/openapi/lib/api_helper.dart b/mobile/openapi/lib/api_helper.dart index 6a917201a..1ebf8314a 100644 Binary files a/mobile/openapi/lib/api_helper.dart and b/mobile/openapi/lib/api_helper.dart differ diff --git a/mobile/openapi/lib/model/sync_ack_delete_dto.dart b/mobile/openapi/lib/model/sync_ack_delete_dto.dart new file mode 100644 index 000000000..998f812f2 Binary files /dev/null and b/mobile/openapi/lib/model/sync_ack_delete_dto.dart differ diff --git a/mobile/openapi/lib/model/sync_ack_dto.dart b/mobile/openapi/lib/model/sync_ack_dto.dart new file mode 100644 index 000000000..c7fafa17d Binary files /dev/null and b/mobile/openapi/lib/model/sync_ack_dto.dart differ diff --git a/mobile/openapi/lib/model/sync_ack_set_dto.dart b/mobile/openapi/lib/model/sync_ack_set_dto.dart new file mode 100644 index 000000000..0d9eedc38 Binary files /dev/null and b/mobile/openapi/lib/model/sync_ack_set_dto.dart differ diff --git a/mobile/openapi/lib/model/sync_entity_type.dart b/mobile/openapi/lib/model/sync_entity_type.dart new file mode 100644 index 000000000..ed82205a3 Binary files /dev/null and b/mobile/openapi/lib/model/sync_entity_type.dart differ diff --git a/mobile/openapi/lib/model/sync_request_type.dart b/mobile/openapi/lib/model/sync_request_type.dart new file mode 100644 index 000000000..d7f1bde54 Binary files /dev/null and b/mobile/openapi/lib/model/sync_request_type.dart differ diff --git a/mobile/openapi/lib/model/sync_stream_dto.dart b/mobile/openapi/lib/model/sync_stream_dto.dart new file mode 100644 index 000000000..28fd3dfae Binary files /dev/null and b/mobile/openapi/lib/model/sync_stream_dto.dart differ diff --git a/mobile/openapi/lib/model/sync_user_delete_v1.dart b/mobile/openapi/lib/model/sync_user_delete_v1.dart new file mode 100644 index 000000000..09411cb79 Binary files /dev/null and b/mobile/openapi/lib/model/sync_user_delete_v1.dart differ diff --git a/mobile/openapi/lib/model/sync_user_v1.dart b/mobile/openapi/lib/model/sync_user_v1.dart new file mode 100644 index 000000000..b9b41bb72 Binary files /dev/null and b/mobile/openapi/lib/model/sync_user_v1.dart differ diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 14245e11b..1d0f06599 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -5802,6 +5802,107 @@ ] } }, + "/sync/ack": { + "delete": { + "operationId": "deleteSyncAck", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncAckDeleteDto" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + }, + "get": { + "operationId": "getSyncAck", + "parameters": [], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "items": { + "$ref": "#/components/schemas/SyncAckDto" + }, + "type": "array" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + }, + "post": { + "operationId": "sendSyncAck", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncAckSetDto" + } + } + }, + "required": true + }, + "responses": { + "204": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/sync/delta-sync": { "post": { "operationId": "getDeltaSync", @@ -5889,6 +5990,41 @@ ] } }, + "/sync/stream": { + "post": { + "operationId": "getSyncStream", + "parameters": [], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SyncStreamDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Sync" + ] + } + }, "/system-config": { "get": { "operationId": "getConfig", @@ -11696,6 +11832,113 @@ }, "type": "object" }, + "SyncAckDeleteDto": { + "properties": { + "types": { + "items": { + "$ref": "#/components/schemas/SyncEntityType" + }, + "type": "array" + } + }, + "type": "object" + }, + "SyncAckDto": { + "properties": { + "ack": { + "type": "string" + }, + "type": { + "allOf": [ + { + "$ref": "#/components/schemas/SyncEntityType" + } + ] + } + }, + "required": [ + "ack", + "type" + ], + "type": "object" + }, + "SyncAckSetDto": { + "properties": { + "acks": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "acks" + ], + "type": "object" + }, + "SyncEntityType": { + "enum": [ + "UserV1", + "UserDeleteV1" + ], + "type": "string" + }, + "SyncRequestType": { + "enum": [ + "UsersV1" + ], + "type": "string" + }, + "SyncStreamDto": { + "properties": { + "types": { + "items": { + "$ref": "#/components/schemas/SyncRequestType" + }, + "type": "array" + } + }, + "required": [ + "types" + ], + "type": "object" + }, + "SyncUserDeleteV1": { + "properties": { + "userId": { + "type": "string" + } + }, + "required": [ + "userId" + ], + "type": "object" + }, + "SyncUserV1": { + "properties": { + "deletedAt": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "email": { + "type": "string" + }, + "id": { + "type": "string" + }, + "name": { + "type": "string" + } + }, + "required": [ + "deletedAt", + "email", + "id", + "name" + ], + "type": "object" + }, "SystemConfigBackupsDto": { "properties": { "database": { diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index 9ff35331f..8b2e88183 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -1104,6 +1104,16 @@ export type StackCreateDto = { export type StackUpdateDto = { primaryAssetId?: string; }; +export type SyncAckDeleteDto = { + types?: SyncEntityType[]; +}; +export type SyncAckDto = { + ack: string; + "type": SyncEntityType; +}; +export type SyncAckSetDto = { + acks: string[]; +}; export type AssetDeltaSyncDto = { updatedAfter: string; userIds: string[]; @@ -1119,6 +1129,9 @@ export type AssetFullSyncDto = { updatedUntil: string; userId?: string; }; +export type SyncStreamDto = { + types: SyncRequestType[]; +}; export type DatabaseBackupConfig = { cronExpression: string; enabled: boolean; @@ -2912,6 +2925,32 @@ export function updateStack({ id, stackUpdateDto }: { body: stackUpdateDto }))); } +export function deleteSyncAck({ syncAckDeleteDto }: { + syncAckDeleteDto: SyncAckDeleteDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/ack", oazapfts.json({ + ...opts, + method: "DELETE", + body: syncAckDeleteDto + }))); +} +export function getSyncAck(opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchJson<{ + status: 200; + data: SyncAckDto[]; + }>("/sync/ack", { + ...opts + })); +} +export function sendSyncAck({ syncAckSetDto }: { + syncAckSetDto: SyncAckSetDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/ack", oazapfts.json({ + ...opts, + method: "POST", + body: syncAckSetDto + }))); +} export function getDeltaSync({ assetDeltaSyncDto }: { assetDeltaSyncDto: AssetDeltaSyncDto; }, opts?: Oazapfts.RequestOpts) { @@ -2936,6 +2975,15 @@ export function getFullSyncForUser({ assetFullSyncDto }: { body: assetFullSyncDto }))); } +export function getSyncStream({ syncStreamDto }: { + syncStreamDto: SyncStreamDto; +}, opts?: Oazapfts.RequestOpts) { + return oazapfts.ok(oazapfts.fetchText("/sync/stream", oazapfts.json({ + ...opts, + method: "POST", + body: syncStreamDto + }))); +} export function getConfig(opts?: Oazapfts.RequestOpts) { return oazapfts.ok(oazapfts.fetchJson<{ status: 200; @@ -3548,6 +3596,13 @@ export enum Error2 { NoPermission = "no_permission", NotFound = "not_found" } +export enum SyncEntityType { + UserV1 = "UserV1", + UserDeleteV1 = "UserDeleteV1" +} +export enum SyncRequestType { + UsersV1 = "UsersV1" +} export enum TranscodeHWAccel { Nvenc = "nvenc", Qsv = "qsv", diff --git a/server/src/app.module.ts b/server/src/app.module.ts index a4518598a..b02d869a1 100644 --- a/server/src/app.module.ts +++ b/server/src/app.module.ts @@ -29,7 +29,7 @@ import { AuthService } from 'src/services/auth.service'; import { CliService } from 'src/services/cli.service'; import { DatabaseService } from 'src/services/database.service'; -const common = [...repositories, ...services]; +const common = [...repositories, ...services, GlobalExceptionFilter]; const middleware = [ FileUploadInterceptor, diff --git a/server/src/controllers/sync.controller.ts b/server/src/controllers/sync.controller.ts index 4d970a710..0945810be 100644 --- a/server/src/controllers/sync.controller.ts +++ b/server/src/controllers/sync.controller.ts @@ -1,15 +1,28 @@ -import { Body, Controller, HttpCode, HttpStatus, Post } from '@nestjs/common'; +import { Body, Controller, Delete, Get, Header, HttpCode, HttpStatus, Post, Res } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; +import { Response } from 'express'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; +import { + AssetDeltaSyncDto, + AssetDeltaSyncResponseDto, + AssetFullSyncDto, + SyncAckDeleteDto, + SyncAckDto, + SyncAckSetDto, + SyncStreamDto, +} from 'src/dtos/sync.dto'; import { Auth, Authenticated } from 'src/middleware/auth.guard'; +import { GlobalExceptionFilter } from 'src/middleware/global-exception.filter'; import { SyncService } from 'src/services/sync.service'; @ApiTags('Sync') @Controller('sync') export class SyncController { - constructor(private service: SyncService) {} + constructor( + private service: SyncService, + private errorService: GlobalExceptionFilter, + ) {} @Post('full-sync') @HttpCode(HttpStatus.OK) @@ -24,4 +37,37 @@ export class SyncController { getDeltaSync(@Auth() auth: AuthDto, @Body() dto: AssetDeltaSyncDto): Promise { return this.service.getDeltaSync(auth, dto); } + + @Post('stream') + @Header('Content-Type', 'application/jsonlines+json') + @HttpCode(HttpStatus.OK) + @Authenticated() + async getSyncStream(@Auth() auth: AuthDto, @Res() res: Response, @Body() dto: SyncStreamDto) { + try { + await this.service.stream(auth, res, dto); + } catch (error: Error | any) { + res.setHeader('Content-Type', 'application/json'); + this.errorService.handleError(res, error); + } + } + + @Get('ack') + @Authenticated() + getSyncAck(@Auth() auth: AuthDto): Promise { + return this.service.getAcks(auth); + } + + @Post('ack') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated() + sendSyncAck(@Auth() auth: AuthDto, @Body() dto: SyncAckSetDto) { + return this.service.setAcks(auth, dto); + } + + @Delete('ack') + @HttpCode(HttpStatus.NO_CONTENT) + @Authenticated() + deleteSyncAck(@Auth() auth: AuthDto, @Body() dto: SyncAckDeleteDto) { + return this.service.deleteAcks(auth, dto); + } } diff --git a/server/src/database.ts b/server/src/database.ts index 4fcab0fd6..c3fb4cbab 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -1,3 +1,4 @@ +import { sql } from 'kysely'; import { Permission } from 'src/enum'; export type AuthUser = { @@ -29,6 +30,8 @@ export type AuthSession = { }; export const columns = { + ackEpoch: (columnName: 'createdAt' | 'updatedAt' | 'deletedAt') => + sql.raw(`extract(epoch from "${columnName}")::text`).as('ackEpoch'), authUser: [ 'users.id', 'users.name', diff --git a/server/src/db.d.ts b/server/src/db.d.ts index 2e10e1ade..255ac8cd2 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -4,7 +4,7 @@ */ import type { ColumnType } from 'kysely'; -import { Permission } from 'src/enum'; +import { Permission, SyncEntityType } from 'src/enum'; export type ArrayType = ArrayTypeImpl extends (infer U)[] ? U[] : ArrayTypeImpl; @@ -294,6 +294,15 @@ export interface Sessions { userId: string; } +export interface SessionSyncCheckpoints { + ack: string; + createdAt: Generated; + sessionId: string; + type: SyncEntityType; + updatedAt: Generated; +} + + export interface SharedLinkAsset { assetsId: string; sharedLinksId: string; @@ -384,6 +393,11 @@ export interface Users { updatedAt: Generated; } +export interface UsersAudit { + userId: string; + deletedAt: Generated; +} + export interface VectorsPgVectorIndexStat { idx_growing: ArrayType | null; idx_indexing: boolean | null; @@ -429,6 +443,7 @@ export interface DB { partners: Partners; person: Person; sessions: Sessions; + session_sync_checkpoints: SessionSyncCheckpoints; shared_link__asset: SharedLinkAsset; shared_links: SharedLinks; smart_search: SmartSearch; @@ -440,6 +455,7 @@ export interface DB { typeorm_metadata: TypeormMetadata; user_metadata: UserMetadata; users: Users; + users_audit: UsersAudit; 'vectors.pg_vector_index_stat': VectorsPgVectorIndexStat; version_history: VersionHistory; } diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts index 820de8d6c..0628a566c 100644 --- a/server/src/dtos/sync.dto.ts +++ b/server/src/dtos/sync.dto.ts @@ -1,7 +1,8 @@ import { ApiProperty } from '@nestjs/swagger'; -import { IsInt, IsPositive } from 'class-validator'; +import { IsEnum, IsInt, IsPositive, IsString } from 'class-validator'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; -import { ValidateDate, ValidateUUID } from 'src/validation'; +import { SyncEntityType, SyncRequestType } from 'src/enum'; +import { Optional, ValidateDate, ValidateUUID } from 'src/validation'; export class AssetFullSyncDto { @ValidateUUID({ optional: true }) @@ -32,3 +33,51 @@ export class AssetDeltaSyncResponseDto { upserted!: AssetResponseDto[]; deleted!: string[]; } + +export class SyncUserV1 { + id!: string; + name!: string; + email!: string; + deletedAt!: Date | null; +} + +export class SyncUserDeleteV1 { + userId!: string; +} + +export type SyncItem = { + [SyncEntityType.UserV1]: SyncUserV1; + [SyncEntityType.UserDeleteV1]: SyncUserDeleteV1; +}; + +const responseDtos = [ + // + SyncUserV1, + SyncUserDeleteV1, +]; + +export const extraSyncModels = responseDtos; + +export class SyncStreamDto { + @IsEnum(SyncRequestType, { each: true }) + @ApiProperty({ enumName: 'SyncRequestType', enum: SyncRequestType, isArray: true }) + types!: SyncRequestType[]; +} + +export class SyncAckDto { + @ApiProperty({ enumName: 'SyncEntityType', enum: SyncEntityType }) + type!: SyncEntityType; + ack!: string; +} + +export class SyncAckSetDto { + @IsString({ each: true }) + acks!: string[]; +} + +export class SyncAckDeleteDto { + @IsEnum(SyncEntityType, { each: true }) + @ApiProperty({ enumName: 'SyncEntityType', enum: SyncEntityType, isArray: true }) + @Optional() + types?: SyncEntityType[]; +} diff --git a/server/src/entities/index.ts b/server/src/entities/index.ts index 75e92038a..a1df269c0 100644 --- a/server/src/entities/index.ts +++ b/server/src/entities/index.ts @@ -20,8 +20,10 @@ import { SessionEntity } from 'src/entities/session.entity'; import { SharedLinkEntity } from 'src/entities/shared-link.entity'; import { SmartSearchEntity } from 'src/entities/smart-search.entity'; import { StackEntity } from 'src/entities/stack.entity'; +import { SessionSyncCheckpointEntity } from 'src/entities/sync-checkpoint.entity'; import { SystemMetadataEntity } from 'src/entities/system-metadata.entity'; import { TagEntity } from 'src/entities/tag.entity'; +import { UserAuditEntity } from 'src/entities/user-audit.entity'; import { UserMetadataEntity } from 'src/entities/user-metadata.entity'; import { UserEntity } from 'src/entities/user.entity'; import { VersionHistoryEntity } from 'src/entities/version-history.entity'; @@ -44,12 +46,14 @@ export const entities = [ MoveEntity, PartnerEntity, PersonEntity, + SessionSyncCheckpointEntity, SharedLinkEntity, SmartSearchEntity, StackEntity, SystemMetadataEntity, TagEntity, UserEntity, + UserAuditEntity, UserMetadataEntity, SessionEntity, LibraryEntity, diff --git a/server/src/entities/sync-checkpoint.entity.ts b/server/src/entities/sync-checkpoint.entity.ts new file mode 100644 index 000000000..2a91d2386 --- /dev/null +++ b/server/src/entities/sync-checkpoint.entity.ts @@ -0,0 +1,24 @@ +import { SessionEntity } from 'src/entities/session.entity'; +import { SyncEntityType } from 'src/enum'; +import { Column, CreateDateColumn, Entity, ManyToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; + +@Entity('session_sync_checkpoints') +export class SessionSyncCheckpointEntity { + @ManyToOne(() => SessionEntity, { onDelete: 'CASCADE', onUpdate: 'CASCADE' }) + session?: SessionEntity; + + @PrimaryColumn() + sessionId!: string; + + @PrimaryColumn({ type: 'varchar' }) + type!: SyncEntityType; + + @CreateDateColumn({ type: 'timestamptz' }) + createdAt!: Date; + + @UpdateDateColumn({ type: 'timestamptz' }) + updatedAt!: Date; + + @Column() + ack!: string; +} diff --git a/server/src/entities/user-audit.entity.ts b/server/src/entities/user-audit.entity.ts new file mode 100644 index 000000000..305994a6d --- /dev/null +++ b/server/src/entities/user-audit.entity.ts @@ -0,0 +1,14 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryGeneratedColumn } from 'typeorm'; + +@Entity('users_audit') +@Index('IDX_users_audit_deleted_at_asc_user_id_asc', ['deletedAt', 'userId']) +export class UserAuditEntity { + @PrimaryGeneratedColumn('increment') + id!: number; + + @Column({ type: 'uuid' }) + userId!: string; + + @CreateDateColumn({ type: 'timestamptz' }) + deletedAt!: Date; +} diff --git a/server/src/entities/user.entity.ts b/server/src/entities/user.entity.ts index 3f5b470ce..b597d15cf 100644 --- a/server/src/entities/user.entity.ts +++ b/server/src/entities/user.entity.ts @@ -10,12 +10,14 @@ import { CreateDateColumn, DeleteDateColumn, Entity, + Index, OneToMany, PrimaryGeneratedColumn, UpdateDateColumn, } from 'typeorm'; @Entity('users') +@Index('IDX_users_updated_at_asc_id_asc', ['updatedAt', 'id']) export class UserEntity { @PrimaryGeneratedColumn('uuid') id!: string; diff --git a/server/src/enum.ts b/server/src/enum.ts index 0c1fb01a1..b99518c4f 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -537,3 +537,12 @@ export enum DatabaseLock { GetSystemConfig = 69, BackupDatabase = 42, } + +export enum SyncRequestType { + UsersV1 = 'UsersV1', +} + +export enum SyncEntityType { + UserV1 = 'UserV1', + UserDeleteV1 = 'UserDeleteV1', +} diff --git a/server/src/middleware/global-exception.filter.ts b/server/src/middleware/global-exception.filter.ts index 7d7ade471..a8afa91cb 100644 --- a/server/src/middleware/global-exception.filter.ts +++ b/server/src/middleware/global-exception.filter.ts @@ -22,6 +22,13 @@ export class GlobalExceptionFilter implements ExceptionFilter { } } + handleError(res: Response, error: Error) { + const { status, body } = this.fromError(error); + if (!res.headersSent) { + res.status(status).json({ ...body, statusCode: status, correlationId: this.cls.getId() }); + } + } + private fromError(error: Error) { logGlobalError(this.logger, error); diff --git a/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts b/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts new file mode 100644 index 000000000..ef75dd7c0 --- /dev/null +++ b/server/src/migrations/1740001232576-AddSessionSyncCheckpointTable.ts @@ -0,0 +1,22 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddSessionSyncCheckpointTable1740001232576 implements MigrationInterface { + name = 'AddSessionSyncCheckpointTable1740001232576' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "session_sync_checkpoints" ("sessionId" uuid NOT NULL, "type" character varying NOT NULL, "createdAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), "ack" character varying NOT NULL, CONSTRAINT "PK_b846ab547a702863ef7cd9412fb" PRIMARY KEY ("sessionId", "type"))`); + await queryRunner.query(`ALTER TABLE "session_sync_checkpoints" ADD CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc" FOREIGN KEY ("sessionId") REFERENCES "sessions"("id") ON DELETE CASCADE ON UPDATE CASCADE`); + await queryRunner.query(` + create trigger session_sync_checkpoints_updated_at + before update on session_sync_checkpoints + for each row execute procedure updated_at() + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`drop trigger session_sync_checkpoints_updated_at on session_sync_checkpoints`); + await queryRunner.query(`ALTER TABLE "session_sync_checkpoints" DROP CONSTRAINT "FK_d8ddd9d687816cc490432b3d4bc"`); + await queryRunner.query(`DROP TABLE "session_sync_checkpoints"`); + } + +} diff --git a/server/src/migrations/1740064899123-AddUsersAuditTable.ts b/server/src/migrations/1740064899123-AddUsersAuditTable.ts new file mode 100644 index 000000000..b8f2ce5e3 --- /dev/null +++ b/server/src/migrations/1740064899123-AddUsersAuditTable.ts @@ -0,0 +1,34 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AddUsersAuditTable1740064899123 implements MigrationInterface { + name = 'AddUsersAuditTable1740064899123' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_users_updated_at_asc_id_asc" ON "users" ("updatedAt" ASC, "id" ASC);`) + await queryRunner.query(`CREATE TABLE "users_audit" ("id" SERIAL NOT NULL, "userId" uuid NOT NULL, "deletedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT now(), CONSTRAINT "PK_e9b2bdfd90e7eb5961091175180" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX IF NOT EXISTS "IDX_users_audit_deleted_at_asc_user_id_asc" ON "users_audit" ("deletedAt" ASC, "userId" ASC);`) + await queryRunner.query(`CREATE OR REPLACE FUNCTION users_delete_audit() RETURNS TRIGGER AS + $$ + BEGIN + INSERT INTO users_audit ("userId") + SELECT "id" + FROM OLD; + RETURN NULL; + END; + $$ LANGUAGE plpgsql` + ); + await queryRunner.query(`CREATE OR REPLACE TRIGGER users_delete_audit + AFTER DELETE ON users + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION users_delete_audit(); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TRIGGER users_delete_audit`); + await queryRunner.query(`DROP FUNCTION users_delete_audit`); + await queryRunner.query(`DROP TABLE "users_audit"`); + } + +} diff --git a/server/src/repositories/index.ts b/server/src/repositories/index.ts index d3a8aeeb6..180d8ccd4 100644 --- a/server/src/repositories/index.ts +++ b/server/src/repositories/index.ts @@ -30,6 +30,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -71,6 +72,7 @@ export const repositories = [ SharedLinkRepository, StackRepository, StorageRepository, + SyncRepository, SystemMetadataRepository, TagRepository, TelemetryRepository, diff --git a/server/src/repositories/sync.repository.ts b/server/src/repositories/sync.repository.ts new file mode 100644 index 000000000..4023bf890 --- /dev/null +++ b/server/src/repositories/sync.repository.ts @@ -0,0 +1,80 @@ +import { Injectable } from '@nestjs/common'; +import { Insertable, Kysely, sql } from 'kysely'; +import { InjectKysely } from 'nestjs-kysely'; +import { columns } from 'src/database'; +import { DB, SessionSyncCheckpoints } from 'src/db'; +import { SyncEntityType } from 'src/enum'; +import { SyncAck } from 'src/types'; + +@Injectable() +export class SyncRepository { + constructor(@InjectKysely() private db: Kysely) {} + + getCheckpoints(sessionId: string) { + return this.db + .selectFrom('session_sync_checkpoints') + .select(['type', 'ack']) + .where('sessionId', '=', sessionId) + .execute(); + } + + upsertCheckpoints(items: Insertable[]) { + return this.db + .insertInto('session_sync_checkpoints') + .values(items) + .onConflict((oc) => + oc.columns(['sessionId', 'type']).doUpdateSet((eb) => ({ + ack: eb.ref('excluded.ack'), + })), + ) + .execute(); + } + + deleteCheckpoints(sessionId: string, types?: SyncEntityType[]) { + return this.db + .deleteFrom('session_sync_checkpoints') + .where('sessionId', '=', sessionId) + .$if(!!types, (qb) => qb.where('type', 'in', types!)) + .execute(); + } + + getUserUpserts(ack?: SyncAck) { + return this.db + .selectFrom('users') + .select(['id', 'name', 'email', 'deletedAt']) + .select(columns.ackEpoch('updatedAt')) + .$if(!!ack, (qb) => + qb.where((eb) => + eb.or([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<', eb.ref('updatedAt')), + eb.and([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<=', eb.ref('updatedAt')), + eb('id', '>', ack!.ids[0]), + ]), + ]), + ), + ) + .orderBy(['updatedAt asc', 'id asc']) + .stream(); + } + + getUserDeletes(ack?: SyncAck) { + return this.db + .selectFrom('users_audit') + .select(['userId']) + .select(columns.ackEpoch('deletedAt')) + .$if(!!ack, (qb) => + qb.where((eb) => + eb.or([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<', eb.ref('deletedAt')), + eb.and([ + eb(eb.fn('to_timestamp', [sql.val(ack!.ackEpoch)]), '<=', eb.ref('deletedAt')), + eb('userId', '>', ack!.ids[0]), + ]), + ]), + ), + ) + .orderBy(['deletedAt asc', 'userId asc']) + .stream(); + } +} diff --git a/server/src/services/base.service.ts b/server/src/services/base.service.ts index f476adba1..63cca43cc 100644 --- a/server/src/services/base.service.ts +++ b/server/src/services/base.service.ts @@ -38,6 +38,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -85,6 +86,7 @@ export class BaseService { protected sharedLinkRepository: SharedLinkRepository, protected stackRepository: StackRepository, protected storageRepository: StorageRepository, + protected syncRepository: SyncRepository, protected systemMetadataRepository: SystemMetadataRepository, protected tagRepository: TagRepository, protected telemetryRepository: TelemetryRepository, diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index fe967e37e..b94e8cfcb 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -1,18 +1,112 @@ -import { Injectable } from '@nestjs/common'; +import { ForbiddenException, Injectable } from '@nestjs/common'; +import { Insertable } from 'kysely'; import { DateTime } from 'luxon'; +import { Writable } from 'node:stream'; import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; +import { SessionSyncCheckpoints } from 'src/db'; import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; -import { AssetDeltaSyncDto, AssetDeltaSyncResponseDto, AssetFullSyncDto } from 'src/dtos/sync.dto'; -import { DatabaseAction, EntityType, Permission } from 'src/enum'; +import { + AssetDeltaSyncDto, + AssetDeltaSyncResponseDto, + AssetFullSyncDto, + SyncAckDeleteDto, + SyncAckSetDto, + SyncStreamDto, +} from 'src/dtos/sync.dto'; +import { DatabaseAction, EntityType, Permission, SyncEntityType, SyncRequestType } from 'src/enum'; import { BaseService } from 'src/services/base.service'; +import { SyncAck } from 'src/types'; import { getMyPartnerIds } from 'src/utils/asset.util'; import { setIsEqual } from 'src/utils/set'; +import { fromAck, serialize } from 'src/utils/sync'; const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] }; +const SYNC_TYPES_ORDER = [ + // + SyncRequestType.UsersV1, +]; + +const throwSessionRequired = () => { + throw new ForbiddenException('Sync endpoints cannot be used with API keys'); +}; @Injectable() export class SyncService extends BaseService { + getAcks(auth: AuthDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + return this.syncRepository.getCheckpoints(sessionId); + } + + async setAcks(auth: AuthDto, dto: SyncAckSetDto) { + // TODO ack validation + + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + const checkpoints: Insertable[] = []; + for (const ack of dto.acks) { + const { type } = fromAck(ack); + checkpoints.push({ sessionId, type, ack }); + } + + await this.syncRepository.upsertCheckpoints(checkpoints); + } + + async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + await this.syncRepository.deleteCheckpoints(sessionId, dto.types); + } + + async stream(auth: AuthDto, response: Writable, dto: SyncStreamDto) { + const sessionId = auth.session?.id; + if (!sessionId) { + return throwSessionRequired(); + } + + const checkpoints = await this.syncRepository.getCheckpoints(sessionId); + const checkpointMap: Partial> = Object.fromEntries( + checkpoints.map(({ type, ack }) => [type, fromAck(ack)]), + ); + + // TODO pre-filter/sort list based on optimal sync order + + for (const type of SYNC_TYPES_ORDER.filter((type) => dto.types.includes(type))) { + switch (type) { + case SyncRequestType.UsersV1: { + const deletes = this.syncRepository.getUserDeletes(checkpointMap[SyncEntityType.UserDeleteV1]); + for await (const { ackEpoch, ...data } of deletes) { + response.write(serialize({ type: SyncEntityType.UserDeleteV1, ackEpoch, ids: [data.userId], data })); + } + + const upserts = this.syncRepository.getUserUpserts(checkpointMap[SyncEntityType.UserV1]); + for await (const { ackEpoch, ...data } of upserts) { + response.write(serialize({ type: SyncEntityType.UserV1, ackEpoch, ids: [data.id], data })); + } + + break; + } + + default: { + this.logger.warn(`Unsupported sync type: ${type}`); + break; + } + } + } + + response.end(); + } + async getFullSync(auth: AuthDto, dto: AssetFullSyncDto): Promise { // mobile implementation is faster if this is a single id const userId = dto.userId || auth.user.id; diff --git a/server/src/types.ts b/server/src/types.ts index 3a331127e..544d35524 100644 --- a/server/src/types.ts +++ b/server/src/types.ts @@ -4,6 +4,7 @@ import { ImageFormat, JobName, QueueName, + SyncEntityType, TranscodeTarget, VideoCodec, } from 'src/enum'; @@ -409,3 +410,9 @@ export interface IBulkAsset { addAssetIds: (id: string, assetIds: string[]) => Promise; removeAssetIds: (id: string, assetIds: string[]) => Promise; } + +export type SyncAck = { + type: SyncEntityType; + ackEpoch: string; + ids: string[]; +}; diff --git a/server/src/utils/misc.ts b/server/src/utils/misc.ts index 13969543e..e07d0fe03 100644 --- a/server/src/utils/misc.ts +++ b/server/src/utils/misc.ts @@ -12,6 +12,7 @@ import { writeFileSync } from 'node:fs'; import path from 'node:path'; import { SystemConfig } from 'src/config'; import { CLIP_MODEL_INFO, serverVersion } from 'src/constants'; +import { extraSyncModels } from 'src/dtos/sync.dto'; import { ImmichCookie, ImmichHeader, MetadataKey } from 'src/enum'; import { LoggingRepository } from 'src/repositories/logging.repository'; @@ -245,6 +246,7 @@ export const useSwagger = (app: INestApplication, { write }: { write: boolean }) const options: SwaggerDocumentOptions = { operationIdFactory: (controllerKey: string, methodKey: string) => methodKey, + extraModels: extraSyncModels, }; const specification = SwaggerModule.createDocument(app, config, options); diff --git a/server/src/utils/sync.ts b/server/src/utils/sync.ts new file mode 100644 index 000000000..8e426ab86 --- /dev/null +++ b/server/src/utils/sync.ts @@ -0,0 +1,30 @@ +import { SyncItem } from 'src/dtos/sync.dto'; +import { SyncEntityType } from 'src/enum'; +import { SyncAck } from 'src/types'; + +type Impossible = { + [P in K]: never; +}; + +type Exact = U & Impossible>; + +export const fromAck = (ack: string): SyncAck => { + const [type, timestamp, ...ids] = ack.split('|'); + return { type: type as SyncEntityType, ackEpoch: timestamp, ids }; +}; + +export const toAck = ({ type, ackEpoch, ids }: SyncAck) => [type, ackEpoch, ...ids].join('|'); + +export const mapJsonLine = (object: unknown) => JSON.stringify(object) + '\n'; + +export const serialize = ({ + type, + ackEpoch, + ids, + data, +}: { + type: T; + ackEpoch: string; + ids: string[]; + data: Exact; +}) => mapJsonLine({ type, data, ack: toAck({ type, ackEpoch, ids }) }); diff --git a/server/test/repositories/sync.repository.mock.ts b/server/test/repositories/sync.repository.mock.ts new file mode 100644 index 000000000..fbb8ec2f6 --- /dev/null +++ b/server/test/repositories/sync.repository.mock.ts @@ -0,0 +1,13 @@ +import { SyncRepository } from 'src/repositories/sync.repository'; +import { RepositoryInterface } from 'src/types'; +import { Mocked, vitest } from 'vitest'; + +export const newSyncRepositoryMock = (): Mocked> => { + return { + getCheckpoints: vitest.fn(), + upsertCheckpoints: vitest.fn(), + deleteCheckpoints: vitest.fn(), + getUserUpserts: vitest.fn(), + getUserDeletes: vitest.fn(), + }; +}; diff --git a/server/test/utils.ts b/server/test/utils.ts index d1dda3eed..ca2272f6b 100644 --- a/server/test/utils.ts +++ b/server/test/utils.ts @@ -34,6 +34,7 @@ import { SessionRepository } from 'src/repositories/session.repository'; import { SharedLinkRepository } from 'src/repositories/shared-link.repository'; import { StackRepository } from 'src/repositories/stack.repository'; import { StorageRepository } from 'src/repositories/storage.repository'; +import { SyncRepository } from 'src/repositories/sync.repository'; import { SystemMetadataRepository } from 'src/repositories/system-metadata.repository'; import { TagRepository } from 'src/repositories/tag.repository'; import { TelemetryRepository } from 'src/repositories/telemetry.repository'; @@ -75,6 +76,7 @@ import { newSessionRepositoryMock } from 'test/repositories/session.repository.m import { newSharedLinkRepositoryMock } from 'test/repositories/shared-link.repository.mock'; import { newStackRepositoryMock } from 'test/repositories/stack.repository.mock'; import { newStorageRepositoryMock } from 'test/repositories/storage.repository.mock'; +import { newSyncRepositoryMock } from 'test/repositories/sync.repository.mock'; import { newSystemMetadataRepositoryMock } from 'test/repositories/system-metadata.repository.mock'; import { newTagRepositoryMock } from 'test/repositories/tag.repository.mock'; import { ITelemetryRepositoryMock, newTelemetryRepositoryMock } from 'test/repositories/telemetry.repository.mock'; @@ -178,6 +180,7 @@ export const newTestService = ( const sharedLinkMock = newSharedLinkRepositoryMock(); const stackMock = newStackRepositoryMock(); const storageMock = newStorageRepositoryMock(); + const syncMock = newSyncRepositoryMock(); const systemMock = newSystemMetadataRepositoryMock(); const tagMock = newTagRepositoryMock(); const telemetryMock = newTelemetryRepositoryMock(); @@ -219,6 +222,7 @@ export const newTestService = ( sharedLinkMock as RepositoryInterface as SharedLinkRepository, stackMock as RepositoryInterface as StackRepository, storageMock as RepositoryInterface as StorageRepository, + syncMock as RepositoryInterface as SyncRepository, systemMock as RepositoryInterface as SystemMetadataRepository, tagMock as RepositoryInterface as TagRepository, telemetryMock as unknown as TelemetryRepository,