From b19884d01e6e9ba5d4eb5c9b894570b0e380cff8 Mon Sep 17 00:00:00 2001 From: Zack Pollard Date: Thu, 10 Jul 2025 16:32:42 +0100 Subject: [PATCH] feat(server): people sync (#19854) * chore: fix missing usage of deleteType for syncMemoriesV1 * chore: add src path for proper absolute imports in jetbrains * feat: people sync --- mobile/openapi/README.md | Bin 38173 -> 38265 bytes mobile/openapi/lib/api.dart | Bin 13654 -> 13729 bytes mobile/openapi/lib/api_client.dart | Bin 34291 -> 34455 bytes .../openapi/lib/model/sync_entity_type.dart | Bin 7992 -> 8280 bytes .../lib/model/sync_person_delete_v1.dart | Bin 0 -> 2894 bytes mobile/openapi/lib/model/sync_person_v1.dart | Bin 0 -> 5630 bytes .../openapi/lib/model/sync_request_type.dart | Bin 4721 -> 4852 bytes open-api/immich-openapi-specs.json | 73 ++++++++++++++- open-api/typescript-sdk/src/fetch-client.ts | 5 +- server/src/dtos/sync.dto.ts | 22 +++++ server/src/enum.ts | 4 + server/src/queries/sync.repository.sql | 34 +++++++ server/src/repositories/sync.repository.ts | 43 ++++++++- server/src/schema/functions.ts | 13 +++ server/src/schema/index.ts | 5 + .../1752152941084-PeopleAuditTable.ts | 41 +++++++++ .../src/schema/tables/person-audit.table.ts | 17 ++++ server/src/schema/tables/person.table.ts | 8 ++ server/src/services/sync.service.ts | 18 +++- .../medium/specs/sync/sync-person.spec.ts | 87 ++++++++++++++++++ server/tsconfig.json | 3 + 21 files changed, 368 insertions(+), 5 deletions(-) create mode 100644 mobile/openapi/lib/model/sync_person_delete_v1.dart create mode 100644 mobile/openapi/lib/model/sync_person_v1.dart create mode 100644 server/src/schema/migrations/1752152941084-PeopleAuditTable.ts create mode 100644 server/src/schema/tables/person-audit.table.ts create mode 100644 server/test/medium/specs/sync/sync-person.spec.ts diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 1beaec0ae6521e38958cfb0002b97c6ff0918fa1..3ba9327ec494d5a98c065bbfe7e8989ca34cac0b 100644 GIT binary patch delta 76 zcmbQcis|PnrVSj;@&Tzu#rb(IsX3`7sbPk(8Y%h7`uf3@dC3ro$p;%mB_T{KvYUT4 HsR#i8Z)6=< delta 14 VcmeylifQgDrVSj;n^T%ag#a^o1?vC+ diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 3998961720520a5dafe2d89c5bb8365792966965..e771664827f73d6377e4ebdb271cf51fc996cab2 100644 GIT binary patch delta 34 jcmcbXwJ>{wml|6^YEf~1-sJmg$}r|bRhi9RYCQY^{>BYe delta 12 TcmZ3OeJyK)m)hp3YCQY^CmjU# diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 0edc2638bc4ac1c62f9114e142a5568351f1cce2..8bdda3a320a087a00124e1b308e59d3e919c0450 100644 GIT binary patch delta 58 vcmey|%{0B2X~X+?wt&>4;{3eHAEFdsOvN}I7?Uwx3dU57SKItPo<$7+x2G0G delta 14 VcmbQ<%k;UMX~X;Y&65*Y)BrTz26X@c diff --git a/mobile/openapi/lib/model/sync_entity_type.dart b/mobile/openapi/lib/model/sync_entity_type.dart index ecaadc9c31f68454974e1758e19e4bb21af2ede1..60cc4da6cb05b4581b497b55d17040eb52f5dff2 100644 GIT binary patch delta 128 zcmdmCcf(;rHaA;AYEf~1-sFu;5^MnwrXaH@Of;WK4kkL0NqTcOcOy+iQ}~VeH;c(TF#-T=jR(pA diff --git a/mobile/openapi/lib/model/sync_person_delete_v1.dart b/mobile/openapi/lib/model/sync_person_delete_v1.dart new file mode 100644 index 0000000000000000000000000000000000000000..002f5c5b83b2cd6eb55ed168612010663cb8d219 GIT binary patch literal 2894 zcmbVOZExE)5dQ98aVdsG0aSV2ry{An7E3auYvZ9!3k*geFcNLElSP%JY8a{i`|ggS zEJMngEkF{9Jl@OmJa^Q1JQ|PT&8OSh^WQITE{>vxwiT+HBRb_?%k7uQ#R z9ibUnzRrbllV6e-FGuvMmZdf_U1*aoR3XoxE^90ER2FhAOBaWCwXU>rgC|6>5nG#9 zwM&isS}TLcOR>hkl`#0IiP2*Wwh3(%D;MWqt_Js*v-LRbqQw`KMxCZfv=EX~Q!FW_MWFh&SINLQQE zf`C$PVBWX91H~z3GQlJL{fImPgyO-iluarQcAdw#w>!l=@T4=UTte94wb%NaGzLq@ z{d=toO5O;k4NNA{?EZ_)uq3yV<`gDV@DOF9h$u!JcD{f2PoM@1JwE3A*iD-k`hPqV zRXSKo2PF|H2@X5}-C`Rsoc0e9*LTtf zcOHCAehN=_Ck)+tQN8kqW5~CtIF(Jo>9@!+dh?ttR4EGhf=e)gp7q1-obv^?n46Oj z;p7g@4`}p;hNZ&Rw6G&q|D%LJlsWirvY?!WZ=$HBg@w`U#@n2iP<0D5I3Z!qvDUj7 z%DO;-kQz41vKTvrR7NhaC%BzVouPzmO5#efyPV2WY{Zq$i6Fl)Amm>;3kI+O)Mppa ztgEFFxx8>X;uK?e)+g*4a!u`k*?TKeCFUQT0349Qj*?7R*P|!$s`0%=Jvxwb#IeXE zaCZi4QGF&*XN%iyB^$TXh*S_mhvd^EYEJrTQr_`vLo$KglH5S{PEne@Q8I-BH?EMu z1MGaj2X5=7(Hn7JU?;&s6jsK&fQLuJNLv?2(3M{M-$d$*D1|GDkTwl?)mSW z^Z--ike>s%n^R6xVNrnV48jw8;f8Ct=(c}9_bI7hkdctAhX$<}bRNy|00`4CB&g~D zKF0o6?AdPOxp_?5I5?Wy_YB&LGR3dkJ{s qSqw`JA>i%D*$sW3H5%Y+u>BqKNX}Z^o~+ zvkO7@0bD_R@~H$r z^IN!#X|6z+YZZKq#4%<>+yGux&@-XN*Fx&q5z{>DK^9Hb$Xh&exiU^4^%qNS*B4!{^Ip@hPlDItvP6Tv~0W_R5*6^LMs7-nq{^N{ZG-sv2J z5?u|#D*{1F1Ag25;8J2Nqps!>7>$zYMnayC&ac1o0%KIqxZHt!A8$)BU$AOKVU&5y z&!2XM8YHm?&!3z4G6LG5L(#1@QyODV!rMi{AM9%I-pDC`WM!g?w}~uK5*q)f(BJ@r zZFg%_)+Lt4trH4z5Mj-4f^n`Qq+cfZ#BW5E0utKbNTLgyj#ovk;f8CNF*W<6De+?a z#wnqiQW33r!IffiMMV&OJmX3zLpBO=Z!}s~6&%1z*lRtwJ@%q^;1|mi+*r5t2G&RG zJ=ql#yN}G-3X3p3B1PGi)h1UT&U&$nArE$;&UIZx;;k+^IH^T-++_<3=*BKaChvxE zMZXLl$p>MX*^J2t#>}|$Rz-+kW^q2`gUE&%rb0fz8XeUj+=m1pxHS<$?J6_joe#)F zSP5+1%?ECgrIDKt64PLx_O&)2gyq%HSmPsG{BD)cpj@GNQS75V-eWUFYuI)+6*a5i zJAxpG4)xvr8xx0O3Z_s+rDxlQk(UET#-$tvLk_P{V%j9P$GZ8di_s4Lo~q{cZ!ceY z=h$=Zfme#Vt6^UC3#IUN^y%$vwmft*gKckRwvQj%a;zU2ZWdx9670idleWRCQ2@b1 zw~uJrqaxtrnnZ+#`$2s~*-EFDN+VBDwvFS6D8zP$*oJpZY9VWc6!$F=?bZlfCbTpT zutt_7MoXN4*H}WvwB!M_$$QcaktRY(>OBoPPaZCT)g{>3T7l3Cez0oE6-;F_cXBQb z2Oc$0r#K;-;aV;WoIlirCX)%y+Bk4ba4erS zC3PeVNXCo*+I`^e9>=t__~X7`(G*2S}=o zAsH7;=}z4l*_n}qIWx!SjLa#748>*-=4|07vk{fE4B}cuyWx(VRf*ALwqg24hCiQP ztQOo|V?XfQ{EyjuYWJVl76W_Uc6(ryfE$mEXyZGJaw8{Z?@n}Ro39;C8c|zCTf#Md zo3IMhYe|q_i1cXJWndf zmMH*qGiMs1YJ zIkh}vp~k+RZ-t&Ah;oI#_QckIVM&m~by6I2k(54pkt7TqmwO+hk(i<;&BJ{nCU9>o zWbm|HL>Y0E^c?elV&#cp@RX2p?K8^rjS+)7rH+lQKPHz1lSP4(g;Z>lG7HhvlhJrg z6a>3)G2_AC;lmq-mcoYxL)R`b6s|eo(4+8J3V8#%?GYN&Sc7G9Xc): Promise { + await sql`CREATE OR REPLACE FUNCTION person_delete_audit() + RETURNS TRIGGER + LANGUAGE PLPGSQL + AS $$ + BEGIN + INSERT INTO person_audit ("personId", "ownerId") + SELECT "id", "ownerId" + FROM OLD; + RETURN NULL; + END + $$;`.execute(db); + await sql`CREATE TABLE "person_audit" ( + "id" uuid NOT NULL DEFAULT immich_uuid_v7(), + "personId" uuid NOT NULL, + "ownerId" uuid NOT NULL, + "deletedAt" timestamp with time zone NOT NULL DEFAULT clock_timestamp(), + CONSTRAINT "PK_46c1ad23490b9312ffaa052aa59" PRIMARY KEY ("id") +);`.execute(db); + await sql`CREATE INDEX "IDX_person_audit_person_id" ON "person_audit" ("personId");`.execute(db); + await sql`CREATE INDEX "IDX_person_audit_owner_id" ON "person_audit" ("ownerId");`.execute(db); + await sql`CREATE INDEX "IDX_person_audit_deleted_at" ON "person_audit" ("deletedAt");`.execute(db); + await sql`CREATE OR REPLACE TRIGGER "person_delete_audit" + AFTER DELETE ON "person" + REFERENCING OLD TABLE AS "old" + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION person_delete_audit();`.execute(db); + await sql`INSERT INTO "migration_overrides" ("name", "value") VALUES ('function_person_delete_audit', '{"type":"function","name":"person_delete_audit","sql":"CREATE OR REPLACE FUNCTION person_delete_audit()\\n RETURNS TRIGGER\\n LANGUAGE PLPGSQL\\n AS $$\\n BEGIN\\n INSERT INTO person_audit (\\"personId\\", \\"ownerId\\")\\n SELECT \\"id\\", \\"ownerId\\"\\n FROM OLD;\\n RETURN NULL;\\n END\\n $$;"}'::jsonb);`.execute(db); + await sql`INSERT INTO "migration_overrides" ("name", "value") VALUES ('trigger_person_delete_audit', '{"type":"trigger","name":"person_delete_audit","sql":"CREATE OR REPLACE TRIGGER \\"person_delete_audit\\"\\n AFTER DELETE ON \\"person\\"\\n REFERENCING OLD TABLE AS \\"old\\"\\n FOR EACH STATEMENT\\n WHEN (pg_trigger_depth() = 0)\\n EXECUTE FUNCTION person_delete_audit();"}'::jsonb);`.execute(db); +} + +export async function down(db: Kysely): Promise { + await sql`DROP TRIGGER "person_delete_audit" ON "person";`.execute(db); + await sql`DROP TABLE "person_audit";`.execute(db); + await sql`DROP FUNCTION person_delete_audit;`.execute(db); + await sql`DELETE FROM "migration_overrides" WHERE "name" = 'function_person_delete_audit';`.execute(db); + await sql`DELETE FROM "migration_overrides" WHERE "name" = 'trigger_person_delete_audit';`.execute(db); +} diff --git a/server/src/schema/tables/person-audit.table.ts b/server/src/schema/tables/person-audit.table.ts new file mode 100644 index 000000000..f5790629d --- /dev/null +++ b/server/src/schema/tables/person-audit.table.ts @@ -0,0 +1,17 @@ +import { PrimaryGeneratedUuidV7Column } from 'src/decorators'; +import { Column, CreateDateColumn, Generated, Table, Timestamp } from 'src/sql-tools'; + +@Table('person_audit') +export class PersonAuditTable { + @PrimaryGeneratedUuidV7Column() + id!: Generated; + + @Column({ type: 'uuid', indexName: 'IDX_person_audit_person_id' }) + personId!: string; + + @Column({ type: 'uuid', indexName: 'IDX_person_audit_owner_id' }) + ownerId!: string; + + @CreateDateColumn({ default: () => 'clock_timestamp()', indexName: 'IDX_person_audit_deleted_at' }) + deletedAt!: Generated; +} diff --git a/server/src/schema/tables/person.table.ts b/server/src/schema/tables/person.table.ts index 5835b2528..193ecb892 100644 --- a/server/src/schema/tables/person.table.ts +++ b/server/src/schema/tables/person.table.ts @@ -1,7 +1,9 @@ import { UpdatedAtTrigger, UpdateIdColumn } from 'src/decorators'; +import { person_delete_audit } from 'src/schema/functions'; import { AssetFaceTable } from 'src/schema/tables/asset-face.table'; import { UserTable } from 'src/schema/tables/user.table'; import { + AfterDeleteTrigger, Check, Column, CreateDateColumn, @@ -15,6 +17,12 @@ import { @Table('person') @UpdatedAtTrigger('person_updated_at') +@AfterDeleteTrigger({ + scope: 'statement', + function: person_delete_audit, + referencingOldTableAs: 'old', + when: 'pg_trigger_depth() = 0', +}) @Check({ name: 'CHK_b0f82b0ed662bfc24fbb58bb45', expression: `"birthDate" <= CURRENT_DATE` }) export class PersonTable { @PrimaryGeneratedColumn('uuid') diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index e3322de2e..5e9c679d7 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -69,6 +69,7 @@ export const SYNC_TYPES_ORDER = [ SyncRequestType.PartnerAssetExifsV1, SyncRequestType.MemoriesV1, SyncRequestType.MemoryToAssetsV1, + SyncRequestType.PeopleV1, ]; const throwSessionRequired = () => { @@ -141,6 +142,7 @@ export class SyncService extends BaseService { [SyncRequestType.MemoryToAssetsV1]: () => this.syncMemoryAssetsV1(response, checkpointMap, auth), [SyncRequestType.StacksV1]: () => this.syncStackV1(response, checkpointMap, auth), [SyncRequestType.PartnerStacksV1]: () => this.syncPartnerStackV1(response, checkpointMap, auth, sessionId), + [SyncRequestType.PeopleV1]: () => this.syncPeopleV1(response, checkpointMap, auth), }; for (const type of SYNC_TYPES_ORDER.filter((type) => dto.types.includes(type))) { @@ -488,7 +490,7 @@ export class SyncService extends BaseService { private async syncMemoriesV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { const deleteType = SyncEntityType.MemoryDeleteV1; - const deletes = this.syncRepository.memory.getDeletes(auth.user.id, checkpointMap[SyncEntityType.MemoryDeleteV1]); + const deletes = this.syncRepository.memory.getDeletes(auth.user.id, checkpointMap[deleteType]); for await (const { id, ...data } of deletes) { send(response, { type: deleteType, ids: [id], data }); } @@ -576,6 +578,20 @@ export class SyncService extends BaseService { } } + private async syncPeopleV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto) { + const deleteType = SyncEntityType.PersonDeleteV1; + const deletes = this.syncRepository.people.getDeletes(auth.user.id, checkpointMap[deleteType]); + for await (const { id, ...data } of deletes) { + send(response, { type: deleteType, ids: [id], data }); + } + + const upsertType = SyncEntityType.PersonV1; + const upserts = this.syncRepository.people.getUpserts(auth.user.id, checkpointMap[upsertType]); + for await (const { updateId, ...data } of upserts) { + send(response, { type: upsertType, ids: [updateId], data }); + } + } + private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) { const { type, sessionId, createId } = item; await this.syncCheckpointRepository.upsertAll([ diff --git a/server/test/medium/specs/sync/sync-person.spec.ts b/server/test/medium/specs/sync/sync-person.spec.ts new file mode 100644 index 000000000..807e41894 --- /dev/null +++ b/server/test/medium/specs/sync/sync-person.spec.ts @@ -0,0 +1,87 @@ +import { Kysely } from 'kysely'; +import { SyncEntityType, SyncRequestType } from 'src/enum'; +import { PersonRepository } from 'src/repositories/person.repository'; +import { DB } from 'src/schema'; +import { SyncTestContext } from 'test/medium.factory'; +import { factory } from 'test/small.factory'; +import { getKyselyDB } from 'test/utils'; + +let defaultDatabase: Kysely; + +const setup = async (db?: Kysely) => { + const ctx = new SyncTestContext(db || defaultDatabase); + const { auth, user, session } = await ctx.newSyncAuthUser(); + return { auth, user, session, ctx }; +}; + +beforeAll(async () => { + defaultDatabase = await getKyselyDB(); +}); + +describe(SyncEntityType.PersonV1, () => { + it('should detect and sync the first person', async () => { + const { auth, ctx } = await setup(); + const { person } = await ctx.newPerson({ ownerId: auth.user.id }); + + const response = await ctx.syncStream(auth, [SyncRequestType.PeopleV1]); + expect(response).toHaveLength(1); + expect(response).toEqual([ + { + ack: expect.any(String), + data: expect.objectContaining({ + id: person.id, + name: person.name, + thumbnailPath: person.thumbnailPath, + isHidden: person.isHidden, + birthDate: person.birthDate, + faceAssetId: person.faceAssetId, + isFavorite: person.isFavorite, + ownerId: auth.user.id, + color: person.color, + }), + type: 'PersonV1', + }, + ]); + + await ctx.syncAckAll(auth, response); + await expect(ctx.syncStream(auth, [SyncRequestType.PeopleV1])).resolves.toEqual([]); + }); + + it('should detect and sync a deleted person', async () => { + const { auth, ctx } = await setup(); + const personRepo = ctx.get(PersonRepository); + const { person } = await ctx.newPerson({ ownerId: auth.user.id }); + await personRepo.delete([person.id]); + + const response = await ctx.syncStream(auth, [SyncRequestType.PeopleV1]); + expect(response).toHaveLength(1); + expect(response).toEqual([ + { + ack: expect.any(String), + data: { + personId: person.id, + }, + type: 'PersonDeleteV1', + }, + ]); + + await ctx.syncAckAll(auth, response); + await expect(ctx.syncStream(auth, [SyncRequestType.PeopleV1])).resolves.toEqual([]); + }); + + it('should not sync a person or person delete for an unrelated user', async () => { + const { auth, ctx } = await setup(); + const personRepo = ctx.get(PersonRepository); + const { user: user2 } = await ctx.newUser(); + const { session } = await ctx.newSession({ userId: user2.id }); + const { person } = await ctx.newPerson({ ownerId: user2.id }); + const auth2 = factory.auth({ session, user: user2 }); + + expect(await ctx.syncStream(auth2, [SyncRequestType.PeopleV1])).toHaveLength(1); + expect(await ctx.syncStream(auth, [SyncRequestType.PeopleV1])).toHaveLength(0); + + await personRepo.delete([person.id]); + expect(await ctx.syncStream(auth2, [SyncRequestType.PeopleV1])).toHaveLength(1); + expect(await ctx.syncStream(auth, [SyncRequestType.PeopleV1])).toHaveLength(0); + }); +}); diff --git a/server/tsconfig.json b/server/tsconfig.json index 8d8d12c54..e12b614f0 100644 --- a/server/tsconfig.json +++ b/server/tsconfig.json @@ -17,6 +17,9 @@ "skipLibCheck": true, "esModuleInterop": true, "preserveWatchOutput": true, + "paths": { + "src/*": ["./src/*"], + }, "baseUrl": "./", "jsx": "react", "types": ["vitest/globals"],