From a96bba4b26a4cec2f60f1680011e40265fa04f5b Mon Sep 17 00:00:00 2001 From: Jason Rasmussen Date: Mon, 10 Mar 2025 12:05:39 -0400 Subject: [PATCH] feat: sync assets, partner assets, exif, and partner exif (#16658) * feat: sync assets, partner assets, exif, and partner exif Co-authored-by: Zack Pollard Co-authored-by: Alex Tran * refactor: remove duplicate where clause and orderBy statements in sync queries * fix: asset deletes not filtering by ownerId --------- Co-authored-by: Zack Pollard Co-authored-by: Alex Tran Co-authored-by: Zack Pollard --- mobile/openapi/README.md | Bin 33479 -> 33613 bytes mobile/openapi/lib/api.dart | Bin 12214 -> 12325 bytes mobile/openapi/lib/api_client.dart | Bin 31188 -> 31430 bytes .../lib/model/sync_asset_delete_v1.dart | Bin 0 -> 2863 bytes .../openapi/lib/model/sync_asset_exif_v1.dart | Bin 0 -> 11859 bytes mobile/openapi/lib/model/sync_asset_v1.dart | Bin 0 -> 8777 bytes .../openapi/lib/model/sync_entity_type.dart | Bin 2977 -> 3926 bytes .../openapi/lib/model/sync_request_type.dart | Bin 2698 -> 3332 bytes open-api/immich-openapi-specs.json | 224 ++++++++- open-api/typescript-sdk/src/fetch-client.ts | 14 +- server/src/database.ts | 42 ++ server/src/db.d.ts | 10 + server/src/dtos/asset-response.dto.ts | 2 +- server/src/dtos/sync.dto.ts | 67 ++- server/src/entities/asset-audit.entity.ts | 19 + server/src/entities/exif.entity.ts | 9 +- server/src/enum.ts | 13 + .../1741191762113-AssetAuditTable.ts | 37 ++ ...328985-FixAssetAndUserCascadeConditions.ts | 50 ++ .../1741281344519-AddExifUpdateId.ts | 25 + server/src/queries/asset.repository.sql | 8 +- server/src/repositories/asset.repository.ts | 10 +- server/src/repositories/sync.repository.ts | 107 ++++- server/src/services/sync.service.ts | 100 +++- server/test/factory.ts | 12 +- .../test/medium/specs/audit.database.spec.ts | 74 +++ server/test/medium/specs/sync.service.spec.ts | 447 +++++++++++++++++- .../test/repositories/sync.repository.mock.ts | 6 + 28 files changed, 1230 insertions(+), 46 deletions(-) create mode 100644 mobile/openapi/lib/model/sync_asset_delete_v1.dart create mode 100644 mobile/openapi/lib/model/sync_asset_exif_v1.dart create mode 100644 mobile/openapi/lib/model/sync_asset_v1.dart create mode 100644 server/src/entities/asset-audit.entity.ts create mode 100644 server/src/migrations/1741191762113-AssetAuditTable.ts create mode 100644 server/src/migrations/1741280328985-FixAssetAndUserCascadeConditions.ts create mode 100644 server/src/migrations/1741281344519-AddExifUpdateId.ts create mode 100644 server/test/medium/specs/audit.database.spec.ts diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index e3f6a74856a90deab0ad8e7efe32bb8aa9c4aa15..85409a7934920082eb234be7c6ec7dd88dad4f39 100644 GIT binary patch delta 142 zcmX@!%5=7kX@gZ^y<>54YKco~PHIVNm|?6&N`A7wesE=8GK#ofZi*(Cg04a|T*kE` XGYwNcL=;msrphowh^o!Tg;N9pF6S`> delta 14 VcmX@x#&o=uX@gba<^zSjf&enK27v$o diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 04dc43f88c7b5166047b15deef878fb4b0be635f..eddd63a7326796e257cc7ce330a24ffa0b5248d7 100644 GIT binary patch delta 55 zcmdlMzcgV3vm$F^adB$NTyo$)ZM5HHeRoGn zmM3S;79fd49`EIOo;zwd91MqW_4{o4^0(`|>yKBn>j~W4zQ2xPGKIV83_eUJw>N*B zpcz@d$%S#FpOROv2lT2|r8Y8MYLhNiAupgVYb*0q7IGs?7dP)}Q)%Olo)E=W>}*=q zE;aJ6S{XcEiVgm)gu(w#8jH@2J$^jXrFEiosmd{+S}JMyxIG%IP^GkKq01GTxwRkbYExPe zP|7XLdxi^8oM0vsJki^Z$PYkMIk>g52}YFeHji=Z@f>r&bIz!81!04i-srE=7_1t% z@3k%{ZzG&GFd9YE`!6!XYTR0ya~O@mqmzjuq6l%b^X;2|0(Gp$(^F24y);Ro_b1a( zrGu4puoiaxtIj0`W(TcMNFSxlq_x8Ah+m{^Ev>R_7KQSX=De@HTWG++Y5x#$ZD&1H z=fTtHhw!jCW6VB?>Sup4hJ1%AQ`s1tert7!t~?b>Rf+<>;1Z0WXZ`Sa$+-es%*|N{ zaJE3}E_vR}usYb579NSq|E3xcWe&c5EGTE;t0-z|VQKUxIGghl3T=r7XC%cr7P^R` ztP7L{X<(}?>#xH^W#kf@fcwYP84AayB&ihJ$*C;G&Rgr8nDGmJ?f$|^Fn9q>pIn1v zU9F7B<;3ZTGmPOykEs{PF*N|@?u|%^n0?~|;D8VIlVrlW9y5_wi|;1t$$r8Sha!u> z+68Px^_c*jE@!(+2B+hQ=#OE~?)eEdBmIPYANe)VOE|am2DIKQ%JH9LV+;kZ-ynAf zu=#Kg+{P=Tx8lCQ=7FUstc)KUEIblM+PZM8&7Sx9Vvi>IBrquvI6Y+zCd%GvcfG0H zu5pS0$mj980ZpEUERkVNFUXTEdN zqf7lleh#n>r<|s`q6VxBXr9;$_gNsK+w;BD$0UHGbTqkoV9<&Y=P?}jAs9Dvfog7$ z$H@N*Jll;tF;7Vvr$xAAFQB~-QymZ6*$$3B`u9{s#V?u9A+NK=B)0gxNI@wdg zh8=RgLn}fx_Xq!!JDQ4o-s^JDX2a?Mw|KN|Bdi-=X_zrg+Fxi}<<_`FO+;69kI$Fh z4v&RFiPIhJ>7#|*p`4DWf|PWmRaqOfcT^uSJg;3bYF9Z=#ri+`d8Ra;BeO8vI-}sb zD#F4JdG3*s5KS$39Wljix9~erp+xu439^{Tq)Je6)Qi7+ zcNbV<0fMBPhdUFyzx_jCcd;OJe0*?xtls^2bN$udFRm_rcz1JguHIjMxR|K(Yjt&f zqds1rU%vm>D}W*L+htbOvp-MYygk5S{!o;ap0CPkzREZHMD2>5(aX8s=to`D6CUS} z+p?;Y64~Y{+nf1zSI;Z`pIu&oc#%EA|J$sBzqPImD6dRfd{Gvr&Wd`TFTr5G%5_EJ zCTVPw7uw7>W$^&8dYu*Xx3A}mbyk^!129*sa;3_xF0yStg@4~194t1OG3wL4SiCbv z*B8I#tDoMe&j(5!!~6fB>s?hCm8n8MsoMy4r*d*`p{Ib4m}AtT6u+~o*SVR3EnQzO zCkfUfulGrMxhv`_$y#Q$zR4f;b(KHlMYc(aFSS`z`L@o>BE{D5zL$^LgZ_ukAJ%m* zQ7_;kQxJl96m%bjg19w)01@X09A`wa zQ*@@9024S4s{n}c6cR9yh~lTs$4~$j!4uHiQ4_O8xDbn`+6GAgrRWGK92P|Uw)|CB z6;x{oj{CCQV7&z`Wu<1b@%24gQDwc>)sdQ=DqGbd!^c=3C(QZ%cl)IBc5VLhORAov zg9|F?<`7f?A7c=i^f%<#U?iq{4ES+aBCtgZC45E_SPTPP6uyX!NohPqbpW@N2<-#i zV#PQ>y5L6;KZg7u7YG(oLbT5if{I~?+Zy7-pMVbegLM28&^eWm**u2`F5CoZ1|fzM zjWEq1z+l=ToBoCz%s*n$L(Frv+yQtLphQ|COb!EFJ53RupvF=nd!2-&1ZbPnh;mN> zm(m-V_95VLq9dE1TS*UfLp|tPtuwR!8_XA^IVEh^2*9re3QPO6yk%q|a>$w$uc^mDlioLNI2hG>Bx6ggqxA7bkuw z8*%&Oo7y1!Zp4Y1loquiHtksnNiGyb; z#JLMJPDvp0BTzZ%fXa^(Y7u55%rJPyesfuv@th5r2f4Q8a?6LVfo>+;8X zF+PI7m!^FC_3JlW9r_)P6_JT<&wb8!JMA&TUr!wH#{>Iai3&`oI}y&i6fyeUioh>) z&30j3tJ~-#Si&))ztAtQqjT|OP3@U3UwEZUo;u`^NA}MW_l#ZBVW#xkW>j*b zE4uNeQw3^y#yhV6=O$QX#ZuYpaHI0ts9&?qP8+qV%15W_mPl}UyaK=>UM}ClA?$8q zKMT;k##3(SPFZ|2-Bx-9m)dk6UhNiepj-LRZH_zBVYXhk~C-9=~PX^c8V_Yb8Semq{TS0y1K;TV)J5Q= z9y)EjP>nsAh!>LQkd_X6YL(uI^Mp|1`uvuB@G5*5ka9skJ|G@otvXulAn0!i*8TcYvGG~H|<&OKmDJXo0K+nJQbUaln;Tcw6i zyeTzs!&20=g<`HzBH30}ibg8VN72~~+7(#&rtRiqP*YnZQYc!6mnai(smN~rEd{~i zxduVZCT-s>p?K-+CNIQz=N*(}p!d}6Jwq%I-ICS*cbTd|;eB^R8z1v`U-Zrlzl-k?z|&zbm3f+!WCXxW%R} zVw#()vc7oS)_Yb78zuZ4V(c`RPw+D`f_gr0rFNU0#e4Y_y9s(1aNErcZg_rYZgpzN zwl6WwmP@;T3rRb#*zX`Pm}Mtbm-tp=o~RSWPozaIf&8M}S_#<3lMBJ`ZYKp$@=J59 zg}*rC`|_NF)!r#5+{`XXS5pF=p8Lt;iE4hGIENMleg}!aceFRu!SVsM+VcWVag=Mr zFN1SwUubY4NzasEVp#?j6sGCc3jU(^__dwnZ9MvQ6+`KR(UJ_oesTIjT7(MV>;R;8~FvaC*CQby=q{&a8sgowM5@`;WL_i#?o3(D% zzWEU=gqeXWoeb7^2Vrrq;r2=!FpC+Gsqa2rU*S(--O|%prTGQy_bA(3ADw=t*=YK9 z+khWZH_p$l!*;`V-)uF=7T7`Kjy5LS4z;0*hSSlwkK|$47cy0}dQ5^Y&OKs-<{q22 zi^tq@H2`j+m_SXBL)$M_e!$;*97T2iquz DbJd`a literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/sync_asset_v1.dart b/mobile/openapi/lib/model/sync_asset_v1.dart new file mode 100644 index 0000000000000000000000000000000000000000..6f9d7d7eafedb15520078dbb9e711eb464ab7fd3 GIT binary patch literal 8777 zcmd5?ZExGi5&rI9u|60l8P&;2KNNwlvd=gxcS_H8kvIVgK@eziW$_}B>XM2e}k4G@K#otfR8_ZjBw?C9)FTz$O1`{9@2?eP88{qS169sM@!iR(LYdv`Bx z?yg5~|8Wd7!u%;qR6Y1}|MiTn`B!}SL=GJL6fC}*Nv$s$?hef<6L=qOzzRV6-bi}b3hWc}&2 z_p4mDjR;^;2Hu4~y2tMVTgxYaHo5U$!EY zn?kBl>$dfLy?mG_)x2%jdb?tNPp41{gj%b-cobH4471x_3MaN5cqFlwH7~4ilRV?M zhq7DUu>byv*u~MTehtk-8PqE zb;jq2)>@8Xl-y+MbsIBu@D^s|+n!Uy%T2VaOa0Jg$L<7)Wo(r@II&MXufD;F({L|# zzQIYPE4}Kaub2IM556`yLDC4~Al{va5!?uyz~-?RyUpMv5-lQk(^U(9|F*tEj z0~A660KbzCi|8g<{Z&f@m))Xh$kPi^m+z{wfQWRio#jQc5WfQ?2BLxe_31};0l^q! z)v?m+On@GxXN{Q81jo_7lc+e>K4Uj@tQ>~t%y8KwD-BbKKs>(NidG*?=R`2s9 ziuP_5bhx&*y0@|dfGcXrmo4n!{B1rKoC1>bexb0Ev6jfg} zrFw9RE~w7aDH^7EnNiBufUvl`AMhFV8 z>yvPUl6nL$ERVQDZ-UOVhHJUO=z>xGy7g=A^Mg=QUP3sokp}sm4-t*xh?e!?`$?ARKoU@DSyHei z#7MG+sxIk9T`Qn-VV{^wDE6>CjHHin(kgs=XDtk1g(YLi!Iq75Pb+8ac2LZWH_&R> z7Zs6S%t-BAxarpv11QYdXr87!xkzPzs;yO8pWbp=*^7`2^<@q!k9qVNB~ZmPN`!dE zn((vSQnk$V+Wx%mtPUehf;32|6x}KNv4BcFzS!jT93XNN8p#E+|3PDbz~odE{J^BK zBr$^77^k}lnggDeq=-RLVTq=G^RqG;?YO$pOtd1*iQR>0jgVmhis{OIQDmmyT%eiPN<^$r_lME7ZQJ-j*XZ;0%4UnX?Tbs zCExIcgGIj>V^<+qR@w^r`udS?q#yZ^^oV)`zeb=f4EJ{&m`u0e(?RZ4QdEdpOQ|p) z$z+OIr>hhEZoARe-QmAfTsi*#>mti;{Qp>NAue%Huqs0fimFq8Gvm;mCWU}`k`5>6 ziD9xZmLip8(yazd>0En#k6BY6J=R7ru|qy&`#KR1N&1L>fq9MoZY{H+ zQl-+DoTIlXVo^{M+>r&&xy7fMBHM}$-3A=5{yxGCBf+<3h#pc3QZkF5M2H>;`4}CT zl1q$n_`4`cgz)htT>(Q$1lSzme_c_=1>;xbeqf@r%kv@oW6tyZMwc8@s zsT4_x6zUpHs{h&18a-TviXxrs)v(n-TO%<{(W&;_D6;K9eMX@YQ2#N6I@cZ~WRwP> zPPPXL8Kps}v+Y4bM)8E1oF(YI&)wN|w|nfw7z%`LSCjpSEA_ah4vy`r+xb|0E?`x< zVA95bNT#)PE>1}7x^^%EGM#X1Hx*4B$aX%!Oz__Z5Qy7C<4*X4jhm1hlnO02v_p33 JL!hos{{>WC^gI9n literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/sync_entity_type.dart b/mobile/openapi/lib/model/sync_entity_type.dart index 5d130f7f93ba82f871cdad82f2c0d761842412b0..5e52a10e7a14b218e765ad137615e82af5bac134 100644 GIT binary patch delta 565 zcmZ1|eobzJJoDtaO#C*9#l@*5VTKB}3c;0m$*y@NnI)AWl?ADK@ft5HE)r>HvcR%5h1}Ni9i5 z5rUWtQG@Ihm?C7c$qSf8IpMC^e2V!L+hhSQc4&wua7oL8Lp00~6#5X5oLHQy0F9T) zQCy0wU=@@9aY`Z#nhey*2^I%C-hoRBZlV&Dg%FG4(twKn<5WN~LkYp#+`%Qu2moms By+;55 delta 21 dcmca6w@`e8JoDt;EE_jJWoYei-n zNFgU!A($7%B+Clsfy8+N5{pXmQi~u0Kq(}JKxqMFX|V1YOjF$zAPxXps{^zY$^*&5 zMGBBi#VQC=y;+Spl#M4ju{c$sNFCw;K2BMsM5vcQ9)bX5C16JYm2rZVfqD9z(ohcp Z#d(meo(z;i@(@s309hLBh&)aiMgYLabd&%9 delta 21 dcmZpX>Jr^BpK0=D=DyASEPia8zjE_40svdY2g3jW diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index c1921da82..d503e565d 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -12049,12 +12049,228 @@ ], "type": "object" }, + "SyncAssetDeleteV1": { + "properties": { + "assetId": { + "type": "string" + } + }, + "required": [ + "assetId" + ], + "type": "object" + }, + "SyncAssetExifV1": { + "properties": { + "assetId": { + "type": "string" + }, + "city": { + "nullable": true, + "type": "string" + }, + "country": { + "nullable": true, + "type": "string" + }, + "dateTimeOriginal": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "description": { + "nullable": true, + "type": "string" + }, + "exifImageHeight": { + "nullable": true, + "type": "integer" + }, + "exifImageWidth": { + "nullable": true, + "type": "integer" + }, + "exposureTime": { + "nullable": true, + "type": "string" + }, + "fNumber": { + "nullable": true, + "type": "integer" + }, + "fileSizeInByte": { + "nullable": true, + "type": "integer" + }, + "focalLength": { + "nullable": true, + "type": "integer" + }, + "fps": { + "nullable": true, + "type": "integer" + }, + "iso": { + "nullable": true, + "type": "integer" + }, + "latitude": { + "nullable": true, + "type": "integer" + }, + "lensModel": { + "nullable": true, + "type": "string" + }, + "longitude": { + "nullable": true, + "type": "integer" + }, + "make": { + "nullable": true, + "type": "string" + }, + "model": { + "nullable": true, + "type": "string" + }, + "modifyDate": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "orientation": { + "nullable": true, + "type": "string" + }, + "profileDescription": { + "nullable": true, + "type": "string" + }, + "projectionType": { + "nullable": true, + "type": "string" + }, + "rating": { + "nullable": true, + "type": "integer" + }, + "state": { + "nullable": true, + "type": "string" + }, + "timeZone": { + "nullable": true, + "type": "string" + } + }, + "required": [ + "assetId", + "city", + "country", + "dateTimeOriginal", + "description", + "exifImageHeight", + "exifImageWidth", + "exposureTime", + "fNumber", + "fileSizeInByte", + "focalLength", + "fps", + "iso", + "latitude", + "lensModel", + "longitude", + "make", + "model", + "modifyDate", + "orientation", + "profileDescription", + "projectionType", + "rating", + "state", + "timeZone" + ], + "type": "object" + }, + "SyncAssetV1": { + "properties": { + "checksum": { + "type": "string" + }, + "deletedAt": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "fileCreatedAt": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "fileModifiedAt": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "id": { + "type": "string" + }, + "isFavorite": { + "type": "boolean" + }, + "isVisible": { + "type": "boolean" + }, + "localDateTime": { + "format": "date-time", + "nullable": true, + "type": "string" + }, + "ownerId": { + "type": "string" + }, + "thumbhash": { + "nullable": true, + "type": "string" + }, + "type": { + "enum": [ + "IMAGE", + "VIDEO", + "AUDIO", + "OTHER" + ], + "type": "string" + } + }, + "required": [ + "checksum", + "deletedAt", + "fileCreatedAt", + "fileModifiedAt", + "id", + "isFavorite", + "isVisible", + "localDateTime", + "ownerId", + "thumbhash", + "type" + ], + "type": "object" + }, "SyncEntityType": { "enum": [ "UserV1", "UserDeleteV1", "PartnerV1", - "PartnerDeleteV1" + "PartnerDeleteV1", + "AssetV1", + "AssetDeleteV1", + "AssetExifV1", + "PartnerAssetV1", + "PartnerAssetDeleteV1", + "PartnerAssetExifV1" ], "type": "string" }, @@ -12095,7 +12311,11 @@ "SyncRequestType": { "enum": [ "UsersV1", - "PartnersV1" + "PartnersV1", + "AssetsV1", + "AssetExifsV1", + "PartnerAssetsV1", + "PartnerAssetExifsV1" ], "type": "string" }, diff --git a/open-api/typescript-sdk/src/fetch-client.ts b/open-api/typescript-sdk/src/fetch-client.ts index fb70a42e8..85f80eec9 100644 --- a/open-api/typescript-sdk/src/fetch-client.ts +++ b/open-api/typescript-sdk/src/fetch-client.ts @@ -3647,11 +3647,21 @@ export enum SyncEntityType { UserV1 = "UserV1", UserDeleteV1 = "UserDeleteV1", PartnerV1 = "PartnerV1", - PartnerDeleteV1 = "PartnerDeleteV1" + PartnerDeleteV1 = "PartnerDeleteV1", + AssetV1 = "AssetV1", + AssetDeleteV1 = "AssetDeleteV1", + AssetExifV1 = "AssetExifV1", + PartnerAssetV1 = "PartnerAssetV1", + PartnerAssetDeleteV1 = "PartnerAssetDeleteV1", + PartnerAssetExifV1 = "PartnerAssetExifV1" } export enum SyncRequestType { UsersV1 = "UsersV1", - PartnersV1 = "PartnersV1" + PartnersV1 = "PartnersV1", + AssetsV1 = "AssetsV1", + AssetExifsV1 = "AssetExifsV1", + PartnerAssetsV1 = "PartnerAssetsV1", + PartnerAssetExifsV1 = "PartnerAssetExifsV1" } export enum TranscodeHWAccel { Nvenc = "nvenc", diff --git a/server/src/database.ts b/server/src/database.ts index 08ed7240d..e89920057 100644 --- a/server/src/database.ts +++ b/server/src/database.ts @@ -117,4 +117,46 @@ export const columns = { userDto: ['id', 'name', 'email', 'profileImagePath', 'profileChangedAt'], tagDto: ['id', 'value', 'createdAt', 'updatedAt', 'color', 'parentId'], apiKey: ['id', 'name', 'userId', 'createdAt', 'updatedAt', 'permissions'], + syncAsset: [ + 'id', + 'ownerId', + 'thumbhash', + 'checksum', + 'fileCreatedAt', + 'fileModifiedAt', + 'localDateTime', + 'type', + 'deletedAt', + 'isFavorite', + 'isVisible', + 'updateId', + ], + syncAssetExif: [ + 'exif.assetId', + 'exif.description', + 'exif.exifImageWidth', + 'exif.exifImageHeight', + 'exif.fileSizeInByte', + 'exif.orientation', + 'exif.dateTimeOriginal', + 'exif.modifyDate', + 'exif.timeZone', + 'exif.latitude', + 'exif.longitude', + 'exif.projectionType', + 'exif.city', + 'exif.state', + 'exif.country', + 'exif.make', + 'exif.model', + 'exif.lensModel', + 'exif.fNumber', + 'exif.focalLength', + 'exif.iso', + 'exif.exposureTime', + 'exif.profileDescription', + 'exif.rating', + 'exif.fps', + 'exif.updateId', + ], } as const; diff --git a/server/src/db.d.ts b/server/src/db.d.ts index a27faac9b..85aade2c9 100644 --- a/server/src/db.d.ts +++ b/server/src/db.d.ts @@ -119,6 +119,13 @@ export interface AssetJobStatus { thumbnailAt: Timestamp | null; } +export interface AssetsAudit { + deletedAt: Generated; + id: Generated; + assetId: string; + ownerId: string; +} + export interface Assets { checksum: Buffer; createdAt: Generated; @@ -168,6 +175,8 @@ export interface Audit { export interface Exif { assetId: string; + updateId: Generated; + updatedAt: Generated; autoStackId: string | null; bitsPerSample: number | null; city: string | null; @@ -459,6 +468,7 @@ export interface DB { asset_job_status: AssetJobStatus; asset_stack: AssetStack; assets: Assets; + assets_audit: AssetsAudit; audit: Audit; exif: Exif; face_search: FaceSearch; diff --git a/server/src/dtos/asset-response.dto.ts b/server/src/dtos/asset-response.dto.ts index 9a963e1e9..b12a4378f 100644 --- a/server/src/dtos/asset-response.dto.ts +++ b/server/src/dtos/asset-response.dto.ts @@ -102,7 +102,7 @@ const mapStack = (entity: AssetEntity) => { }; // if an asset is jsonified in the DB before being returned, its buffer fields will be hex-encoded strings -const hexOrBufferToBase64 = (encoded: string | Buffer) => { +export const hexOrBufferToBase64 = (encoded: string | Buffer) => { if (typeof encoded === 'string') { return Buffer.from(encoded.slice(2), 'hex').toString('base64'); } diff --git a/server/src/dtos/sync.dto.ts b/server/src/dtos/sync.dto.ts index d191c82bb..a035f8ecb 100644 --- a/server/src/dtos/sync.dto.ts +++ b/server/src/dtos/sync.dto.ts @@ -1,7 +1,7 @@ import { ApiProperty } from '@nestjs/swagger'; import { IsEnum, IsInt, IsPositive, IsString } from 'class-validator'; import { AssetResponseDto } from 'src/dtos/asset-response.dto'; -import { SyncEntityType, SyncRequestType } from 'src/enum'; +import { AssetType, SyncEntityType, SyncRequestType } from 'src/enum'; import { Optional, ValidateDate, ValidateUUID } from 'src/validation'; export class AssetFullSyncDto { @@ -56,11 +56,73 @@ export class SyncPartnerDeleteV1 { sharedWithId!: string; } +export class SyncAssetV1 { + id!: string; + ownerId!: string; + thumbhash!: string | null; + checksum!: string; + fileCreatedAt!: Date | null; + fileModifiedAt!: Date | null; + localDateTime!: Date | null; + type!: AssetType; + deletedAt!: Date | null; + isFavorite!: boolean; + isVisible!: boolean; +} + +export class SyncAssetDeleteV1 { + assetId!: string; +} + +export class SyncAssetExifV1 { + assetId!: string; + description!: string | null; + @ApiProperty({ type: 'integer' }) + exifImageWidth!: number | null; + @ApiProperty({ type: 'integer' }) + exifImageHeight!: number | null; + @ApiProperty({ type: 'integer' }) + fileSizeInByte!: number | null; + orientation!: string | null; + dateTimeOriginal!: Date | null; + modifyDate!: Date | null; + timeZone!: string | null; + @ApiProperty({ type: 'integer' }) + latitude!: number | null; + @ApiProperty({ type: 'integer' }) + longitude!: number | null; + projectionType!: string | null; + city!: string | null; + state!: string | null; + country!: string | null; + make!: string | null; + model!: string | null; + lensModel!: string | null; + @ApiProperty({ type: 'integer' }) + fNumber!: number | null; + @ApiProperty({ type: 'integer' }) + focalLength!: number | null; + @ApiProperty({ type: 'integer' }) + iso!: number | null; + exposureTime!: string | null; + profileDescription!: string | null; + @ApiProperty({ type: 'integer' }) + rating!: number | null; + @ApiProperty({ type: 'integer' }) + fps!: number | null; +} + export type SyncItem = { [SyncEntityType.UserV1]: SyncUserV1; [SyncEntityType.UserDeleteV1]: SyncUserDeleteV1; [SyncEntityType.PartnerV1]: SyncPartnerV1; [SyncEntityType.PartnerDeleteV1]: SyncPartnerDeleteV1; + [SyncEntityType.AssetV1]: SyncAssetV1; + [SyncEntityType.AssetDeleteV1]: SyncAssetDeleteV1; + [SyncEntityType.AssetExifV1]: SyncAssetExifV1; + [SyncEntityType.PartnerAssetV1]: SyncAssetV1; + [SyncEntityType.PartnerAssetDeleteV1]: SyncAssetDeleteV1; + [SyncEntityType.PartnerAssetExifV1]: SyncAssetExifV1; }; const responseDtos = [ @@ -69,6 +131,9 @@ const responseDtos = [ SyncUserDeleteV1, SyncPartnerV1, SyncPartnerDeleteV1, + SyncAssetV1, + SyncAssetDeleteV1, + SyncAssetExifV1, ]; export const extraSyncModels = responseDtos; diff --git a/server/src/entities/asset-audit.entity.ts b/server/src/entities/asset-audit.entity.ts new file mode 100644 index 000000000..0172d15ce --- /dev/null +++ b/server/src/entities/asset-audit.entity.ts @@ -0,0 +1,19 @@ +import { Column, CreateDateColumn, Entity, Index, PrimaryColumn } from 'typeorm'; + +@Entity('assets_audit') +export class AssetAuditEntity { + @PrimaryColumn({ type: 'uuid', nullable: false, default: () => 'immich_uuid_v7()' }) + id!: string; + + @Index('IDX_assets_audit_asset_id') + @Column({ type: 'uuid' }) + assetId!: string; + + @Index('IDX_assets_audit_owner_id') + @Column({ type: 'uuid' }) + ownerId!: string; + + @Index('IDX_assets_audit_deleted_at') + @CreateDateColumn({ type: 'timestamptz', default: () => 'clock_timestamp()' }) + deletedAt!: Date; +} diff --git a/server/src/entities/exif.entity.ts b/server/src/entities/exif.entity.ts index c9c29d732..5b402109a 100644 --- a/server/src/entities/exif.entity.ts +++ b/server/src/entities/exif.entity.ts @@ -1,5 +1,5 @@ import { AssetEntity } from 'src/entities/asset.entity'; -import { Index, JoinColumn, OneToOne, PrimaryColumn } from 'typeorm'; +import { Index, JoinColumn, OneToOne, PrimaryColumn, UpdateDateColumn } from 'typeorm'; import { Column } from 'typeorm/decorator/columns/Column.js'; import { Entity } from 'typeorm/decorator/entity/Entity.js'; @@ -12,6 +12,13 @@ export class ExifEntity { @PrimaryColumn() assetId!: string; + @UpdateDateColumn({ type: 'timestamptz', default: () => 'clock_timestamp()' }) + updatedAt?: Date; + + @Index('IDX_asset_exif_update_id') + @Column({ type: 'uuid', nullable: false, default: () => 'immich_uuid_v7()' }) + updateId?: string; + /* General info */ @Column({ type: 'text', default: '' }) description!: string; // or caption diff --git a/server/src/enum.ts b/server/src/enum.ts index 47154e325..55e435a70 100644 --- a/server/src/enum.ts +++ b/server/src/enum.ts @@ -549,11 +549,24 @@ export enum DatabaseLock { export enum SyncRequestType { UsersV1 = 'UsersV1', PartnersV1 = 'PartnersV1', + AssetsV1 = 'AssetsV1', + AssetExifsV1 = 'AssetExifsV1', + PartnerAssetsV1 = 'PartnerAssetsV1', + PartnerAssetExifsV1 = 'PartnerAssetExifsV1', } export enum SyncEntityType { UserV1 = 'UserV1', UserDeleteV1 = 'UserDeleteV1', + PartnerV1 = 'PartnerV1', PartnerDeleteV1 = 'PartnerDeleteV1', + + AssetV1 = 'AssetV1', + AssetDeleteV1 = 'AssetDeleteV1', + AssetExifV1 = 'AssetExifV1', + + PartnerAssetV1 = 'PartnerAssetV1', + PartnerAssetDeleteV1 = 'PartnerAssetDeleteV1', + PartnerAssetExifV1 = 'PartnerAssetExifV1', } diff --git a/server/src/migrations/1741191762113-AssetAuditTable.ts b/server/src/migrations/1741191762113-AssetAuditTable.ts new file mode 100644 index 000000000..c02408c38 --- /dev/null +++ b/server/src/migrations/1741191762113-AssetAuditTable.ts @@ -0,0 +1,37 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class AssetAuditTable1741191762113 implements MigrationInterface { + name = 'AssetAuditTable1741191762113' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE TABLE "assets_audit" ("id" uuid NOT NULL DEFAULT immich_uuid_v7(), "assetId" uuid NOT NULL, "ownerId" uuid NOT NULL, "deletedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp(), CONSTRAINT "PK_99bd5c015f81a641927a32b4212" PRIMARY KEY ("id"))`); + await queryRunner.query(`CREATE INDEX "IDX_assets_audit_asset_id" ON "assets_audit" ("assetId") `); + await queryRunner.query(`CREATE INDEX "IDX_assets_audit_owner_id" ON "assets_audit" ("ownerId") `); + await queryRunner.query(`CREATE INDEX "IDX_assets_audit_deleted_at" ON "assets_audit" ("deletedAt") `); + await queryRunner.query(`CREATE OR REPLACE FUNCTION assets_delete_audit() RETURNS TRIGGER AS + $$ + BEGIN + INSERT INTO assets_audit ("assetId", "ownerId") + SELECT "id", "ownerId" + FROM OLD; + RETURN NULL; + END; + $$ LANGUAGE plpgsql` + ); + await queryRunner.query(`CREATE OR REPLACE TRIGGER assets_delete_audit + AFTER DELETE ON assets + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION assets_delete_audit(); + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP TRIGGER assets_delete_audit`); + await queryRunner.query(`DROP FUNCTION assets_delete_audit`); + await queryRunner.query(`DROP INDEX "IDX_assets_audit_deleted_at"`); + await queryRunner.query(`DROP INDEX "IDX_assets_audit_owner_id"`); + await queryRunner.query(`DROP INDEX "IDX_assets_audit_asset_id"`); + await queryRunner.query(`DROP TABLE "assets_audit"`); + } +} diff --git a/server/src/migrations/1741280328985-FixAssetAndUserCascadeConditions.ts b/server/src/migrations/1741280328985-FixAssetAndUserCascadeConditions.ts new file mode 100644 index 000000000..20215c1b5 --- /dev/null +++ b/server/src/migrations/1741280328985-FixAssetAndUserCascadeConditions.ts @@ -0,0 +1,50 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class FixAssetAndUserCascadeConditions1741280328985 implements MigrationInterface { + name = 'FixAssetAndUserCascadeConditions1741280328985'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE OR REPLACE TRIGGER assets_delete_audit + AFTER DELETE ON assets + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION assets_delete_audit();`); + await queryRunner.query(` + CREATE OR REPLACE TRIGGER users_delete_audit + AFTER DELETE ON users + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION users_delete_audit();`); + await queryRunner.query(` + CREATE OR REPLACE TRIGGER partners_delete_audit + AFTER DELETE ON partners + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + WHEN (pg_trigger_depth() = 0) + EXECUTE FUNCTION partners_delete_audit();`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE OR REPLACE TRIGGER assets_delete_audit + AFTER DELETE ON assets + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION assets_delete_audit();`); + await queryRunner.query(` + CREATE OR REPLACE TRIGGER users_delete_audit + AFTER DELETE ON users + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION users_delete_audit();`); + await queryRunner.query(` + CREATE OR REPLACE TRIGGER partners_delete_audit + AFTER DELETE ON partners + REFERENCING OLD TABLE AS OLD + FOR EACH STATEMENT + EXECUTE FUNCTION partners_delete_audit();`); + } +} diff --git a/server/src/migrations/1741281344519-AddExifUpdateId.ts b/server/src/migrations/1741281344519-AddExifUpdateId.ts new file mode 100644 index 000000000..eb32836a1 --- /dev/null +++ b/server/src/migrations/1741281344519-AddExifUpdateId.ts @@ -0,0 +1,25 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class AddExifUpdateId1741281344519 implements MigrationInterface { + name = 'AddExifUpdateId1741281344519'; + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query( + `ALTER TABLE "exif" ADD "updatedAt" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT clock_timestamp()`, + ); + await queryRunner.query(`ALTER TABLE "exif" ADD "updateId" uuid NOT NULL DEFAULT immich_uuid_v7()`); + await queryRunner.query(`CREATE INDEX "IDX_asset_exif_update_id" ON "exif" ("updateId") `); + await queryRunner.query(` + create trigger asset_exif_updated_at + before update on exif + for each row execute procedure updated_at() + `); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`DROP INDEX "public"."IDX_asset_exif_update_id"`); + await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "updateId"`); + await queryRunner.query(`ALTER TABLE "exif" DROP COLUMN "updatedAt"`); + await queryRunner.query(`DROP TRIGGER asset_exif_updated_at on exif`); + } +} diff --git a/server/src/queries/asset.repository.sql b/server/src/queries/asset.repository.sql index c0b778bb5..1fd8f55f3 100644 --- a/server/src/queries/asset.repository.sql +++ b/server/src/queries/asset.repository.sql @@ -420,8 +420,8 @@ from ) as "stacked_assets" on "asset_stack"."id" is not null where "assets"."ownerId" = $1::uuid - and "isVisible" = $2 - and "updatedAt" <= $3 + and "assets"."isVisible" = $2 + and "assets"."updatedAt" <= $3 and "assets"."id" > $4 order by "assets"."id" @@ -450,7 +450,7 @@ from ) as "stacked_assets" on "asset_stack"."id" is not null where "assets"."ownerId" = any ($1::uuid[]) - and "isVisible" = $2 - and "updatedAt" > $3 + and "assets"."isVisible" = $2 + and "assets"."updatedAt" > $3 limit $4 diff --git a/server/src/repositories/asset.repository.ts b/server/src/repositories/asset.repository.ts index 5f020ba83..1e6429b32 100644 --- a/server/src/repositories/asset.repository.ts +++ b/server/src/repositories/asset.repository.ts @@ -551,7 +551,7 @@ export class AssetRepository { return this.getById(asset.id, { exifInfo: true, faces: { person: true } }) as Promise; } - async remove(asset: AssetEntity): Promise { + async remove(asset: { id: string }): Promise { await this.db.deleteFrom('assets').where('id', '=', asUuid(asset.id)).execute(); } @@ -968,8 +968,8 @@ export class AssetRepository { ) .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.ownerId', '=', asUuid(ownerId)) - .where('isVisible', '=', true) - .where('updatedAt', '<=', updatedUntil) + .where('assets.isVisible', '=', true) + .where('assets.updatedAt', '<=', updatedUntil) .$if(!!lastId, (qb) => qb.where('assets.id', '>', lastId!)) .orderBy('assets.id') .limit(limit) @@ -996,8 +996,8 @@ export class AssetRepository { ) .select((eb) => eb.fn.toJson(eb.table('stacked_assets')).as('stack')) .where('assets.ownerId', '=', anyUuid(options.userIds)) - .where('isVisible', '=', true) - .where('updatedAt', '>', options.updatedAfter) + .where('assets.isVisible', '=', true) + .where('assets.updatedAt', '>', options.updatedAfter) .limit(options.limit) .execute() as any as Promise; } diff --git a/server/src/repositories/sync.repository.ts b/server/src/repositories/sync.repository.ts index f2c5a1fc1..bc3205c0a 100644 --- a/server/src/repositories/sync.repository.ts +++ b/server/src/repositories/sync.repository.ts @@ -1,10 +1,14 @@ import { Injectable } from '@nestjs/common'; -import { Insertable, Kysely, sql } from 'kysely'; +import { Insertable, Kysely, SelectQueryBuilder, sql } from 'kysely'; import { InjectKysely } from 'nestjs-kysely'; +import { columns } from 'src/database'; import { DB, SessionSyncCheckpoints } from 'src/db'; import { SyncEntityType } from 'src/enum'; import { SyncAck } from 'src/types'; +type auditTables = 'users_audit' | 'partners_audit' | 'assets_audit'; +type upsertTables = 'users' | 'partners' | 'assets' | 'exif'; + @Injectable() export class SyncRepository { constructor(@InjectKysely() private db: Kysely) {} @@ -41,9 +45,7 @@ export class SyncRepository { return this.db .selectFrom('users') .select(['id', 'name', 'email', 'deletedAt', 'updateId']) - .$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId)) - .where('updatedAt', '<', sql.raw("now() - interval '1 millisecond'")) - .orderBy(['updateId asc']) + .$call((qb) => this.upsertTableFilters(qb, ack)) .stream(); } @@ -51,9 +53,7 @@ export class SyncRepository { return this.db .selectFrom('users_audit') .select(['id', 'userId']) - .$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId)) - .where('deletedAt', '<', sql.raw("now() - interval '1 millisecond'")) - .orderBy(['id asc']) + .$call((qb) => this.auditTableFilters(qb, ack)) .stream(); } @@ -61,10 +61,8 @@ export class SyncRepository { return this.db .selectFrom('partners') .select(['sharedById', 'sharedWithId', 'inTimeline', 'updateId']) - .$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId)) .where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)])) - .where('updatedAt', '<', sql.raw("now() - interval '1 millisecond'")) - .orderBy(['updateId asc']) + .$call((qb) => this.upsertTableFilters(qb, ack)) .stream(); } @@ -72,10 +70,93 @@ export class SyncRepository { return this.db .selectFrom('partners_audit') .select(['id', 'sharedById', 'sharedWithId']) - .$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId)) .where((eb) => eb.or([eb('sharedById', '=', userId), eb('sharedWithId', '=', userId)])) - .where('deletedAt', '<', sql.raw("now() - interval '1 millisecond'")) - .orderBy(['id asc']) + .$call((qb) => this.auditTableFilters(qb, ack)) .stream(); } + + getAssetUpserts(userId: string, ack?: SyncAck) { + return this.db + .selectFrom('assets') + .select(columns.syncAsset) + .where('ownerId', '=', userId) + .$call((qb) => this.upsertTableFilters(qb, ack)) + .stream(); + } + + getPartnerAssetsUpserts(userId: string, ack?: SyncAck) { + return this.db + .selectFrom('assets') + .select(columns.syncAsset) + .where('ownerId', 'in', (eb) => + eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId), + ) + .$call((qb) => this.upsertTableFilters(qb, ack)) + .stream(); + } + + getAssetDeletes(userId: string, ack?: SyncAck) { + return this.db + .selectFrom('assets_audit') + .select(['id', 'assetId']) + .where('ownerId', '=', userId) + .$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId)) + .$call((qb) => this.auditTableFilters(qb, ack)) + .stream(); + } + + getPartnerAssetDeletes(userId: string, ack?: SyncAck) { + return this.db + .selectFrom('assets_audit') + .select(['id', 'assetId']) + .where('ownerId', 'in', (eb) => + eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId), + ) + .$call((qb) => this.auditTableFilters(qb, ack)) + .stream(); + } + + getAssetExifsUpserts(userId: string, ack?: SyncAck) { + return this.db + .selectFrom('exif') + .select(columns.syncAssetExif) + .where('assetId', 'in', (eb) => eb.selectFrom('assets').select('id').where('ownerId', '=', userId)) + .$call((qb) => this.upsertTableFilters(qb, ack)) + .stream(); + } + + getPartnerAssetExifsUpserts(userId: string, ack?: SyncAck) { + return this.db + .selectFrom('exif') + .select(columns.syncAssetExif) + .where('assetId', 'in', (eb) => + eb + .selectFrom('assets') + .select('id') + .where('ownerId', 'in', (eb) => + eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId), + ), + ) + .$call((qb) => this.upsertTableFilters(qb, ack)) + .stream(); + } + + private auditTableFilters, D>(qb: SelectQueryBuilder, ack?: SyncAck) { + const builder = qb as SelectQueryBuilder; + return builder + .where('deletedAt', '<', sql.raw("now() - interval '1 millisecond'")) + .$if(!!ack, (qb) => qb.where('id', '>', ack!.updateId)) + .orderBy(['id asc']) as SelectQueryBuilder; + } + + private upsertTableFilters, D>( + qb: SelectQueryBuilder, + ack?: SyncAck, + ) { + const builder = qb as SelectQueryBuilder; + return builder + .where('updatedAt', '<', sql.raw("now() - interval '1 millisecond'")) + .$if(!!ack, (qb) => qb.where('updateId', '>', ack!.updateId)) + .orderBy(['updateId asc']) as SelectQueryBuilder; + } } diff --git a/server/src/services/sync.service.ts b/server/src/services/sync.service.ts index 45b1b7ff8..c88348b39 100644 --- a/server/src/services/sync.service.ts +++ b/server/src/services/sync.service.ts @@ -4,7 +4,7 @@ import { DateTime } from 'luxon'; import { Writable } from 'node:stream'; import { AUDIT_LOG_MAX_DURATION } from 'src/constants'; import { SessionSyncCheckpoints } from 'src/db'; -import { AssetResponseDto, mapAsset } from 'src/dtos/asset-response.dto'; +import { AssetResponseDto, hexOrBufferToBase64, mapAsset } from 'src/dtos/asset-response.dto'; import { AuthDto } from 'src/dtos/auth.dto'; import { AssetDeltaSyncDto, @@ -22,10 +22,14 @@ import { setIsEqual } from 'src/utils/set'; import { fromAck, serialize } from 'src/utils/sync'; const FULL_SYNC = { needsFullSync: true, deleted: [], upserted: [] }; -const SYNC_TYPES_ORDER = [ +export const SYNC_TYPES_ORDER = [ // SyncRequestType.UsersV1, SyncRequestType.PartnersV1, + SyncRequestType.AssetsV1, + SyncRequestType.AssetExifsV1, + SyncRequestType.PartnerAssetsV1, + SyncRequestType.PartnerAssetExifsV1, ]; const throwSessionRequired = () => { @@ -49,17 +53,22 @@ export class SyncService extends BaseService { return throwSessionRequired(); } - const checkpoints: Insertable[] = []; + const checkpoints: Record> = {}; for (const ack of dto.acks) { const { type } = fromAck(ack); // TODO proper ack validation via class validator if (!Object.values(SyncEntityType).includes(type)) { throw new BadRequestException(`Invalid ack type: ${type}`); } - checkpoints.push({ sessionId, type, ack }); + + if (checkpoints[type]) { + throw new BadRequestException('Only one ack per type is allowed'); + } + + checkpoints[type] = { sessionId, type, ack }; } - await this.syncRepository.upsertCheckpoints(checkpoints); + await this.syncRepository.upsertCheckpoints(Object.values(checkpoints)); } async deleteAcks(auth: AuthDto, dto: SyncAckDeleteDto) { @@ -115,6 +124,87 @@ export class SyncService extends BaseService { break; } + case SyncRequestType.AssetsV1: { + const deletes = this.syncRepository.getAssetDeletes( + auth.user.id, + checkpointMap[SyncEntityType.AssetDeleteV1], + ); + for await (const { id, ...data } of deletes) { + response.write(serialize({ type: SyncEntityType.AssetDeleteV1, updateId: id, data })); + } + + const upserts = this.syncRepository.getAssetUpserts(auth.user.id, checkpointMap[SyncEntityType.AssetV1]); + for await (const { updateId, checksum, thumbhash, ...data } of upserts) { + response.write( + serialize({ + type: SyncEntityType.AssetV1, + updateId, + data: { + ...data, + checksum: hexOrBufferToBase64(checksum), + thumbhash: thumbhash ? hexOrBufferToBase64(thumbhash) : null, + }, + }), + ); + } + + break; + } + + case SyncRequestType.PartnerAssetsV1: { + const deletes = this.syncRepository.getPartnerAssetDeletes( + auth.user.id, + checkpointMap[SyncEntityType.PartnerAssetDeleteV1], + ); + for await (const { id, ...data } of deletes) { + response.write(serialize({ type: SyncEntityType.PartnerAssetDeleteV1, updateId: id, data })); + } + + const upserts = this.syncRepository.getPartnerAssetsUpserts( + auth.user.id, + checkpointMap[SyncEntityType.PartnerAssetV1], + ); + for await (const { updateId, checksum, thumbhash, ...data } of upserts) { + response.write( + serialize({ + type: SyncEntityType.PartnerAssetV1, + updateId, + data: { + ...data, + checksum: hexOrBufferToBase64(checksum), + thumbhash: thumbhash ? hexOrBufferToBase64(thumbhash) : null, + }, + }), + ); + } + + break; + } + + case SyncRequestType.AssetExifsV1: { + const upserts = this.syncRepository.getAssetExifsUpserts( + auth.user.id, + checkpointMap[SyncEntityType.AssetExifV1], + ); + for await (const { updateId, ...data } of upserts) { + response.write(serialize({ type: SyncEntityType.AssetExifV1, updateId, data })); + } + + break; + } + + case SyncRequestType.PartnerAssetExifsV1: { + const upserts = this.syncRepository.getPartnerAssetExifsUpserts( + auth.user.id, + checkpointMap[SyncEntityType.PartnerAssetExifV1], + ); + for await (const { updateId, ...data } of upserts) { + response.write(serialize({ type: SyncEntityType.PartnerAssetExifV1, updateId, data })); + } + + break; + } + default: { this.logger.warn(`Unsupported sync type: ${type}`); break; diff --git a/server/test/factory.ts b/server/test/factory.ts index 9cb2c818a..66c2fbb50 100644 --- a/server/test/factory.ts +++ b/server/test/factory.ts @@ -55,7 +55,7 @@ class CustomWritable extends Writable { } } -type Asset = Insertable; +type Asset = Partial>; type User = Partial>; type Session = Omit, 'token'> & { token?: string }; type Partner = Insertable; @@ -160,10 +160,6 @@ export class TestFactory { } async create() { - for (const asset of this.assets) { - await this.context.createAsset(asset); - } - for (const user of this.users) { await this.context.createUser(user); } @@ -176,6 +172,10 @@ export class TestFactory { await this.context.createSession(session); } + for (const asset of this.assets) { + await this.context.createAsset(asset); + } + return this.context; } } @@ -212,7 +212,7 @@ export class TestContext { versionHistory: VersionHistoryRepository; view: ViewRepository; - private constructor(private db: Kysely) { + private constructor(public db: Kysely) { const logger = newLoggingRepositoryMock() as unknown as LoggingRepository; const config = new ConfigRepository(); diff --git a/server/test/medium/specs/audit.database.spec.ts b/server/test/medium/specs/audit.database.spec.ts new file mode 100644 index 000000000..5332193e4 --- /dev/null +++ b/server/test/medium/specs/audit.database.spec.ts @@ -0,0 +1,74 @@ +import { TestContext, TestFactory } from 'test/factory'; +import { getKyselyDB } from 'test/utils'; + +describe('audit', () => { + let context: TestContext; + + beforeAll(async () => { + const db = await getKyselyDB(); + context = await TestContext.from(db).create(); + }); + + describe('partners_audit', () => { + it('should not cascade user deletes to partners_audit', async () => { + const user1 = TestFactory.user(); + const user2 = TestFactory.user(); + + await context + .getFactory() + .withUser(user1) + .withUser(user2) + .withPartner({ sharedById: user1.id, sharedWithId: user2.id }) + .create(); + + await context.user.delete(user1, true); + + await expect( + context.db.selectFrom('partners_audit').select(['id']).where('sharedById', '=', user1.id).execute(), + ).resolves.toHaveLength(0); + }); + }); + + describe('assets_audit', () => { + it('should not cascade user deletes to assets_audit', async () => { + const user = TestFactory.user(); + const asset = TestFactory.asset({ ownerId: user.id }); + + await context.getFactory().withUser(user).withAsset(asset).create(); + + await context.user.delete(user, true); + + await expect( + context.db.selectFrom('assets_audit').select(['id']).where('assetId', '=', asset.id).execute(), + ).resolves.toHaveLength(0); + }); + }); + + describe('exif', () => { + it('should automatically set updatedAt and updateId when the row is updated', async () => { + const user = TestFactory.user(); + const asset = TestFactory.asset({ ownerId: user.id }); + const exif = { assetId: asset.id, make: 'Canon' }; + + await context.getFactory().withUser(user).withAsset(asset).create(); + await context.asset.upsertExif(exif); + + const before = await context.db + .selectFrom('exif') + .select(['updatedAt', 'updateId']) + .where('assetId', '=', asset.id) + .executeTakeFirstOrThrow(); + + await context.asset.upsertExif({ assetId: asset.id, make: 'Canon 2' }); + + const after = await context.db + .selectFrom('exif') + .select(['updatedAt', 'updateId']) + .where('assetId', '=', asset.id) + .executeTakeFirstOrThrow(); + + expect(before.updateId).not.toEqual(after.updateId); + expect(before.updatedAt).not.toEqual(after.updatedAt); + }); + }); +}); diff --git a/server/test/medium/specs/sync.service.spec.ts b/server/test/medium/specs/sync.service.spec.ts index b33b01025..574ddde93 100644 --- a/server/test/medium/specs/sync.service.spec.ts +++ b/server/test/medium/specs/sync.service.spec.ts @@ -1,6 +1,6 @@ import { AuthDto } from 'src/dtos/auth.dto'; -import { SyncRequestType } from 'src/enum'; -import { SyncService } from 'src/services/sync.service'; +import { SyncEntityType, SyncRequestType } from 'src/enum'; +import { SYNC_TYPES_ORDER, SyncService } from 'src/services/sync.service'; import { TestContext, TestFactory } from 'test/factory'; import { getKyselyDB, newTestService } from 'test/utils'; @@ -33,7 +33,15 @@ const setup = async () => { }; describe(SyncService.name, () => { - describe.concurrent('users', () => { + it('should have all the types in the ordering variable', () => { + for (const key in SyncRequestType) { + expect(SYNC_TYPES_ORDER).includes(key); + } + + expect(SYNC_TYPES_ORDER.length).toBe(Object.keys(SyncRequestType).length); + }); + + describe.concurrent(SyncEntityType.UserV1, () => { it('should detect and sync the first user', async () => { const { context, auth, sut, testSync } = await setup(); @@ -189,7 +197,7 @@ describe(SyncService.name, () => { }); }); - describe.concurrent('partners', () => { + describe.concurrent(SyncEntityType.PartnerV1, () => { it('should detect and sync the first partner', async () => { const { auth, context, sut, testSync } = await setup(); @@ -349,7 +357,7 @@ describe(SyncService.name, () => { ); }); - it('should not sync a partner for an unrelated user', async () => { + it('should not sync a partner or partner delete for an unrelated user', async () => { const { auth, context, testSync } = await setup(); const user2 = await context.createUser(); @@ -357,9 +365,436 @@ describe(SyncService.name, () => { await context.createPartner({ sharedById: user2.id, sharedWithId: user3.id }); - const response = await testSync(auth, [SyncRequestType.PartnersV1]); + expect(await testSync(auth, [SyncRequestType.PartnersV1])).toHaveLength(0); + + await context.partner.remove({ sharedById: user2.id, sharedWithId: user3.id }); + + expect(await testSync(auth, [SyncRequestType.PartnersV1])).toHaveLength(0); + }); + + it('should not sync a partner delete after a user is deleted', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + await context.createPartner({ sharedById: user2.id, sharedWithId: auth.user.id }); + await context.user.delete({ id: user2.id }, true); + + expect(await testSync(auth, [SyncRequestType.PartnersV1])).toHaveLength(0); + }); + }); + + describe.concurrent(SyncEntityType.AssetV1, () => { + it('should detect and sync the first asset', async () => { + const { auth, context, sut, testSync } = await setup(); + + const checksum = '1115vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA='; + const thumbhash = '2225vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA='; + const date = new Date().toISOString(); + + const asset = TestFactory.asset({ + ownerId: auth.user.id, + checksum: Buffer.from(checksum, 'base64'), + thumbhash: Buffer.from(thumbhash, 'base64'), + fileCreatedAt: date, + fileModifiedAt: date, + deletedAt: null, + }); + await context.createAsset(asset); + + const initialSyncResponse = await testSync(auth, [SyncRequestType.AssetsV1]); + + expect(initialSyncResponse).toHaveLength(1); + expect(initialSyncResponse).toEqual( + expect.arrayContaining([ + { + ack: expect.any(String), + data: { + id: asset.id, + ownerId: asset.ownerId, + thumbhash, + checksum, + deletedAt: null, + fileCreatedAt: date, + fileModifiedAt: date, + isFavorite: false, + isVisible: true, + localDateTime: null, + type: asset.type, + }, + type: 'AssetV1', + }, + ]), + ); + + const acks = [initialSyncResponse[0].ack]; + await sut.setAcks(auth, { acks }); + + const ackSyncResponse = await testSync(auth, [SyncRequestType.AssetsV1]); + + expect(ackSyncResponse).toHaveLength(0); + }); + + it('should detect and sync a deleted asset', async () => { + const { auth, context, sut, testSync } = await setup(); + + const asset = TestFactory.asset({ ownerId: auth.user.id }); + await context.createAsset(asset); + await context.asset.remove(asset); + + const response = await testSync(auth, [SyncRequestType.AssetsV1]); + + expect(response).toHaveLength(1); + expect(response).toEqual( + expect.arrayContaining([ + { + ack: expect.any(String), + data: { + assetId: asset.id, + }, + type: 'AssetDeleteV1', + }, + ]), + ); + + const acks = response.map(({ ack }) => ack); + await sut.setAcks(auth, { acks }); + + const ackSyncResponse = await testSync(auth, [SyncRequestType.AssetsV1]); + + expect(ackSyncResponse).toHaveLength(0); + }); + + it('should not sync an asset or asset delete for an unrelated user', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + const session = TestFactory.session({ userId: user2.id }); + const auth2 = TestFactory.auth({ session, user: user2 }); + + const asset = TestFactory.asset({ ownerId: user2.id }); + await context.createAsset(asset); + + expect(await testSync(auth2, [SyncRequestType.AssetsV1])).toHaveLength(1); + expect(await testSync(auth, [SyncRequestType.AssetsV1])).toHaveLength(0); + + await context.asset.remove(asset); + expect(await testSync(auth2, [SyncRequestType.AssetsV1])).toHaveLength(1); + expect(await testSync(auth, [SyncRequestType.AssetsV1])).toHaveLength(0); + }); + }); + + describe.concurrent(SyncRequestType.PartnerAssetsV1, () => { + it('should detect and sync the first partner asset', async () => { + const { auth, context, sut, testSync } = await setup(); + + const checksum = '1115vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA='; + const thumbhash = '2225vHcVkZzNp3Q9G+FEA0nu6zUbGb4Tj4UOXkN0wRA='; + const date = new Date().toISOString(); + + const user2 = await context.createUser(); + + const asset = TestFactory.asset({ + ownerId: user2.id, + checksum: Buffer.from(checksum, 'base64'), + thumbhash: Buffer.from(thumbhash, 'base64'), + fileCreatedAt: date, + fileModifiedAt: date, + deletedAt: null, + }); + await context.createAsset(asset); + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + + const initialSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]); + + expect(initialSyncResponse).toHaveLength(1); + expect(initialSyncResponse).toEqual( + expect.arrayContaining([ + { + ack: expect.any(String), + data: { + id: asset.id, + ownerId: asset.ownerId, + thumbhash, + checksum, + deletedAt: null, + fileCreatedAt: date, + fileModifiedAt: date, + isFavorite: false, + isVisible: true, + localDateTime: null, + type: asset.type, + }, + type: SyncEntityType.PartnerAssetV1, + }, + ]), + ); + + const acks = [initialSyncResponse[0].ack]; + await sut.setAcks(auth, { acks }); + + const ackSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]); + + expect(ackSyncResponse).toHaveLength(0); + }); + + it('should detect and sync a deleted partner asset', async () => { + const { auth, context, sut, testSync } = await setup(); + + const user2 = await context.createUser(); + const asset = TestFactory.asset({ ownerId: user2.id }); + await context.createAsset(asset); + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + await context.asset.remove(asset); + + const response = await testSync(auth, [SyncRequestType.PartnerAssetsV1]); + + expect(response).toHaveLength(1); + expect(response).toEqual( + expect.arrayContaining([ + { + ack: expect.any(String), + data: { + assetId: asset.id, + }, + type: SyncEntityType.PartnerAssetDeleteV1, + }, + ]), + ); + + const acks = response.map(({ ack }) => ack); + await sut.setAcks(auth, { acks }); + + const ackSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetsV1]); + + expect(ackSyncResponse).toHaveLength(0); + }); + + it('should not sync a deleted partner asset due to a user delete', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + await context.createAsset({ ownerId: user2.id }); + await context.user.delete({ id: user2.id }, true); + + const response = await testSync(auth, [SyncRequestType.PartnerAssetsV1]); expect(response).toHaveLength(0); }); + + it('should not sync a deleted partner asset due to a partner delete (unshare)', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + await context.createAsset({ ownerId: user2.id }); + const partner = { sharedById: user2.id, sharedWithId: auth.user.id }; + await context.partner.create(partner); + + await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(1); + + await context.partner.remove(partner); + + await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0); + }); + + it('should not sync an asset or asset delete for own user', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + const asset = await context.createAsset({ ownerId: auth.user.id }); + const partner = { sharedById: user2.id, sharedWithId: auth.user.id }; + await context.partner.create(partner); + + await expect(testSync(auth, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0); + + await context.asset.remove(asset); + + await expect(testSync(auth, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0); + }); + + it('should not sync an asset or asset delete for unrelated user', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + const session = TestFactory.session({ userId: user2.id }); + const auth2 = TestFactory.auth({ session, user: user2 }); + const asset = await context.createAsset({ ownerId: user2.id }); + + await expect(testSync(auth2, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0); + + await context.asset.remove(asset); + + await expect(testSync(auth2, [SyncRequestType.AssetsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.PartnerAssetsV1])).resolves.toHaveLength(0); + }); + }); + + describe.concurrent(SyncRequestType.AssetExifsV1, () => { + it('should detect and sync the first asset exif', async () => { + const { auth, context, sut, testSync } = await setup(); + + const asset = TestFactory.asset({ ownerId: auth.user.id }); + const exif = { assetId: asset.id, make: 'Canon' }; + + await context.createAsset(asset); + await context.asset.upsertExif(exif); + + const initialSyncResponse = await testSync(auth, [SyncRequestType.AssetExifsV1]); + + expect(initialSyncResponse).toHaveLength(1); + expect(initialSyncResponse).toEqual( + expect.arrayContaining([ + { + ack: expect.any(String), + data: { + assetId: asset.id, + city: null, + country: null, + dateTimeOriginal: null, + description: '', + exifImageHeight: null, + exifImageWidth: null, + exposureTime: null, + fNumber: null, + fileSizeInByte: null, + focalLength: null, + fps: null, + iso: null, + latitude: null, + lensModel: null, + longitude: null, + make: 'Canon', + model: null, + modifyDate: null, + orientation: null, + profileDescription: null, + projectionType: null, + rating: null, + state: null, + timeZone: null, + }, + type: SyncEntityType.AssetExifV1, + }, + ]), + ); + + const acks = [initialSyncResponse[0].ack]; + await sut.setAcks(auth, { acks }); + + const ackSyncResponse = await testSync(auth, [SyncRequestType.AssetExifsV1]); + + expect(ackSyncResponse).toHaveLength(0); + }); + + it('should only sync asset exif for own user', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + const session = TestFactory.session({ userId: user2.id }); + const auth2 = TestFactory.auth({ session, user: user2 }); + + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + const asset = TestFactory.asset({ ownerId: user2.id }); + const exif = { assetId: asset.id, make: 'Canon' }; + + await context.createAsset(asset); + await context.asset.upsertExif(exif); + + await expect(testSync(auth2, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(0); + }); + }); + + describe.concurrent(SyncRequestType.PartnerAssetExifsV1, () => { + it('should detect and sync the first partner asset exif', async () => { + const { auth, context, sut, testSync } = await setup(); + + const user2 = await context.createUser(); + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + const asset = TestFactory.asset({ ownerId: user2.id }); + await context.createAsset(asset); + const exif = { assetId: asset.id, make: 'Canon' }; + await context.asset.upsertExif(exif); + + const initialSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]); + + expect(initialSyncResponse).toHaveLength(1); + expect(initialSyncResponse).toEqual( + expect.arrayContaining([ + { + ack: expect.any(String), + data: { + assetId: asset.id, + city: null, + country: null, + dateTimeOriginal: null, + description: '', + exifImageHeight: null, + exifImageWidth: null, + exposureTime: null, + fNumber: null, + fileSizeInByte: null, + focalLength: null, + fps: null, + iso: null, + latitude: null, + lensModel: null, + longitude: null, + make: 'Canon', + model: null, + modifyDate: null, + orientation: null, + profileDescription: null, + projectionType: null, + rating: null, + state: null, + timeZone: null, + }, + type: SyncEntityType.PartnerAssetExifV1, + }, + ]), + ); + + const acks = [initialSyncResponse[0].ack]; + await sut.setAcks(auth, { acks }); + + const ackSyncResponse = await testSync(auth, [SyncRequestType.PartnerAssetExifsV1]); + + expect(ackSyncResponse).toHaveLength(0); + }); + + it('should not sync partner asset exif for own user', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + const asset = TestFactory.asset({ ownerId: auth.user.id }); + const exif = { assetId: asset.id, make: 'Canon' }; + await context.createAsset(asset); + await context.asset.upsertExif(exif); + + await expect(testSync(auth, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.PartnerAssetExifsV1])).resolves.toHaveLength(0); + }); + + it('should not sync partner asset exif for unrelated user', async () => { + const { auth, context, testSync } = await setup(); + + const user2 = await context.createUser(); + const user3 = await context.createUser(); + const session = TestFactory.session({ userId: user3.id }); + const authUser3 = TestFactory.auth({ session, user: user3 }); + await context.partner.create({ sharedById: user2.id, sharedWithId: auth.user.id }); + const asset = TestFactory.asset({ ownerId: user3.id }); + const exif = { assetId: asset.id, make: 'Canon' }; + await context.createAsset(asset); + await context.asset.upsertExif(exif); + + await expect(testSync(authUser3, [SyncRequestType.AssetExifsV1])).resolves.toHaveLength(1); + await expect(testSync(auth, [SyncRequestType.PartnerAssetExifsV1])).resolves.toHaveLength(0); + }); }); }); diff --git a/server/test/repositories/sync.repository.mock.ts b/server/test/repositories/sync.repository.mock.ts index 6d94f6e03..c7fb154db 100644 --- a/server/test/repositories/sync.repository.mock.ts +++ b/server/test/repositories/sync.repository.mock.ts @@ -11,5 +11,11 @@ export const newSyncRepositoryMock = (): Mocked