From b3c7bebbd4f85ce41630c7f5e5f0e94e06f5f922 Mon Sep 17 00:00:00 2001 From: Jonathan Jogenfors Date: Tue, 20 Feb 2024 16:53:12 +0100 Subject: [PATCH] feat(server,web) Semantic import path validation (#7076) * add library validation api * chore: open api * show warning i UI * add flex row * fix e2e * tests * fix tests * enforce path validation * enforce validation on refresh * return 400 on bad import path * add limits to import paths * set response code to 200 * fix e2e * fix lint * fix test * restore e2e folder * fix import * use startsWith * icon color * notify user of failed validation * add parent div to validation * add docs to the import validation * improve library troubleshooting docs * fix button alignment --------- Co-authored-by: Alex Tran --- docs/docs/features/libraries.md | 17 +- mobile/openapi/.openapi-generator/FILES | 9 + mobile/openapi/README.md | Bin 24512 -> 24816 bytes mobile/openapi/doc/LibraryApi.md | Bin 16466 -> 18679 bytes mobile/openapi/doc/ValidateLibraryDto.md | Bin 0 -> 536 bytes .../ValidateLibraryImportPathResponseDto.md | Bin 0 -> 536 bytes .../openapi/doc/ValidateLibraryResponseDto.md | Bin 0 -> 538 bytes mobile/openapi/lib/api.dart | Bin 8381 -> 8531 bytes mobile/openapi/lib/api/library_api.dart | Bin 11890 -> 13728 bytes mobile/openapi/lib/api_client.dart | Bin 23692 -> 24008 bytes .../lib/model/validate_library_dto.dart | Bin 0 -> 3454 bytes ...date_library_import_path_response_dto.dart | Bin 0 -> 4087 bytes .../model/validate_library_response_dto.dart | Bin 0 -> 3094 bytes mobile/openapi/test/library_api_test.dart | Bin 1542 -> 1708 bytes .../test/validate_library_dto_test.dart | Bin 0 -> 767 bytes ...library_import_path_response_dto_test.dart | Bin 0 -> 855 bytes .../validate_library_response_dto_test.dart | Bin 0 -> 670 bytes open-api/immich-openapi-specs.json | 98 ++++++++++ open-api/typescript-sdk/axios-client/api.ts | 159 ++++++++++++++++ open-api/typescript-sdk/fetch-client.ts | Bin 73711 -> 74452 bytes server/e2e/api/setup.ts | 15 +- server/e2e/api/specs/library.e2e-spec.ts | 96 +++++++++- server/e2e/client/library-api.ts | 10 + server/src/domain/library/library.dto.ts | 32 +++- .../domain/library/library.service.spec.ts | 178 ++++++++++++++++-- server/src/domain/library/library.service.ts | 106 ++++++++++- .../immich/controllers/library.controller.ts | 14 +- server/src/infra/infra.utils.ts | 37 ++-- .../infra/repositories/filesystem.provider.ts | 26 +-- server/test/fixtures/user.stub.ts | 19 ++ .../forms/library-import-paths-form.svelte | 102 ++++++++-- .../user-settings-page/library-list.svelte | 5 +- 32 files changed, 848 insertions(+), 75 deletions(-) create mode 100644 mobile/openapi/doc/ValidateLibraryDto.md create mode 100644 mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md create mode 100644 mobile/openapi/doc/ValidateLibraryResponseDto.md create mode 100644 mobile/openapi/lib/model/validate_library_dto.dart create mode 100644 mobile/openapi/lib/model/validate_library_import_path_response_dto.dart create mode 100644 mobile/openapi/lib/model/validate_library_response_dto.dart create mode 100644 mobile/openapi/test/validate_library_dto_test.dart create mode 100644 mobile/openapi/test/validate_library_import_path_response_dto_test.dart create mode 100644 mobile/openapi/test/validate_library_response_dto_test.dart diff --git a/docs/docs/features/libraries.md b/docs/docs/features/libraries.md index 6dcb9982b..995d65249 100644 --- a/docs/docs/features/libraries.md +++ b/docs/docs/features/libraries.md @@ -42,23 +42,24 @@ Finally, files can be deleted from Immich via the `Remove Offline Files` job. Th ### Import Paths -External libraries use import paths to determine which files to scan. Each library can have multiple import paths so that files from different locations can be added to the same library. Import paths are scanned recursively, and if a file is in multiple import paths, it will only be added once. If the import paths are edited in a way that an external file is no longer in any import path, it will be removed from the library in the same way a deleted file would. If the file is moved back to an import path, it will be added again as if it was a new file. +External libraries use import paths to determine which files to scan. Each library can have multiple import paths so that files from different locations can be added to the same library. Import paths are scanned recursively, and if a file is in multiple import paths, it will only be added once. Each import file must be a readable directory that exists on the filesystem; the import path dialog will alert you of any paths that are not accessible. + +If the import paths are edited in a way that an external file is no longer in any import path, it will be removed from the library in the same way a deleted file would. If the file is moved back to an import path, it will be added again as if it was a new file. ### Troubleshooting -Sometimes, an external library will not scan correctly. This can happen if the immich_server or immich_microservices can't access the files. Here are some things to check: +Sometimes, an external library will not scan correctly. This can happen if immich_server or immich_microservices can't access the files. Here are some things to check: -- Is the external path set correctly? +- Is the external path set correctly? Each import path must be contained in the external path. +- Make sure the external path does not contain spaces - In the docker-compose file, are the volumes mounted correctly? - Are the volumes identical between the `server` and `microservices` container? - Are the import paths set correctly, and do they match the path set in docker-compose file? -- Are you using symbolic link in your import library? +- Make sure you don't use symlinks in your import libraries, and that you aren't linking across docker mounts. - Are the permissions set correctly? -- Are you using forward slashes everywhere? (`/`) -- Are you using symlink across docker mounts? -- Are you using [spaces in the internal path](/docs/features/libraries#:~:text=can%20be%20accessed.-,NOTE,-Spaces%20in%20the)? +- Make sure you are using forward slashes (`/`) and not backward slashes. -If all else fails, you can always start a shell inside the container and check if the path is accessible. For example, `docker exec -it immich_microservices /bin/bash` will start a bash shell. If your import path, for instance, is `/data/import/photos`, you can check if the files are accessible by running `ls /data/import/photos`. Also check the `immich_server` container in the same way. +To validate that Immich can reach your external library, start a shell inside the container. Run `docker exec -it immich_microservices /bin/bash` to a bash shell. If your import path is `/data/import/photos`, check it with `ls /data/import/photos`. Do the same check for the `immich_server` container. If you cannot access this directory in both the `microservices` and `server` containers, Immich won't be able to import files. ### Security Considerations diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index f42a9c82c..0679a1749 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -182,6 +182,9 @@ doc/UserAvatarColor.md doc/UserDto.md doc/UserResponseDto.md doc/ValidateAccessTokenResponseDto.md +doc/ValidateLibraryDto.md +doc/ValidateLibraryImportPathResponseDto.md +doc/ValidateLibraryResponseDto.md doc/VideoCodec.md git_push.sh lib/api.dart @@ -372,6 +375,9 @@ lib/model/user_avatar_color.dart lib/model/user_dto.dart lib/model/user_response_dto.dart lib/model/validate_access_token_response_dto.dart +lib/model/validate_library_dto.dart +lib/model/validate_library_import_path_response_dto.dart +lib/model/validate_library_response_dto.dart lib/model/video_codec.dart pubspec.yaml test/activity_api_test.dart @@ -553,4 +559,7 @@ test/user_avatar_color_test.dart test/user_dto_test.dart test/user_response_dto_test.dart test/validate_access_token_response_dto_test.dart +test/validate_library_dto_test.dart +test/validate_library_import_path_response_dto_test.dart +test/validate_library_response_dto_test.dart test/video_codec_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 9e241b0a44e30227c1b0f28910d87e6afd74bc4b..8cb0919df181ca3a88ccdd5a070f5900ee5031b1 100644 GIT binary patch delta 203 zcmX@GpYg*%#tjY1lg}s$a+D?JWTqsRq)z^*D30JvRFZ~sHdiYjw~~b_^T|vqN-U~$ zDanu3NXbvu*AD|pO#bgC<^pB`O@b=%%q_?-DhWs|$p}gaf=s>K6lIvqgm^0a(BOdOs$qQ}go50i{%+k_Qs8IlV6KoF9 do5(V;8X(chd3L%SFzLzp@?vn#=8bj>837L?Y_R|U delta 14 Wcmex9k?~Rk3e$ev(yfy3?%cceC(jYcoZ4353P zS1EJo<8|8hsw`)Z%lTrpEC-!w?hXT4!IVd_WnYZRhbeq@&+CUj_QroG S$ZE&N$~VMm;E&-`rPL>K$EyJV literal 0 HcmV?d00001 diff --git a/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md b/mobile/openapi/doc/ValidateLibraryImportPathResponseDto.md new file mode 100644 index 0000000000000000000000000000000000000000..4601d8d2f254760827141a9aff76db0b3c4ff165 GIT binary patch literal 536 zcma)2!D<3A5WVLs0(-D7?0Roc7Ci_mi!Hq@3p<)ogV{_-#)Ht0ZxSuFrO@UQ-pqUR zW?mu5p#`VwffSnh4UAU6xYSz@{<0Vxd2WD@8w`$Yz!{SeMIo02O7an9)=3Au6C6z8Qv{*N!#Wh}OiBQQf^* zyhBy>7ZzoCT`S#FmXThTFYL8FWDs#;gdD-Gy(KcA&6e^iU^>|<+{4+hWRiO@0lTOi zFoYPvuY>QjLbUC3Oe<+uDRXIb;I4b9>&a$5ovr5epbAC1^MIBiU0?0VS2^b}mS5^= Yv-qbrre2{{jk~4SG?w zbL<61SN!R!Jx%(FFnUhZL22vbqZdm-ZLlNwhPSgzS4pAq)`{glRvVVc$Zt*rZ>l{1 zuU1-LY;94FLF>q`v|iaDFL%o>2lt6~v-Ydj_4pBD(V0oqPVzjs$F_#b&;d`afG{*M zNP()Zm8zG4k^1S7)Zw>X#+vWTqsRq)wizo-Yvw<@jVK6(tr`x|HO{ zDu5JBR?yJnfU8_DAugeSsT87ey0+}(1xyZ;?<=z*bTA4>awyn>6{KudWZuL$IbKL| z@_%(+d^W0hpxF_WT3nEySDXrTmc1RsFrdRVf=h}r^U@VEQ*`h+%0w3C{P4_@43CnM m0?)j(d<~E$ki_QK+7?Wjxb0Kd(Nst*hMPZmzqkogEf)aq#(W+C delta 10 RcmZ3G{V8TcfgV#W7XTbH1Tg>r diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 3a0e535e73df53d2a09593e67e5c0316064d5f20..2df5e67119073f18eec7e21d7da21905bab6a2d1 100644 GIT binary patch delta 135 zcmeC#$#`Nn;|7Z$zQml&l*E!$pUkA9#G=Z{2NlFw!hm9vx%~AcK@22G&)kCiqLP5b hl8nh0ZRGLDHmYj#qv-|d;DCuu)-g2NY#tQK1pveGGY|j( delta 14 VcmX@Ho3Upn;|7bM%_YGhTmUh?1>FDu diff --git a/mobile/openapi/lib/model/validate_library_dto.dart b/mobile/openapi/lib/model/validate_library_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..a29a622079b82a0ee4f9f77bc1d6ce1b3eca89a6 GIT binary patch literal 3454 zcmb_fZExE)5dQ98aRrJ-##DLhr@@`w7K<~aP2-}?3JiuJFcKZHlSP-L>KSSNd+$g| zrbFdr!we*`sC(n{Ja=S9qrqqdZ~nZPKmTKPKKuCQVs;ECr)RSmj^}Vbzkv7iKml$U(XD-*ZRV!e^d^m?#z%XT{5 z)FxH@Yb_MIS8R>HH%#GQ!Op0ma@7+GgGiC z{q^TGE11%Q0j4t`S0FdMVjGd*&&$CeE1A~tnUx}EhM$Q`#nkTDNVpyVj1Yp4+|;Vl zz@Xw=ScH~K5VpjOC+HHsk0=s=DTGe=KUrC8A*&C};KniR1ndrBv5fa-4o@peSafg2^PBUw+{kLS+iBhA^3e zgFa(rM4O1)H{ZXzZyKIlyC;0Fn`uNtYq`aPPx{AFu6xhSH9@_u^RU#2o>W%%~a6$U&bls@LPW6zaDU=A! z35xbWE(CN8*LHljuwe^Y+~S*7FHS9r>$)`5*gfGFOIIIBj)D?mR0>XBtARx5b7^Ml zjoGzPQH6L`L``tH#Us#~?%%gMwF1S&Vcvw=(C!F1Zy>UacFKP;v2ExFB`wod5|-z# zKRizda0~PS+>KYDvKBZS#m<<93x8yWwNGKqHoc6r#X4xvia>WmKJEcw+74D9e9V0> z_LiiHHql z_}S02US<)jg>(Fy@8~qK{!W{e{vLYKwQDBaI6l%Sw>U|6o7oCe;1YEbT>?W&pF2bJ zYt9KgAkl&D!xRsDw?`Rwb*$9~z3kP*(vGyCE@ z;{aD0DT%!|@6R(cdnYHIlM{OO*VXvp9~YMw@19*}|)oQJcyHi6H8?m+1 zb>*f;eyWs#dMQ@;yA}rjoirB4jlEkv)1`HybW@eXp_(gcyt=*9S)od4r-d#T5OYgW zPJeqe&6dL0P6zHB={f1OEX7(4@cX3G$qHdD{VfWW3nyQznGt6D(x>xQxaB)(*N8x# zJ54toq7xM5ophBcEeR^+h9+%vuSxkdGC6>eHX2>_MwCiMZmH}bl$qn|`={a1dV4ga zxhSk`JXuL=#loN5_K+*lduLR+pxTZPM zZcBElt0JeR_$+BAWeK73VObT0m=*Gf-m5Z`2wRd}%M8F}!YN%6^C%k6wX|#+#mR~) zm#Ani=b|bceHKL}sashNmyO(uPfqXcBIxyLsW$pAz%|!N*fF|N*+W;|!Om4gO@NnMG0Xi6GNaT4CyCVR^{fz{@1L)&1O2*Y=+(TId z24@t}@dbaXk%fT*q&ym2W8t)+;jlZNeUKTbi!zuV4M!AOArnQH&G*xjFJHnZK-r37 z^xy&9QBZ1d3qBhR`ARC04I=U5o|TvsoA_3Ks&urF4glEYbH-*k@?q@84NsX;W=vkQ zJ4{~p}-Dc@pfs%!*iy|TJQ9AQArRVfPkfFlfm z1L>RFC%!P`B_`&$j^p?m3Sofkrrp=VA-;xWI0#~DPM&{*RNJGW@1+#k&S!jHn~4X( z%q>Ywt(XlskRokrF=SENr(^sC_L6*j%cG6E_J2)g5@jBFZ%H{DW{{=1(W_{G;>Qw( z`5Xerya6V_=(_Lo4t=yyHuid4o~M{sa*lQ6H<_t2bhVvwzfZB)oXV!yflHk;yTd`N z<~{Rqc`8I|gm_c+8J#dhF%1x8IwuU^24lR)AQRRdaHktqM%Y5U@mj^zMH=C!rWzVe ztkE)+9M7*XWTUH5mucIlJy!OP!UAI(u|E#fc|tanBTIX9hN9FfkS#m}#No%L`1-$=K_LQl5{)oX>yA+;ag9!69^^aZ%Ihgqn=ji31oqc^-ecboh+ffgV$ z((-CVo40Z}xehrn6W9%}1r8Sj-FMDF*-Py%R%^FSen1YaoNjcdH@x6jwSJ!H<9UNn z+C(zo4)R$e!B|)>ZcWjH$-$39x&tiWK;(vIfF;4R2U65>np5qjlAcof|1nmN?=b<#&WOXgv(7uoPDXU;1!^<<|^&sK@!6#u{fpnr+pg3U2cO`BRv%eb|_3-HL*c!;TH+kT` zL9!FPEy*39cCT@uuEniOfFk@xGLw8euyD(*Cg|1s@g=`0#O>m*47+#0KUsG-Qbsqv PM9mu=FE3p9Fx~zIg7!*p literal 0 HcmV?d00001 diff --git a/mobile/openapi/lib/model/validate_library_response_dto.dart b/mobile/openapi/lib/model/validate_library_response_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..bb975da23aa209f2656b20c95e70ff2e2aefcd63 GIT binary patch literal 3094 zcmbVOZExE)5dQ98aRr800aSVGry;Gq7K<~aOXDHU3JiuJ&=PI2lS!4NY8a{i`|e0d zp5VV|&UMLb4J*;}Vkno%O@^wJQLe#Z>o$P5LwR*jJ1<*G`|# zhZeS^njh3HFqt>p7!cOLS60^Cz(T846LspAQ0fab=u=lrvGQ3AvMy0!B&W47t^)$T zD$N(zm`>li&QOCUC3U9Q;#OoSPJ=?_#GhX{(LKMpY#gi&%s$H&V@xb^Ig(;AwRRm~ z2YqLfWDL7&pS8$~g?hBX_S4ZR5n;;y6_9nYpZ@ishD}Or?JGpq)k31!r~I7e0KG6F z`r5kBCMUwXgN=^i9r%Ol*=5mq~N8ZqLcC1x!)-<<;3+N8^6wg+?LeN#g z59J+xaaC_xh7)=c+_ZHOQ@*zU!?X*TIFEq0Cm2WYrrz;izM%o@{JnN5jT5w^Yu9`T z06x+vba*L$vaW?KaEUsOE+5C4dFdU{znQmqYM>+C$0#oL`~l^V+D^6Lh+N6&poX5? z?HD6e*OQz6h{m2NpXW%inU=vMc~WUu;jRRZ^Rzp!1Hr{zvp=?d+^NuuXlG{Nrh#|{ za9{9FC+$@kP?82ue1;;ZQ9^SLc!(O@mKuk_`;U`bx@I=o@HfZS`A_ogZWyBnw8Ogv HPNDw*`E~~T literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/library_api_test.dart b/mobile/openapi/test/library_api_test.dart index 85882edbd1f8b1422c839932fb782f36bcfc8129..20abcffe7036bd3f3953564476f90deecb3228a1 100644 GIT binary patch delta 88 zcmZqUS;M=4ZE`FV7pF~FVoqjCVoB=cKxPr`GAKtQxTGjEFI^!sMMnXqz$Y`QD6y#0 Yr6gYgq-^qXR#gs|`pNRl>XX>m04TQ|#{d8T delta 13 UcmZ3(+s3nijjdMGnyZ!z02^WhY5)KL diff --git a/mobile/openapi/test/validate_library_dto_test.dart b/mobile/openapi/test/validate_library_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..8d4922ee4a295726bd22816028ef461ee94dde66 GIT binary patch literal 767 zcmbtRO>5gg5WVYHOi#AKjoO?_5|(v~J60XY)d?<^n%P%h!HXz^C7-VsJ z^7=FzMsA%CWEvk#!wJ(=2HIb^kEZLz1p3)?P`bmNo*f4L z3x%lSA&%24@1md+!?hZ6!xR!dj+$9^6pi(+RhVqinK!RYwZ#y#?2r*A9acZEr8D1= z!;W+PPKK=~=SL8sDEV+a0|wJ!j-ZvmXS7_S(3-yo4EsWScJH}{9kQFw;CBX~_=X)? z;Iu|Xuo$6DXXlxM+u-|dIvET`K$AKA5lymdj&gA$fnEC+=%KQGWbZzrXrV5`w5A4o zD{zmt*N00z!q;yz^?|g58u@eq*%EX?`v|g!yCbc;gEPjxtI|?~f-bfxi!!M_9^xOz WI#jNiw*M38FO`3bqM=fx$LtBesQftq literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/validate_library_import_path_response_dto_test.dart b/mobile/openapi/test/validate_library_import_path_response_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..a118698a13adc1ef79d3753eded36fbebfc4af4c GIT binary patch literal 855 zcmb7CUu)Yi5P$clxIHC-wBGtOS_&&T9f7VMT{`HKdbQ7vm@H}1xg(5z_nqXFJuJ}d z!LrZ%{+?ZyMOnht{iZ(oyt-Z8U2RqsT(56dOQ>qNtvB$cuGZHN?*x{RpIQujd3O5Y ztVpBotqY_&7gVPWox^CyNUb7616f|a&U)}J@IeNeU$~3PkE{ay9CZ-7#X(MwLGDZ; zig<{J>5a3I(X!I580t<_$n<#7%xFtd8E1Qe@fNN6_+B+z46!Jtj4<=q{J@^hohOT4 zmvWt^*0YO$AWV_Vbi4wRX__O*O5i&h-6GSCehL_N>%4@o$lE)LUUE~#u6TC3g#opl zFW`3pAPSwfXh7#;4Z-D6Li5E%A@n}D(a&co#8I@_686T~`Wpvrdzf~SqKYKsN}}oV z1Mrr>2U$@JK>Z;B%FH^+)^2s@=81sr?-cpC7VR%L}%3su> fiL`jzdZPcA@4nzbG0I1N(_)W1&O+33#h2m_FaQ<} literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/validate_library_response_dto_test.dart b/mobile/openapi/test/validate_library_response_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..6750809066693890030faff3573a1dd32a0f9054 GIT binary patch literal 670 zcmaJ;U27XL5PZL1v3ar$Zm!K!Na|9<*}>rELU0PD6lHt2XN$;^D(xkhkpG^QTtjG0 z9`vzdW@lzqmPJ{@^lnvuxSCzhZl|kR1@pynHioK(>v{#>>S{6naU!sU{MusR$=T_r zvm%vxZCxPMxu802=mL7%M`{%r+LGn*q1IdP0v~0ddBR;(zGoHaPp^Z}EpBD^802IM zQJRN%oZdJa87(W_ilFW^g-nk}#f-KTm2q}0Fy5e5=O0zG!4QjL?-2$bn+IIeh4W<5 z>r&QfNMgrf_=oXol`gg#vyQS#GEmiD_N2goZqPD{kJQo1M z>$F7!I>%`UChyRPqsu}_>)?7n93;Vaz=JXT70EJ79r^P=&KCK5Zjd)`Jbr=wMACDF zVM`tMhT#!SFCU%f4nKd5#0TWeV#MVO@P@z#nP1=?+`s8^zxRT1XPX>5aV7eyuIk+9 LB?XgPK3|GgsypIi literal 0 HcmV?d00001 diff --git a/open-api/immich-openapi-specs.json b/open-api/immich-openapi-specs.json index 5dd394108..e15073e13 100644 --- a/open-api/immich-openapi-specs.json +++ b/open-api/immich-openapi-specs.json @@ -3618,6 +3618,58 @@ ] } }, + "/library/{id}/validate": { + "post": { + "operationId": "validate", + "parameters": [ + { + "name": "id", + "required": true, + "in": "path", + "schema": { + "format": "uuid", + "type": "string" + } + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateLibraryDto" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ValidateLibraryResponseDto" + } + } + }, + "description": "" + } + }, + "security": [ + { + "bearer": [] + }, + { + "cookie": [] + }, + { + "api_key": [] + } + ], + "tags": [ + "Library" + ] + } + }, "/oauth/authorize": { "post": { "operationId": "startOAuth", @@ -10406,6 +10458,52 @@ ], "type": "object" }, + "ValidateLibraryDto": { + "properties": { + "exclusionPatterns": { + "items": { + "type": "string" + }, + "type": "array" + }, + "importPaths": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "type": "object" + }, + "ValidateLibraryImportPathResponseDto": { + "properties": { + "importPath": { + "type": "string" + }, + "isValid": { + "default": false, + "type": "boolean" + }, + "message": { + "type": "string" + } + }, + "required": [ + "importPath" + ], + "type": "object" + }, + "ValidateLibraryResponseDto": { + "properties": { + "importPaths": { + "items": { + "$ref": "#/components/schemas/ValidateLibraryImportPathResponseDto" + }, + "type": "array" + } + }, + "type": "object" + }, "VideoCodec": { "enum": [ "h264", diff --git a/open-api/typescript-sdk/axios-client/api.ts b/open-api/typescript-sdk/axios-client/api.ts index 5d3770690..49cf286f3 100644 --- a/open-api/typescript-sdk/axios-client/api.ts +++ b/open-api/typescript-sdk/axios-client/api.ts @@ -5316,6 +5316,63 @@ export interface ValidateAccessTokenResponseDto { */ 'authStatus': boolean; } +/** + * + * @export + * @interface ValidateLibraryDto + */ +export interface ValidateLibraryDto { + /** + * + * @type {Array} + * @memberof ValidateLibraryDto + */ + 'exclusionPatterns'?: Array; + /** + * + * @type {Array} + * @memberof ValidateLibraryDto + */ + 'importPaths'?: Array; +} +/** + * + * @export + * @interface ValidateLibraryImportPathResponseDto + */ +export interface ValidateLibraryImportPathResponseDto { + /** + * + * @type {string} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'importPath': string; + /** + * + * @type {boolean} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'isValid'?: boolean; + /** + * + * @type {string} + * @memberof ValidateLibraryImportPathResponseDto + */ + 'message'?: string; +} +/** + * + * @export + * @interface ValidateLibraryResponseDto + */ +export interface ValidateLibraryResponseDto { + /** + * + * @type {Array} + * @memberof ValidateLibraryResponseDto + */ + 'importPaths'?: Array; +} /** * * @export @@ -12813,6 +12870,54 @@ export const LibraryApiAxiosParamCreator = function (configuration?: Configurati localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; localVarRequestOptions.data = serializeDataIfNeeded(updateLibraryDto, localVarRequestOptions, configuration) + return { + url: toPathString(localVarUrlObj), + options: localVarRequestOptions, + }; + }, + /** + * + * @param {string} id + * @param {ValidateLibraryDto} validateLibraryDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + validate: async (id: string, validateLibraryDto: ValidateLibraryDto, options: RawAxiosRequestConfig = {}): Promise => { + // verify required parameter 'id' is not null or undefined + assertParamExists('validate', 'id', id) + // verify required parameter 'validateLibraryDto' is not null or undefined + assertParamExists('validate', 'validateLibraryDto', validateLibraryDto) + const localVarPath = `/library/{id}/validate` + .replace(`{${"id"}}`, encodeURIComponent(String(id))); + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'POST', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + + + localVarHeaderParameter['Content-Type'] = 'application/json'; + + setSearchParams(localVarUrlObj, localVarQueryParameter); + let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; + localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; + localVarRequestOptions.data = serializeDataIfNeeded(validateLibraryDto, localVarRequestOptions, configuration) + return { url: toPathString(localVarUrlObj), options: localVarRequestOptions, @@ -12925,6 +13030,19 @@ export const LibraryApiFp = function(configuration?: Configuration) { const operationBasePath = operationServerMap['LibraryApi.updateLibrary']?.[index]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); }, + /** + * + * @param {string} id + * @param {ValidateLibraryDto} validateLibraryDto + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async validate(id: string, validateLibraryDto: ValidateLibraryDto, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise> { + const localVarAxiosArgs = await localVarAxiosParamCreator.validate(id, validateLibraryDto, options); + const index = configuration?.serverIndex ?? 0; + const operationBasePath = operationServerMap['LibraryApi.validate']?.[index]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); + }, } }; @@ -13006,6 +13124,15 @@ export const LibraryApiFactory = function (configuration?: Configuration, basePa updateLibrary(requestParameters: LibraryApiUpdateLibraryRequest, options?: RawAxiosRequestConfig): AxiosPromise { return localVarFp.updateLibrary(requestParameters.id, requestParameters.updateLibraryDto, options).then((request) => request(axios, basePath)); }, + /** + * + * @param {LibraryApiValidateRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + validate(requestParameters: LibraryApiValidateRequest, options?: RawAxiosRequestConfig): AxiosPromise { + return localVarFp.validate(requestParameters.id, requestParameters.validateLibraryDto, options).then((request) => request(axios, basePath)); + }, }; }; @@ -13121,6 +13248,27 @@ export interface LibraryApiUpdateLibraryRequest { readonly updateLibraryDto: UpdateLibraryDto } +/** + * Request parameters for validate operation in LibraryApi. + * @export + * @interface LibraryApiValidateRequest + */ +export interface LibraryApiValidateRequest { + /** + * + * @type {string} + * @memberof LibraryApiValidate + */ + readonly id: string + + /** + * + * @type {ValidateLibraryDto} + * @memberof LibraryApiValidate + */ + readonly validateLibraryDto: ValidateLibraryDto +} + /** * LibraryApi - object-oriented interface * @export @@ -13214,6 +13362,17 @@ export class LibraryApi extends BaseAPI { public updateLibrary(requestParameters: LibraryApiUpdateLibraryRequest, options?: RawAxiosRequestConfig) { return LibraryApiFp(this.configuration).updateLibrary(requestParameters.id, requestParameters.updateLibraryDto, options).then((request) => request(this.axios, this.basePath)); } + + /** + * + * @param {LibraryApiValidateRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof LibraryApi + */ + public validate(requestParameters: LibraryApiValidateRequest, options?: RawAxiosRequestConfig) { + return LibraryApiFp(this.configuration).validate(requestParameters.id, requestParameters.validateLibraryDto, options).then((request) => request(this.axios, this.basePath)); + } } diff --git a/open-api/typescript-sdk/fetch-client.ts b/open-api/typescript-sdk/fetch-client.ts index 0f42ff9178126c1734df637cd23ece577a0b3ce4..40600e56fa68e2648b8241d549451d832de47b6f 100644 GIT binary patch delta 299 zcmaFApXJI?mJJ(pSi=%?GE*kEYXwaHC%`WW<|US-`eY^*B^FhB<`(1^l>{V~WK0(1 z5;4t$bFCDLONuh{(yh4^fFQFNtk2#`At^sUCp9q-BAlCAT%4FbInhv>17_~z98GB> zgfYcHEifqzD}z#t3-a@dQ(a2(CkKiMZ{DC2Fjcq={I5{hv}SLm|-?)nFt4*zx#mtrbF*z E0p$O2NdN!< delta 19 bcmca|l;! { + let IMMICH_TEST_ASSET_PATH: string = ''; + + if (process.env.IMMICH_TEST_ASSET_PATH === undefined) { + IMMICH_TEST_ASSET_PATH = path.normalize(`${__dirname}/../../test/assets/`); + process.env.IMMICH_TEST_ASSET_PATH = IMMICH_TEST_ASSET_PATH; + } else { + IMMICH_TEST_ASSET_PATH = process.env.IMMICH_TEST_ASSET_PATH; + } + const pg = await new PostgreSqlContainer('tensorchord/pgvecto-rs:pg14-v0.2.0') .withDatabase('immich') .withUsername('postgres') @@ -11,6 +21,9 @@ export default async () => { process.env.DB_URL = pg.getConnectionUri(); process.env.NODE_ENV = 'development'; - process.env.LOG_LEVEL = 'fatal'; process.env.TZ = 'Z'; + + if (process.env.LOG_LEVEL === undefined) { + process.env.LOG_LEVEL = 'fatal'; + } }; diff --git a/server/e2e/api/specs/library.e2e-spec.ts b/server/e2e/api/specs/library.e2e-spec.ts index 75c973466..5be9e3035 100644 --- a/server/e2e/api/specs/library.e2e-spec.ts +++ b/server/e2e/api/specs/library.e2e-spec.ts @@ -2,6 +2,7 @@ import { LibraryResponseDto, LoginResponseDto } from '@app/domain'; import { LibraryController } from '@app/immich'; import { LibraryType } from '@app/infra/entities'; import { errorStub, userDto, uuidStub } from '@test/fixtures'; +import { IMMICH_TEST_ASSET_TEMP_PATH, restoreTempFolder } from 'src/test-utils/utils'; import request from 'supertest'; import { api } from '../../client'; import { testApp } from '../utils'; @@ -20,6 +21,7 @@ describe(`${LibraryController.name} (e2e)`, () => { }); beforeEach(async () => { + await restoreTempFolder(); await testApp.reset(); await api.authApi.adminSignUp(server); admin = await api.authApi.adminLogin(server); @@ -247,15 +249,16 @@ describe(`${LibraryController.name} (e2e)`, () => { }); it('should change the import paths', async () => { + await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, IMMICH_TEST_ASSET_TEMP_PATH); const { status, body } = await request(server) .put(`/library/${library.id}`) .set('Authorization', `Bearer ${admin.accessToken}`) - .send({ importPaths: ['/path/to/import'] }); + .send({ importPaths: [IMMICH_TEST_ASSET_TEMP_PATH] }); expect(status).toBe(200); expect(body).toEqual( expect.objectContaining({ - importPaths: ['/path/to/import'], + importPaths: [IMMICH_TEST_ASSET_TEMP_PATH], }), ); }); @@ -435,4 +438,93 @@ describe(`${LibraryController.name} (e2e)`, () => { expect(body).toEqual(errorStub.unauthorized); }); }); + + describe('POST /library/:id/validate', () => { + it('should require authentication', async () => { + const { status, body } = await request(server).post(`/library/${uuidStub.notFound}/validate`).send({}); + + expect(status).toBe(401); + expect(body).toEqual(errorStub.unauthorized); + }); + + describe('Validate import path', () => { + let library: LibraryResponseDto; + + beforeEach(async () => { + // Create an external library with default settings + library = await api.libraryApi.create(server, admin.accessToken, { type: LibraryType.EXTERNAL }); + }); + + it('should fail with no external path set', async () => { + const { status, body } = await request(server) + .post(`/library/${library.id}/validate`) + .set('Authorization', `Bearer ${admin.accessToken}`) + + .send({ importPaths: [] }); + + expect(status).toBe(400); + expect(body).toEqual(errorStub.badRequest('User has no external path set')); + }); + + describe('With external path set', () => { + beforeEach(async () => { + await api.userApi.setExternalPath(server, admin.accessToken, admin.userId, IMMICH_TEST_ASSET_TEMP_PATH); + }); + + it('should pass with no import paths', async () => { + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { importPaths: [] }); + expect(response.importPaths).toEqual([]); + }); + + it('should not allow paths outside of the external path', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/../`; + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Not contained in user's external path`, + }); + }); + + it('should fail if path does not exist', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/does/not/exist`; + + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Path does not exist (ENOENT)`, + }); + }); + + it('should fail if path is a file', async () => { + const pathToTest = `${IMMICH_TEST_ASSET_TEMP_PATH}/does/not/exist`; + + const response = await api.libraryApi.validate(server, admin.accessToken, library.id, { + importPaths: [pathToTest], + }); + + expect(response.importPaths?.length).toEqual(1); + const pathResponse = response?.importPaths?.at(0); + + expect(pathResponse).toEqual({ + importPath: pathToTest, + isValid: false, + message: `Path does not exist (ENOENT)`, + }); + }); + }); + }); + }); }); diff --git a/server/e2e/client/library-api.ts b/server/e2e/client/library-api.ts index c40f00545..9f2d0b77e 100644 --- a/server/e2e/client/library-api.ts +++ b/server/e2e/client/library-api.ts @@ -4,6 +4,8 @@ import { LibraryStatsResponseDto, ScanLibraryDto, UpdateLibraryDto, + ValidateLibraryDto, + ValidateLibraryResponseDto, } from '@app/domain'; import request from 'supertest'; @@ -58,4 +60,12 @@ export const libraryApi = { expect(status).toBe(200); return body as LibraryResponseDto; }, + validate: async (server: any, accessToken: string, id: string, data: ValidateLibraryDto) => { + const { body, status } = await request(server) + .post(`/library/${id}/validate`) + .set('Authorization', `Bearer ${accessToken}`) + .send(data); + expect(status).toBe(200); + return body as ValidateLibraryResponseDto; + }, }; diff --git a/server/src/domain/library/library.dto.ts b/server/src/domain/library/library.dto.ts index 638841ec6..db6119d4d 100644 --- a/server/src/domain/library/library.dto.ts +++ b/server/src/domain/library/library.dto.ts @@ -1,6 +1,6 @@ import { LibraryEntity, LibraryType } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; -import { ArrayUnique, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; +import { ArrayMaxSize, ArrayUnique, IsBoolean, IsEnum, IsNotEmpty, IsOptional, IsString } from 'class-validator'; import { ValidateUUID } from '../domain.util'; export class CreateLibraryDto { @@ -21,12 +21,14 @@ export class CreateLibraryDto { @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) importPaths?: string[]; @IsOptional() @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) exclusionPatterns?: string[]; @IsOptional() @@ -48,12 +50,14 @@ export class UpdateLibraryDto { @IsString({ each: true }) @IsNotEmpty({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) importPaths?: string[]; @IsOptional() @IsNotEmpty({ each: true }) @IsString({ each: true }) @ArrayUnique() + @ArrayMaxSize(128) exclusionPatterns?: string[]; } @@ -63,6 +67,32 @@ export class CrawlOptionsDto { exclusionPatterns?: string[]; } +export class ValidateLibraryDto { + @IsOptional() + @IsString({ each: true }) + @IsNotEmpty({ each: true }) + @ArrayUnique() + @ArrayMaxSize(128) + importPaths?: string[]; + + @IsOptional() + @IsNotEmpty({ each: true }) + @IsString({ each: true }) + @ArrayUnique() + @ArrayMaxSize(128) + exclusionPatterns?: string[]; +} + +export class ValidateLibraryResponseDto { + importPaths?: ValidateLibraryImportPathResponseDto[]; +} + +export class ValidateLibraryImportPathResponseDto { + importPath!: string; + isValid?: boolean = false; + message?: string; +} + export class LibrarySearchDto { @ValidateUUID({ optional: true }) userId?: string; diff --git a/server/src/domain/library/library.service.spec.ts b/server/src/domain/library/library.service.spec.ts index c86aa4a4e..cafe70b4d 100644 --- a/server/src/domain/library/library.service.spec.ts +++ b/server/src/domain/library/library.service.spec.ts @@ -54,12 +54,6 @@ describe(LibraryService.name, () => { cryptoMock = newCryptoRepositoryMock(); storageMock = newStorageRepositoryMock(); - storageMock.stat.mockResolvedValue({ - size: 100, - mtime: new Date('2023-01-01'), - ctime: new Date('2023-01-01'), - } as Stats); - // Always validate owner access for library. accessMock.library.checkOwnerAccess.mockImplementation(async (_, libraryIds) => libraryIds); @@ -270,6 +264,39 @@ describe(LibraryService.name, () => { await expect(sut.handleQueueAssetRefresh(mockLibraryJob)).resolves.toBe(false); }); + + it('should ignore import paths that do not exist', async () => { + storageMock.stat.mockImplementation((path): Promise => { + if (path === libraryStub.externalLibraryWithImportPaths1.importPaths[0]) { + const error = { code: 'ENOENT' } as any; + throw error; + } + return Promise.resolve({ + isDirectory: () => true, + } as Stats); + }); + + storageMock.checkFileExists.mockResolvedValue(true); + + const mockLibraryJob: ILibraryRefreshJob = { + id: libraryStub.externalLibraryWithImportPaths1.id, + refreshModifiedFiles: false, + refreshAllFiles: false, + }; + + libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); + storageMock.crawl.mockResolvedValue([]); + assetMock.getByLibraryId.mockResolvedValue([]); + libraryMock.getOnlineAssetPaths.mockResolvedValue([]); + userMock.get.mockResolvedValue(userStub.externalPathRoot); + + await sut.handleQueueAssetRefresh(mockLibraryJob); + + expect(storageMock.crawl).toHaveBeenCalledWith({ + pathsToCrawl: [libraryStub.externalLibraryWithImportPaths1.importPaths[1]], + exclusionPatterns: [], + }); + }); }); describe('handleAssetRefresh', () => { @@ -278,6 +305,12 @@ describe(LibraryService.name, () => { beforeEach(() => { mockUser = userStub.externalPath1; userMock.get.mockResolvedValue(mockUser); + + storageMock.stat.mockResolvedValue({ + size: 100, + mtime: new Date('2023-01-01'), + ctime: new Date('2023-01-01'), + } as Stats); }); it('should reject an unknown file extension', async () => { @@ -1104,13 +1137,19 @@ describe(LibraryService.name, () => { libraryMock.update.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); libraryMock.get.mockResolvedValue(libraryStub.externalLibraryWithImportPaths1); - await expect(sut.update(authStub.admin, authStub.admin.user.id, { importPaths: ['/foo'] })).resolves.toEqual( - mapLibrary(libraryStub.externalLibraryWithImportPaths1), - ); + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(true); + + await expect( + sut.update(authStub.external1, authStub.external1.user.id, { importPaths: ['/data/user1/foo'] }), + ).resolves.toEqual(mapLibrary(libraryStub.externalLibraryWithImportPaths1)); expect(libraryMock.update).toHaveBeenCalledWith( expect.objectContaining({ - id: authStub.admin.user.id, + id: authStub.external1.user.id, }), ); expect(storageMock.watch).toHaveBeenCalledWith( @@ -1142,7 +1181,7 @@ describe(LibraryService.name, () => { }); }); - describe('watchAll new', () => { + describe('watchAll', () => { describe('watching disabled', () => { beforeEach(async () => { configMock.load.mockResolvedValue(systemConfigStub.libraryWatchDisabled); @@ -1523,4 +1562,121 @@ describe(LibraryService.name, () => { ]); }); }); + + describe('validate', () => { + it('should validate directory', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(true); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: true, + message: undefined, + }, + ]); + }); + + it('should error when no external path is set', async () => { + await expect( + sut.validate(authStub.admin, libraryStub.externalLibrary1.id, { importPaths: ['/photos'] }), + ).rejects.toBeInstanceOf(BadRequestException); + }); + + it('should detect when path is outside external path', async () => { + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user2'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user2', + isValid: false, + message: "Not contained in user's external path", + }, + ]); + }); + + it('should detect when path does not exist', async () => { + storageMock.stat.mockImplementation(() => { + const error = { code: 'ENOENT' } as any; + throw error; + }); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Path does not exist (ENOENT)', + }, + ]); + }); + + it('should detect when path is not a directory', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => false, + } as Stats); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/file'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/file', + isValid: false, + message: 'Not a directory', + }, + ]); + }); + + it('should return an unknown exception from stat', async () => { + storageMock.stat.mockImplementation(() => { + throw new Error('Unknown error'); + }); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Error: Unknown error', + }, + ]); + }); + + it('should detect when access rights are missing', async () => { + storageMock.stat.mockResolvedValue({ + isDirectory: () => true, + } as Stats); + + storageMock.checkFileExists.mockResolvedValue(false); + + const result = await sut.validate(authStub.external1, libraryStub.externalLibraryWithImportPaths1.id, { + importPaths: ['/data/user1/'], + }); + + expect(result.importPaths).toEqual([ + { + importPath: '/data/user1/', + isValid: false, + message: 'Lacking read permission for folder', + }, + ]); + }); + }); }); diff --git a/server/src/domain/library/library.service.ts b/server/src/domain/library/library.service.ts index 3454f3547..7c961963e 100644 --- a/server/src/domain/library/library.service.ts +++ b/server/src/domain/library/library.service.ts @@ -30,6 +30,9 @@ import { LibraryStatsResponseDto, ScanLibraryDto, UpdateLibraryDto, + ValidateLibraryDto, + ValidateLibraryImportPathResponseDto, + ValidateLibraryResponseDto, mapLibrary, } from './library.dto'; @@ -270,10 +273,81 @@ export class LibraryService extends EventEmitter { ); } + private async validateImportPath(importPath: string): Promise { + const validation = new ValidateLibraryImportPathResponseDto(); + validation.importPath = importPath; + + try { + const stat = await this.storageRepository.stat(importPath); + + if (!stat.isDirectory()) { + validation.message = 'Not a directory'; + return validation; + } + } catch (error: any) { + if (error.code === 'ENOENT') { + validation.message = 'Path does not exist (ENOENT)'; + return validation; + } + validation.message = String(error); + return validation; + } + + const access = await this.storageRepository.checkFileExists(importPath, R_OK); + + if (!access) { + validation.message = 'Lacking read permission for folder'; + return validation; + } + + validation.isValid = true; + return validation; + } + + public async validate(auth: AuthDto, id: string, dto: ValidateLibraryDto): Promise { + await this.access.requirePermission(auth, Permission.LIBRARY_UPDATE, id); + + if (!auth.user.externalPath) { + throw new BadRequestException('User has no external path set'); + } + + const response = new ValidateLibraryResponseDto(); + + if (dto.importPaths) { + response.importPaths = await Promise.all( + dto.importPaths.map(async (importPath) => { + const normalizedPath = path.normalize(importPath); + + if (!this.isInExternalPath(normalizedPath, auth.user.externalPath)) { + const validation = new ValidateLibraryImportPathResponseDto(); + validation.importPath = importPath; + validation.message = `Not contained in user's external path`; + return validation; + } + + return await this.validateImportPath(importPath); + }), + ); + } + + return response; + } + async update(auth: AuthDto, id: string, dto: UpdateLibraryDto): Promise { await this.access.requirePermission(auth, Permission.LIBRARY_UPDATE, id); const library = await this.repository.update({ id, ...dto }); + if (dto.importPaths) { + const validation = await this.validate(auth, id, { importPaths: dto.importPaths }); + if (validation.importPaths) { + for (const path of validation.importPaths) { + if (!path.isValid) { + throw new BadRequestException(`Invalid import path: ${path.message}`); + } + } + } + } + if (dto.importPaths || dto.exclusionPatterns) { // Re-watch library to use new paths and/or exclusion patterns await this.watch(id); @@ -509,6 +583,14 @@ export class LibraryService extends EventEmitter { return true; } + // Check if a given path is in a user's external path. Both arguments are assumed to be normalized + private isInExternalPath(filePath: string, externalPath: string | null): boolean { + if (externalPath === null) { + return false; + } + return filePath.startsWith(externalPath); + } + async handleQueueAssetRefresh(job: ILibraryRefreshJob): Promise { const library = await this.repository.get(job.id); if (!library || library.type !== LibraryType.EXTERNAL) { @@ -523,17 +605,31 @@ export class LibraryService extends EventEmitter { } this.logger.verbose(`Refreshing library: ${job.id}`); + + const pathValidation = await Promise.all( + library.importPaths.map(async (importPath) => await this.validateImportPath(importPath)), + ); + + const validImportPaths = pathValidation + .map((validation) => { + if (!validation.isValid) { + this.logger.error(`Skipping invalid import path: ${validation.importPath}. Reason: ${validation.message}`); + } + return validation; + }) + .filter((validation) => validation.isValid) + .map((validation) => validation.importPath); + const rawPaths = await this.storageRepository.crawl({ - pathsToCrawl: library.importPaths, + pathsToCrawl: validImportPaths, exclusionPatterns: library.exclusionPatterns, }); const crawledAssetPaths = rawPaths + // Normalize file paths. This is important to prevent security issues like path traversal .map((filePath) => path.normalize(filePath)) - .filter((assetPath) => - // Filter out paths that are not within the user's external path - assetPath.match(new RegExp(`^${user.externalPath}`)), - ) as string[]; + // Filter out paths that are not within the user's external path + .filter((assetPath) => this.isInExternalPath(assetPath, user.externalPath)) as string[]; this.logger.debug(`Found ${crawledAssetPaths.length} asset(s) when crawling import paths ${library.importPaths}`); const assetsInLibrary = await this.assetRepository.getByLibraryId([job.id]); diff --git a/server/src/immich/controllers/library.controller.ts b/server/src/immich/controllers/library.controller.ts index 56dd5d8e7..173e63211 100644 --- a/server/src/immich/controllers/library.controller.ts +++ b/server/src/immich/controllers/library.controller.ts @@ -6,8 +6,10 @@ import { LibraryResponseDto as ResponseDto, ScanLibraryDto, UpdateLibraryDto as UpdateDto, + ValidateLibraryDto, + ValidateLibraryResponseDto, } from '@app/domain'; -import { Body, Controller, Delete, Get, Param, Post, Put } from '@nestjs/common'; +import { Body, Controller, Delete, Get, HttpCode, Param, Post, Put } from '@nestjs/common'; import { ApiTags } from '@nestjs/swagger'; import { Auth, Authenticated } from '../app.guard'; import { UseValidation } from '../app.utils'; @@ -40,6 +42,16 @@ export class LibraryController { return this.service.get(auth, id); } + @Post(':id/validate') + @HttpCode(200) + validate( + @Auth() auth: AuthDto, + @Param() { id }: UUIDParamDto, + @Body() dto: ValidateLibraryDto, + ): Promise { + return this.service.validate(auth, id, dto); + } + @Delete(':id') deleteLibrary(@Auth() auth: AuthDto, @Param() { id }: UUIDParamDto): Promise { return this.service.delete(auth, id); diff --git a/server/src/infra/infra.utils.ts b/server/src/infra/infra.utils.ts index d9eb82363..ab7d74431 100644 --- a/server/src/infra/infra.utils.ts +++ b/server/src/infra/infra.utils.ts @@ -164,7 +164,21 @@ export function searchAssetBuilder( builder.andWhere(_.omitBy(path, _.isUndefined)); const status = _.pick(options, ['isExternal', 'isFavorite', 'isOffline', 'isReadOnly', 'isVisible', 'type']); - const { isArchived, isEncoded, isMotion, withArchived } = options; + const { + isArchived, + isEncoded, + isMotion, + withArchived, + isNotInAlbum, + withFaces, + withPeople, + withSmartInfo, + personIds, + withExif, + withStacked, + trashedAfter, + trashedBefore, + } = options; builder.andWhere( _.omitBy( { @@ -177,38 +191,38 @@ export function searchAssetBuilder( ), ); - if (options.isNotInAlbum) { + if (isNotInAlbum) { builder .leftJoin(`${builder.alias}.albums`, 'albums') .andWhere('albums.id IS NULL') .andWhere(`${builder.alias}.isVisible = true`); } - if (options.withFaces || options.withPeople) { + if (withFaces || withPeople) { builder.leftJoinAndSelect(`${builder.alias}.faces`, 'faces'); } - if (options.withPeople) { + if (withPeople) { builder.leftJoinAndSelect(`${builder.alias}.person`, 'person'); } - if (options.withSmartInfo) { + if (withSmartInfo) { builder.leftJoinAndSelect(`${builder.alias}.smartInfo`, 'smartInfo'); } - if (options.personIds && options.personIds.length > 0) { + if (personIds && personIds.length > 0) { builder .leftJoin(`${builder.alias}.faces`, 'faces') - .andWhere('faces.personId IN (:...personIds)', { personIds: options.personIds }) + .andWhere('faces.personId IN (:...personIds)', { personIds: personIds }) .addGroupBy(`${builder.alias}.id`) - .having('COUNT(faces.id) = :personCount', { personCount: options.personIds.length }); + .having('COUNT(faces.id) = :personCount', { personCount: personIds.length }); - if (options.withExif) { + if (withExif) { builder.addGroupBy('exifInfo.assetId'); } } - if (options.withStacked) { + if (withStacked) { builder .leftJoinAndSelect(`${builder.alias}.stack`, 'stack') .leftJoinAndSelect('stack.assets', 'stackedAssets') @@ -217,8 +231,7 @@ export function searchAssetBuilder( ); } - const withDeleted = - options.withDeleted ?? (options.trashedAfter !== undefined || options.trashedBefore !== undefined); + const withDeleted = options.withDeleted ?? (trashedAfter !== undefined || trashedBefore !== undefined); if (withDeleted) { builder.withDeleted(); } diff --git a/server/src/infra/repositories/filesystem.provider.ts b/server/src/infra/repositories/filesystem.provider.ts index ed009da76..3ffcd8111 100644 --- a/server/src/infra/repositories/filesystem.provider.ts +++ b/server/src/infra/repositories/filesystem.provider.ts @@ -12,12 +12,24 @@ import archiver from 'archiver'; import chokidar, { WatchOptions } from 'chokidar'; import { glob } from 'glob'; import { constants, createReadStream, existsSync, mkdirSync } from 'node:fs'; -import fs, { copyFile, readdir, rename, utimes, writeFile } from 'node:fs/promises'; +import fs, { copyFile, readdir, rename, stat, utimes, writeFile } from 'node:fs/promises'; import path from 'node:path'; export class FilesystemProvider implements IStorageRepository { private logger = new ImmichLogger(FilesystemProvider.name); + readdir = readdir; + + copyFile = copyFile; + + stat = stat; + + writeFile = writeFile; + + rename = rename; + + utimes = utimes; + createZipStream(): ImmichZipStream { const archive = archiver('zip', { store: true }); @@ -50,14 +62,6 @@ export class FilesystemProvider implements IStorageRepository { } } - writeFile = writeFile; - - rename = rename; - - copyFile = copyFile; - - utimes = utimes; - async checkFileExists(filepath: string, mode = constants.F_OK): Promise { try { await fs.access(filepath, mode); @@ -79,8 +83,6 @@ export class FilesystemProvider implements IStorageRepository { } } - stat = fs.stat; - async unlinkDir(folder: string, options: { recursive?: boolean; force?: boolean }) { await fs.rm(folder, options); } @@ -146,6 +148,4 @@ export class FilesystemProvider implements IStorageRepository { return () => watcher.close(); } - - readdir = readdir; } diff --git a/server/test/fixtures/user.stub.ts b/server/test/fixtures/user.stub.ts index 770e61530..1b8a3b3fe 100644 --- a/server/test/fixtures/user.stub.ts +++ b/server/test/fixtures/user.stub.ts @@ -140,6 +140,25 @@ export const userStub = { quotaSizeInBytes: null, quotaUsageInBytes: 0, }), + externalPathRoot: Object.freeze({ + ...authStub.user1.user, + password: 'immich_password', + name: 'immich_name', + storageLabel: 'label-1', + externalPath: '/', + oauthId: '', + shouldChangePassword: false, + profileImagePath: '', + createdAt: new Date('2021-01-01'), + deletedAt: null, + updatedAt: new Date('2021-01-01'), + tags: [], + assets: [], + memoriesEnabled: true, + avatarColor: UserAvatarColor.PRIMARY, + quotaSizeInBytes: null, + quotaUsageInBytes: 0, + }), profilePath: Object.freeze({ ...authStub.user1.user, password: 'immich_password', diff --git a/web/src/lib/components/forms/library-import-paths-form.svelte b/web/src/lib/components/forms/library-import-paths-form.svelte index fddb4b986..17903526b 100644 --- a/web/src/lib/components/forms/library-import-paths-form.svelte +++ b/web/src/lib/components/forms/library-import-paths-form.svelte @@ -1,13 +1,15 @@