From 97e52c5156c40b733a7cb9bdc104eb560e8d0636 Mon Sep 17 00:00:00 2001 From: shenlong <139912620+shenlong-tanwen@users.noreply.github.com> Date: Fri, 4 Apr 2025 01:12:35 +0530 Subject: [PATCH] refactor(mobile): device asset entity to use modified time (#17064) * refactor: device asset entity to use modified time * chore: cleanup * refactor: remove album media dependency from hashservice * refactor: return updated copy of asset * add hash service tests * chore: rename hash batch constants * chore: log the number of assets processed during migration * chore: more logs * refactor: use lookup and more tests * use sort approach * refactor hash service to use for loop instead * refactor: rename to getByIds --------- Co-authored-by: shenlong-tanwen <139912620+shalong-tanwen@users.noreply.github.com> --- mobile/analysis_options.yaml | 2 +- mobile/lib/constants/constants.dart | 3 + .../interfaces/device_asset.interface.dart | 12 + .../lib/domain/models/device_asset.model.dart | 44 ++ mobile/lib/entities/asset.entity.dart | 16 +- .../entities/device_asset.entity.dart | 36 ++ .../entities/device_asset.entity.g.dart | Bin 0 -> 25405 bytes .../repositories/device_asset.repository.dart | 37 ++ mobile/lib/interfaces/asset.interface.dart | 5 - .../infrastructure/device_asset.provider.dart | 8 + mobile/lib/repositories/asset.repository.dart | 18 - mobile/lib/services/hash.service.dart | 291 ++++++------ mobile/lib/services/sync.service.dart | 50 ++- mobile/lib/utils/bootstrap.dart | 2 + mobile/lib/utils/diff.dart | 14 + mobile/lib/utils/migration.dart | 97 +++- mobile/pubspec.lock | 2 +- mobile/pubspec.yaml | 1 + .../domain/services/hash_service_test.dart | 425 ++++++++++++++++++ mobile/test/fixtures/asset.stub.dart | 19 +- .../test/infrastructure/repository.mock.dart | 4 + mobile/test/service.mocks.dart | 3 + mobile/test/test_utils.dart | 2 + 23 files changed, 906 insertions(+), 185 deletions(-) create mode 100644 mobile/lib/domain/interfaces/device_asset.interface.dart create mode 100644 mobile/lib/domain/models/device_asset.model.dart create mode 100644 mobile/lib/infrastructure/entities/device_asset.entity.dart create mode 100644 mobile/lib/infrastructure/entities/device_asset.entity.g.dart create mode 100644 mobile/lib/infrastructure/repositories/device_asset.repository.dart create mode 100644 mobile/lib/providers/infrastructure/device_asset.provider.dart create mode 100644 mobile/test/domain/services/hash_service_test.dart diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml index 07c6f65b7..04f314590 100644 --- a/mobile/analysis_options.yaml +++ b/mobile/analysis_options.yaml @@ -61,6 +61,7 @@ custom_lint: # refactor to make the providers and services testable - lib/providers/backup/{backup,manual_upload}.provider.dart # uses only PMProgressHandler - lib/services/{background,backup}.service.dart # uses only PMProgressHandler + - test/**.dart - import_rule_isar: message: isar must only be used in entities and repositories restrict: package:isar @@ -150,7 +151,6 @@ dart_code_metrics: - avoid-unnecessary-continue - avoid-unnecessary-nullable-return-type: false - binary-expression-operand-order - - move-variable-outside-iteration - pattern-fields-ordering - prefer-abstract-final-static-class - prefer-commenting-future-delayed diff --git a/mobile/lib/constants/constants.dart b/mobile/lib/constants/constants.dart index 868b036d1..83d540d54 100644 --- a/mobile/lib/constants/constants.dart +++ b/mobile/lib/constants/constants.dart @@ -4,3 +4,6 @@ const double downloadFailed = -2; // Number of log entries to retain on app start const int kLogTruncateLimit = 250; + +const int kBatchHashFileLimit = 128; +const int kBatchHashSizeLimit = 1024 * 1024 * 1024; // 1GB diff --git a/mobile/lib/domain/interfaces/device_asset.interface.dart b/mobile/lib/domain/interfaces/device_asset.interface.dart new file mode 100644 index 000000000..1df8cc225 --- /dev/null +++ b/mobile/lib/domain/interfaces/device_asset.interface.dart @@ -0,0 +1,12 @@ +import 'dart:async'; + +import 'package:immich_mobile/domain/interfaces/db.interface.dart'; +import 'package:immich_mobile/domain/models/device_asset.model.dart'; + +abstract interface class IDeviceAssetRepository implements IDatabaseRepository { + Future updateAll(List assetHash); + + Future> getByIds(List localIds); + + Future deleteIds(List ids); +} diff --git a/mobile/lib/domain/models/device_asset.model.dart b/mobile/lib/domain/models/device_asset.model.dart new file mode 100644 index 000000000..2ec56b0d8 --- /dev/null +++ b/mobile/lib/domain/models/device_asset.model.dart @@ -0,0 +1,44 @@ +import 'dart:typed_data'; + +class DeviceAsset { + final String assetId; + final Uint8List hash; + final DateTime modifiedTime; + + const DeviceAsset({ + required this.assetId, + required this.hash, + required this.modifiedTime, + }); + + @override + bool operator ==(covariant DeviceAsset other) { + if (identical(this, other)) return true; + + return other.assetId == assetId && + other.hash == hash && + other.modifiedTime == modifiedTime; + } + + @override + int get hashCode { + return assetId.hashCode ^ hash.hashCode ^ modifiedTime.hashCode; + } + + @override + String toString() { + return 'DeviceAsset(assetId: $assetId, hash: $hash, modifiedTime: $modifiedTime)'; + } + + DeviceAsset copyWith({ + String? assetId, + Uint8List? hash, + DateTime? modifiedTime, + }) { + return DeviceAsset( + assetId: assetId ?? this.assetId, + hash: hash ?? this.hash, + modifiedTime: modifiedTime ?? this.modifiedTime, + ); + } +} diff --git a/mobile/lib/entities/asset.entity.dart b/mobile/lib/entities/asset.entity.dart index 048068ad3..084cd1ee5 100644 --- a/mobile/lib/entities/asset.entity.dart +++ b/mobile/lib/entities/asset.entity.dart @@ -6,6 +6,7 @@ import 'package:immich_mobile/extensions/string_extensions.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart' as entity; import 'package:immich_mobile/infrastructure/utils/exif.converter.dart'; +import 'package:immich_mobile/utils/diff.dart'; import 'package:immich_mobile/utils/hash.dart'; import 'package:isar/isar.dart'; import 'package:openapi/api.dart'; @@ -358,7 +359,7 @@ class Asset { // take most values from newer asset // keep vales that can never be set by the asset not in DB if (a.isRemote) { - return a._copyWith( + return a.copyWith( id: id, localId: localId, width: a.width ?? width, @@ -366,7 +367,7 @@ class Asset { exifInfo: a.exifInfo?.copyWith(assetId: id) ?? exifInfo, ); } else if (isRemote) { - return _copyWith( + return copyWith( localId: localId ?? a.localId, width: width ?? a.width, height: height ?? a.height, @@ -374,7 +375,7 @@ class Asset { ); } else { // TODO: Revisit this and remove all bool field assignments - return a._copyWith( + return a.copyWith( id: id, remoteId: remoteId, livePhotoVideoId: livePhotoVideoId, @@ -394,7 +395,7 @@ class Asset { // fill in potentially missing values, i.e. merge assets if (a.isRemote) { // values from remote take precedence - return _copyWith( + return copyWith( remoteId: a.remoteId, width: a.width, height: a.height, @@ -416,7 +417,7 @@ class Asset { ); } else { // add only missing values (and set isLocal to true) - return _copyWith( + return copyWith( localId: localId ?? a.localId, width: width ?? a.width, height: height ?? a.height, @@ -427,7 +428,7 @@ class Asset { } } - Asset _copyWith({ + Asset copyWith({ Id? id, String? checksum, String? remoteId, @@ -488,6 +489,9 @@ class Asset { static int compareById(Asset a, Asset b) => a.id.compareTo(b.id); + static int compareByLocalId(Asset a, Asset b) => + compareToNullable(a.localId, b.localId); + static int compareByChecksum(Asset a, Asset b) => a.checksum.compareTo(b.checksum); diff --git a/mobile/lib/infrastructure/entities/device_asset.entity.dart b/mobile/lib/infrastructure/entities/device_asset.entity.dart new file mode 100644 index 000000000..d8bfb2aa4 --- /dev/null +++ b/mobile/lib/infrastructure/entities/device_asset.entity.dart @@ -0,0 +1,36 @@ +import 'dart:typed_data'; + +import 'package:immich_mobile/domain/models/device_asset.model.dart'; +import 'package:immich_mobile/utils/hash.dart'; +import 'package:isar/isar.dart'; + +part 'device_asset.entity.g.dart'; + +@Collection(inheritance: false) +class DeviceAssetEntity { + Id get id => fastHash(assetId); + + @Index(replace: true, unique: true, type: IndexType.hash) + final String assetId; + @Index(unique: false, type: IndexType.hash) + final List hash; + final DateTime modifiedTime; + + const DeviceAssetEntity({ + required this.assetId, + required this.hash, + required this.modifiedTime, + }); + + DeviceAsset toModel() => DeviceAsset( + assetId: assetId, + hash: Uint8List.fromList(hash), + modifiedTime: modifiedTime, + ); + + static DeviceAssetEntity fromDto(DeviceAsset dto) => DeviceAssetEntity( + assetId: dto.assetId, + hash: dto.hash, + modifiedTime: dto.modifiedTime, + ); +} diff --git a/mobile/lib/infrastructure/entities/device_asset.entity.g.dart b/mobile/lib/infrastructure/entities/device_asset.entity.g.dart new file mode 100644 index 0000000000000000000000000000000000000000..a66f8288ef2f497e1f7aac311dc11af1d2059c03 GIT binary patch literal 25405 zcmd^HTT|P}5`Ooun1@q^*>xN)2?Qr`3IfSeMM8FAPil)wsT6CBmB^AK$*|rn{`=|f zxoAd`g)y=*IXnQ;+ok2`LX@U+dt)K4dE4+_^_HX2W#vR`WZ!QPG&H+~xh z0q>=L6u##nkKHtiYaR%@>qS@4_a*Q6mthq1Cw)KQcwbyP{U~Jx{Kwdj zxs%4O5A`mg>2T;mfgeJ5JcJePL&+Gbz-;>CkN|f;zzF6F*xRxI8(^9Pp7_v(DP-(k zJ4qVD2)TlA90VW??|p}YZ;>B5F;B;F=v@2h0NDg+4xvvy!@eK70nDc72GGL2iU6l? zi4#ZT&;x}0Fx9t5QR1VFQ2%Qb1srA$0s{Ie$GN+~wL3|iWW;-Zf8wN*5l=^NdmXm3x3jyky|Me^`Nr<{vu8V7 zyPH@91RL=<^?A~v=`iSo6#3X&>#$Fvbdn9k;0L?gz{dEaMpli~eZ;gw=$ylUZNUIh zxGw(0YXdhKRE|O-nxjyQDMNgnr2NDOjUdVNAj8N*<@3({A+H>4OZBU!WVKXwN4XT2SLebW~c2%CMfBrfzRg z*3~|hU>)%FX!<*BZM(hMegU)xYuo^zQX+tpwYh}ebUB%F4hx%*yx@768hT`Ftl~L4GDCapJN|9FuSHd_k~CF(xyLW z6k#+aKo=WdS~)0SHJKb&v^bFNLbut>+8`c<28Ytr>jqm{1JhLqFJ9zCk)wv`MevS2 zu)GKgBJ~G%JwPg^Kleq7A}0w5(5N6Ajt**q!49gN7_^Ol3b;xGhX`2}OW<5*Z{DyA zX&6m}tcE49(inulWTH?fV&SI}3K}?~2YB_VOK_HCTp9!A%p#*bm34wPm8$zWKcWxK&R7U*20JlXDMfq;^m0z_-k+rCHZ zPIeR(1FiZ2c;r{!1U#kR>vqeBA?q)QJ=1U(U#hH}a1^3Dlp+hlwH%Pv1udz>MHS0t z2+(AiMnSZL355(Q$1uhzRPBe~43C`PQh5hHd zzkMp!5O0o~Q*BrX4m7;2xNEt4@39j+#xr zj=AOq|FYgrBkU0KOtd%I+S8atphU0P+E0@^FE&B3bkPe7kQZNaiGne)bW6wD81{C! z(K8thq#`d3Tw`dGbyj?jy)GM#Gm~(OwbwkPHbqo~@6SjGZpG}Kq$=o=w6-nwFKKq- zUcuG?q#U3jm|xx~?U5>w4PBU|jO3Z_qLclb2_>mToQL@n07Y^Y#Dt`P5(TZznLJZe zQq@Nd21g?E3kkV7Vn&ncqhi2=KQn`r2{i>L*juXrs@n7YKIZY{^8of9&!RXb(O{#E z?jDT&0HWcdlWP_IQH%Xfoe1wZPxJjg;nWKx@Y1;C<1LvS> zYi85kw(EJ+>D$1C-QA|x<~ug;wPCd8y0p$Wl3lZcc5lFhTy?-Edy?$(y`vvvH#m=) zl;9|IB-Lxd(nj$sPp>%-MX-ka4x(!wcVu#fHy9g_VB6EYm4_XKH$|J3dKq`}AbgC{ zT5$0Dl4k8|r7P_9Ylb=lX@5__i<6WP6aD03;BCh8T(k-?5jBVk`vOUF4$lsn8D{W| zpcjli{-;2ZHhpAbCCp1jv1i&2$vr5xZ!I}0|YW|w`vbHqB=9Ox@V5cUNP36`qcNl8Xc*+h; z-vrOFDY7syfu}g?>6Mg^9$_Kau*;8mawpr@ST0aT2v84Ji--jZ7#FxKp!D&1*>t2u zCE`2aEjNM1P5ez{E?Edw!(?+u_z<;LqRP5@@Fp(vK6N3pkbEjk##Gx+A0OT zPss9UrmB?`mux8umP0#RhFexrn$+CGOcdK#Fq+PSA}f5nqRc97D$u4CJ(iO0uh#Sl zRe5%^iei-5RJ^SLS2!&YAH$l~1JTM@(;UFaJDtQ%H>DQ-qK%F(st8@bT2{(mB_;(v zkHms5calq9B(2AaN6KGUtzH$$%Jb0~_O{;n0qh0P^yl0S6g|9+LXVCF-gAR1-R zTqsO;WsFBLr2Z|PvnB;_P=C0d?uvtC8#WRFcqh>J?%|vS!9s87&iq-0L1G!0b zZRE|Jh-&m@w5GNugs7A+S~GfxbuqUtG|p09ogFG{_2JFQ<6SkqE%IoWOz)%oU#yq4 zMXW@(wu*%i@_ClA%~|f(;+M%Tn?m#nK1Y%0dEMZ7pTaYruj|)I>n5GQ~TY zK1(Mn;LG!g(9tdFDr;$B^x(GmEQPl}lFx8DNIZjCT}KFSXjqcsN>#6}+(BAtUw9OH zj{<#e$@z0&4{>Jd`eC9DC#mRfYJgGOZgeg_Dg=G-z*$J9Ez9F)`Ue+|VR8-j2u^zQ z7~V`0CmJlPC>_hv;D*;fNVFNzzJ?PHJvWBaw`vl)p*4*MH}1#D(QuT)Ifa@wqHd$@ ze7VY+-k@4u3UZv_mef+gDNPt=QtPDxgu4bu0mrxb)UGBTZX?^c5Ts6-it(~*^CePvbVBl#m)3aj zC&v}FL5VD=HE5a%5&eQTyx(fPQOmM7x9x&u;ZSdgbXO>}m&9a~u|4{ENMZ6SP-`C$2M4L}}<&qJA0(CjO5-^qqO)H1wEr6{KD<74wo{7Xpuz~7t?#U{?$}=L_ zrL1IaxGtX5)}i^;$&=*wAP_u#m`?0)$=!-2L>#_X&~rE~Dx_G7y@VGza<0Cx*LIKG zXf+_VJ<`hIJ8wR;8sJ)nRh@Je76-0=suaiq{HIKS%nuu=6iB^gzd}1)rkHwf^ucVn zDNVte+vxoa`(5UAR||aG&aWu)^IH4Wh28Q18g(N))%SiXZ!l#+9^Nm^a1im7-qfZr zx3+_bGYm_wtuK94e4F%Oa-U&E5BEH5DLaw5MM6UD%kP=gvbhMMycom!KnUKVcWK3` z)jH0wSIh*yUev4YY3j7ucrZwfS!>yJJ{TWbtbabX>BwGpUhEQ?b6U(gmCdVUv1YIo zxF+WV9^PMXMK87LE_@jl1p66u*P6*K-IPJSkOXJeP2`O$l)@vErhtgJ|^x7iwVj>@=JGZ}1egHq}tW(ER8r{i4NRh=yC! zu(TmJof)EsPx4dWM7uc?9CAsjquEpd&V0ZJo2cO_e36GfET44YQ$4Di+SXikh_9>z ztHHkKK$^j)ii7NB0!x>z23eHUDl)jc=qF%D=sUz^(xVD2TJ(9 deleteIds(List ids) { + return transaction(() async { + await _db.deviceAssetEntitys.deleteAllByAssetId(ids.toList()); + }); + } + + @override + Future> getByIds(List localIds) { + return _db.deviceAssetEntitys + .where() + .anyOf(localIds, (query, id) => query.assetIdEqualTo(id)) + .findAll() + .then((value) => value.map((e) => e.toModel()).toList()); + } + + @override + Future updateAll(List assetHash) { + return transaction(() async { + await _db.deviceAssetEntitys + .putAll(assetHash.map(DeviceAssetEntity.fromDto).toList()); + return true; + }); + } +} diff --git a/mobile/lib/interfaces/asset.interface.dart b/mobile/lib/interfaces/asset.interface.dart index ed524c4f3..76744c917 100644 --- a/mobile/lib/interfaces/asset.interface.dart +++ b/mobile/lib/interfaces/asset.interface.dart @@ -1,6 +1,5 @@ import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; -import 'package:immich_mobile/entities/device_asset.entity.dart'; import 'package:immich_mobile/interfaces/database.interface.dart'; abstract interface class IAssetRepository implements IDatabaseRepository { @@ -50,10 +49,6 @@ abstract interface class IAssetRepository implements IDatabaseRepository { int limit = 100, }); - Future> getDeviceAssetsById(List ids); - - Future upsertDeviceAssets(List deviceAssets); - Future upsertDuplicatedAssets(Iterable duplicatedAssets); Future> getAllDuplicatedAssetIds(); diff --git a/mobile/lib/providers/infrastructure/device_asset.provider.dart b/mobile/lib/providers/infrastructure/device_asset.provider.dart new file mode 100644 index 000000000..5fa532b9e --- /dev/null +++ b/mobile/lib/providers/infrastructure/device_asset.provider.dart @@ -0,0 +1,8 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/infrastructure/repositories/device_asset.repository.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; + +final deviceAssetRepositoryProvider = Provider( + (ref) => IsarDeviceAssetRepository(ref.watch(isarProvider)), +); diff --git a/mobile/lib/repositories/asset.repository.dart b/mobile/lib/repositories/asset.repository.dart index d9e8897e9..cda2b25e4 100644 --- a/mobile/lib/repositories/asset.repository.dart +++ b/mobile/lib/repositories/asset.repository.dart @@ -1,12 +1,7 @@ -import 'dart:io'; - import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/entities/album.entity.dart'; -import 'package:immich_mobile/entities/android_device_asset.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; -import 'package:immich_mobile/entities/device_asset.entity.dart'; import 'package:immich_mobile/entities/duplicated_asset.entity.dart'; -import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'; import 'package:immich_mobile/interfaces/asset.interface.dart'; import 'package:immich_mobile/providers/db.provider.dart'; @@ -158,19 +153,6 @@ class AssetRepository extends DatabaseRepository implements IAssetRepository { return _getMatchesImpl(query, fastHash(ownerId), assets, limit); } - @override - Future> getDeviceAssetsById(List ids) => - Platform.isAndroid - ? db.androidDeviceAssets.getAll(ids.cast()) - : db.iOSDeviceAssets.getAllById(ids.cast()); - - @override - Future upsertDeviceAssets(List deviceAssets) => txn( - () => Platform.isAndroid - ? db.androidDeviceAssets.putAll(deviceAssets.cast()) - : db.iOSDeviceAssets.putAll(deviceAssets.cast()), - ); - @override Future update(Asset asset) async { await txn(() => asset.put(db)); diff --git a/mobile/lib/services/hash.service.dart b/mobile/lib/services/hash.service.dart index bb19340d2..ca2b0ee37 100644 --- a/mobile/lib/services/hash.service.dart +++ b/mobile/lib/services/hash.service.dart @@ -1,172 +1,205 @@ +// ignore_for_file: avoid-unsafe-collection-methods + +import 'dart:convert'; import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; -import 'package:immich_mobile/entities/album.entity.dart'; -import 'package:immich_mobile/interfaces/album_media.interface.dart'; -import 'package:immich_mobile/interfaces/asset.interface.dart'; -import 'package:immich_mobile/repositories/album_media.repository.dart'; -import 'package:immich_mobile/repositories/asset.repository.dart'; -import 'package:immich_mobile/services/background.service.dart'; -import 'package:immich_mobile/entities/android_device_asset.entity.dart'; +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/domain/models/device_asset.model.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; -import 'package:immich_mobile/entities/device_asset.entity.dart'; -import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; -import 'package:immich_mobile/extensions/string_extensions.dart'; +import 'package:immich_mobile/providers/infrastructure/device_asset.provider.dart'; +import 'package:immich_mobile/services/background.service.dart'; import 'package:logging/logging.dart'; class HashService { - HashService( - this._assetRepository, - this._backgroundService, - this._albumMediaRepository, - ); - final IAssetRepository _assetRepository; - final BackgroundService _backgroundService; - final IAlbumMediaRepository _albumMediaRepository; - final _log = Logger('HashService'); + HashService({ + required IDeviceAssetRepository deviceAssetRepository, + required BackgroundService backgroundService, + this.batchSizeLimit = kBatchHashSizeLimit, + this.batchFileLimit = kBatchHashFileLimit, + }) : _deviceAssetRepository = deviceAssetRepository, + _backgroundService = backgroundService; - /// Returns all assets that were successfully hashed - Future> getHashedAssets( - Album album, { - int start = 0, - int end = 0x7fffffffffffffff, - DateTime? modifiedFrom, - DateTime? modifiedUntil, - Set? excludedAssets, - }) async { - final entities = await _albumMediaRepository.getAssets( - album.localId!, - start: start, - end: end, - modifiedFrom: modifiedFrom, - modifiedUntil: modifiedUntil, - ); - final filtered = excludedAssets == null - ? entities - : entities.where((e) => !excludedAssets.contains(e.localId!)).toList(); - return _hashAssets(filtered); - } + final IDeviceAssetRepository _deviceAssetRepository; + final BackgroundService _backgroundService; + final int batchSizeLimit; + final int batchFileLimit; + final _log = Logger('HashService'); /// Processes a list of local [Asset]s, storing their hash and returning only those /// that were successfully hashed. Hashes are looked up in a DB table - /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing - /// entries are newly hashed and added to the DB table. - Future> _hashAssets(List assets) async { - const int batchFileCount = 128; - const int batchDataSize = 1024 * 1024 * 1024; // 1GB + /// [DeviceAsset] by local id. Only missing entries are newly hashed and added to the DB table. + Future> hashAssets(List assets) async { + assets.sort(Asset.compareByLocalId); - final ids = assets - .map(Platform.isAndroid ? (a) => a.localId!.toInt() : (a) => a.localId!) - .toList(); - final List hashes = - await _assetRepository.getDeviceAssetsById(ids); - final List toAdd = []; - final List toHash = []; + // Get and sort DB entries - guaranteed to be a subset of assets + final hashesInDB = await _deviceAssetRepository.getByIds( + assets.map((a) => a.localId!).toList(), + ); + hashesInDB.sort((a, b) => a.assetId.compareTo(b.assetId)); - int bytes = 0; + int dbIndex = 0; + int bytesProcessed = 0; + final hashedAssets = []; + final toBeHashed = <_AssetPath>[]; + final toBeDeleted = []; - for (int i = 0; i < assets.length; i++) { - if (hashes[i] != null) { + for (int assetIndex = 0; assetIndex < assets.length; assetIndex++) { + final asset = assets[assetIndex]; + DeviceAsset? matchingDbEntry; + + if (dbIndex < hashesInDB.length) { + final deviceAsset = hashesInDB[dbIndex]; + if (deviceAsset.assetId == asset.localId) { + matchingDbEntry = deviceAsset; + dbIndex++; + } + } + + if (matchingDbEntry != null && + matchingDbEntry.hash.isNotEmpty && + matchingDbEntry.modifiedTime.isAtSameMomentAs(asset.fileModifiedAt)) { + // Reuse the existing hash + hashedAssets.add( + asset.copyWith(checksum: base64.encode(matchingDbEntry.hash)), + ); continue; } - File? file; - - try { - file = await assets[i].local!.originFile; - } catch (error, stackTrace) { - _log.warning( - "Error getting file to hash for asset ${assets[i].localId}, name: ${assets[i].fileName}, created on: ${assets[i].fileCreatedAt}, skipping", - error, - stackTrace, - ); - } - + final file = await _tryGetAssetFile(asset); if (file == null) { - final fileName = assets[i].fileName; - - _log.warning( - "Failed to get file for asset ${assets[i].localId}, name: $fileName, created on: ${assets[i].fileCreatedAt}, skipping", - ); + // Can't access file, delete any DB entry + if (matchingDbEntry != null) { + toBeDeleted.add(matchingDbEntry.assetId); + } continue; } - bytes += await file.length(); - toHash.add(file.path); - final deviceAsset = Platform.isAndroid - ? AndroidDeviceAsset(id: ids[i] as int, hash: const []) - : IOSDeviceAsset(id: ids[i] as String, hash: const []); - toAdd.add(deviceAsset); - hashes[i] = deviceAsset; - if (toHash.length == batchFileCount || bytes >= batchDataSize) { - await _processBatch(toHash, toAdd); - toAdd.clear(); - toHash.clear(); - bytes = 0; + + bytesProcessed += await file.length(); + toBeHashed.add(_AssetPath(asset: asset, path: file.path)); + + if (_shouldProcessBatch(toBeHashed.length, bytesProcessed)) { + hashedAssets.addAll(await _processBatch(toBeHashed, toBeDeleted)); + toBeHashed.clear(); + toBeDeleted.clear(); + bytesProcessed = 0; } } - if (toHash.isNotEmpty) { - await _processBatch(toHash, toAdd); + assert(dbIndex == hashesInDB.length, "All hashes should've been processed"); + + // Process any remaining files + if (toBeHashed.isNotEmpty) { + hashedAssets.addAll(await _processBatch(toBeHashed, toBeDeleted)); } - return _getHashedAssets(assets, hashes); + + // Clean up deleted references + if (toBeDeleted.isNotEmpty) { + await _deviceAssetRepository.deleteIds(toBeDeleted); + } + + return hashedAssets; } - /// Processes a batch of files and saves any successfully hashed - /// values to the DB table. - Future _processBatch( - final List toHash, - final List toAdd, + bool _shouldProcessBatch(int assetCount, int bytesProcessed) => + assetCount >= batchFileLimit || bytesProcessed >= batchSizeLimit; + + Future _tryGetAssetFile(Asset asset) async { + try { + final file = await asset.local!.originFile; + if (file == null) { + _log.warning( + "Failed to get file for asset ${asset.localId ?? ''}, name: ${asset.fileName}, created on: ${asset.fileCreatedAt}, skipping", + ); + return null; + } + return file; + } catch (error, stackTrace) { + _log.warning( + "Error getting file to hash for asset ${asset.localId ?? ''}, name: ${asset.fileName}, created on: ${asset.fileCreatedAt}, skipping", + error, + stackTrace, + ); + return null; + } + } + + /// Processes a batch of files and returns a list of successfully hashed assets after saving + /// them in [DeviceAssetToHash] for future retrieval + Future> _processBatch( + List<_AssetPath> toBeHashed, + List toBeDeleted, ) async { - final hashes = await _hashFiles(toHash); - bool anyNull = false; - for (int j = 0; j < hashes.length; j++) { - if (hashes[j]?.length == 20) { - toAdd[j].hash = hashes[j]!; + _log.info("Hashing ${toBeHashed.length} files"); + final hashes = await _hashFiles(toBeHashed.map((e) => e.path).toList()); + assert( + hashes.length == toBeHashed.length, + "Number of Hashes returned from platform should be the same as the input", + ); + + final hashedAssets = []; + final toBeAdded = []; + + for (final (index, hash) in hashes.indexed) { + final asset = toBeHashed.elementAtOrNull(index)?.asset; + if (asset != null && hash?.length == 20) { + hashedAssets.add(asset.copyWith(checksum: base64.encode(hash!))); + toBeAdded.add( + DeviceAsset( + assetId: asset.localId!, + hash: hash, + modifiedTime: asset.fileModifiedAt, + ), + ); } else { - _log.warning("Failed to hash file ${toHash[j]}, skipping"); - anyNull = true; + _log.warning("Failed to hash file ${asset?.localId ?? ''}"); + if (asset != null) { + toBeDeleted.add(asset.localId!); + } } } - final validHashes = anyNull - ? toAdd.where((e) => e.hash.length == 20).toList(growable: false) - : toAdd; - await _assetRepository - .transaction(() => _assetRepository.upsertDeviceAssets(validHashes)); - _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); + // Update the DB for future retrieval + await _deviceAssetRepository.transaction(() async { + await _deviceAssetRepository.updateAll(toBeAdded); + await _deviceAssetRepository.deleteIds(toBeDeleted); + }); + + _log.fine("Hashed ${hashedAssets.length}/${toBeHashed.length} assets"); + return hashedAssets; } - /// Hashes the given files and returns a list of the same length - /// files that could not be hashed have a `null` value + /// Hashes the given files and returns a list of the same length. + /// Files that could not be hashed will have a `null` value Future> _hashFiles(List paths) async { - final List? hashes = - await _backgroundService.digestFiles(paths); - if (hashes == null) { - throw Exception("Hashing ${paths.length} files failed"); - } - return hashes; - } - - /// Returns all successfully hashed [Asset]s with their hash value set - List _getHashedAssets( - List assets, - List hashes, - ) { - final List result = []; - for (int i = 0; i < assets.length; i++) { - if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { - assets[i].byteHash = hashes[i]!.hash; - result.add(assets[i]); + try { + final hashes = await _backgroundService.digestFiles(paths); + if (hashes != null) { + return hashes; } + _log.severe("Hashing ${paths.length} files failed"); + } catch (e, s) { + _log.severe("Error occurred while hashing assets", e, s); } - return result; + return List.filled(paths.length, null); + } +} + +class _AssetPath { + final Asset asset; + final String path; + + const _AssetPath({required this.asset, required this.path}); + + _AssetPath copyWith({Asset? asset, String? path}) { + return _AssetPath(asset: asset ?? this.asset, path: path ?? this.path); } } final hashServiceProvider = Provider( (ref) => HashService( - ref.watch(assetRepositoryProvider), - ref.watch(backgroundServiceProvider), - ref.watch(albumMediaRepositoryProvider), + deviceAssetRepository: ref.watch(deviceAssetRepositoryProvider), + backgroundService: ref.watch(backgroundServiceProvider), ), ); diff --git a/mobile/lib/services/sync.service.dart b/mobile/lib/services/sync.service.dart index a05d4b648..f2b16b080 100644 --- a/mobile/lib/services/sync.service.dart +++ b/mobile/lib/services/sync.service.dart @@ -577,15 +577,18 @@ class SyncService { Set? excludedAssets, bool forceRefresh = false, ]) async { + _log.info("Syncing a local album to DB: ${deviceAlbum.name}"); if (!forceRefresh && !await _hasAlbumChangeOnDevice(deviceAlbum, dbAlbum)) { - _log.fine( + _log.info( "Local album ${deviceAlbum.name} has not changed. Skipping sync.", ); return false; } + _log.info("Local album ${deviceAlbum.name} has changed. Syncing..."); if (!forceRefresh && excludedAssets == null && await _syncDeviceAlbumFast(deviceAlbum, dbAlbum)) { + _log.info("Fast synced local album ${deviceAlbum.name} to DB"); return true; } // general case, e.g. some assets have been deleted or there are excluded albums on iOS @@ -598,7 +601,7 @@ class SyncService { assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); final int assetCountOnDevice = await _albumMediaRepository.getAssetCount(deviceAlbum.localId!); - final List onDevice = await _hashService.getHashedAssets( + final List onDevice = await _getHashedAssets( deviceAlbum, excludedAssets: excludedAssets, ); @@ -611,7 +614,7 @@ class SyncService { dbAlbum.name == deviceAlbum.name && dbAlbum.modifiedAt.isAtSameMomentAs(deviceAlbum.modifiedAt)) { // changes only affeted excluded albums - _log.fine( + _log.info( "Only excluded assets in local album ${deviceAlbum.name} changed. Stopping sync.", ); if (assetCountOnDevice != @@ -626,11 +629,11 @@ class SyncService { } return false; } - _log.fine( + _log.info( "Syncing local album ${deviceAlbum.name}. ${toAdd.length} assets to add, ${toUpdate.length} to update, ${toDelete.length} to delete", ); final (existingInDb, updated) = await _linkWithExistingFromDb(toAdd); - _log.fine( + _log.info( "Linking assets to add with existing from db. ${existingInDb.length} existing, ${updated.length} to update", ); deleteCandidates.addAll(toDelete); @@ -667,6 +670,9 @@ class SyncService { /// returns `true` if successful, else `false` Future _syncDeviceAlbumFast(Album deviceAlbum, Album dbAlbum) async { if (!deviceAlbum.modifiedAt.isAfter(dbAlbum.modifiedAt)) { + _log.info( + "Local album ${deviceAlbum.name} has not changed. Skipping sync.", + ); return false; } final int totalOnDevice = @@ -676,15 +682,21 @@ class SyncService { ?.assetCount ?? 0; if (totalOnDevice <= lastKnownTotal) { + _log.info( + "Local album ${deviceAlbum.name} totalOnDevice is less than lastKnownTotal. Skipping sync.", + ); return false; } - final List newAssets = await _hashService.getHashedAssets( + final List newAssets = await _getHashedAssets( deviceAlbum, modifiedFrom: dbAlbum.modifiedAt.add(const Duration(seconds: 1)), modifiedUntil: deviceAlbum.modifiedAt, ); if (totalOnDevice != lastKnownTotal + newAssets.length) { + _log.info( + "Local album ${deviceAlbum.name} totalOnDevice is not equal to lastKnownTotal + newAssets.length. Skipping sync.", + ); return false; } dbAlbum.modifiedAt = deviceAlbum.modifiedAt; @@ -719,8 +731,8 @@ class SyncService { List existing, [ Set? excludedAssets, ]) async { - _log.info("Syncing a new local album to DB: ${album.name}"); - final assets = await _hashService.getHashedAssets( + _log.info("Adding a new local album to DB: ${album.name}"); + final assets = await _getHashedAssets( album, excludedAssets: excludedAssets, ); @@ -824,6 +836,28 @@ class SyncService { } } + /// Returns all assets that were successfully hashed + Future> _getHashedAssets( + Album album, { + int start = 0, + int end = 0x7fffffffffffffff, + DateTime? modifiedFrom, + DateTime? modifiedUntil, + Set? excludedAssets, + }) async { + final entities = await _albumMediaRepository.getAssets( + album.localId!, + start: start, + end: end, + modifiedFrom: modifiedFrom, + modifiedUntil: modifiedUntil, + ); + final filtered = excludedAssets == null + ? entities + : entities.where((e) => !excludedAssets.contains(e.localId!)).toList(); + return _hashService.hashAssets(filtered); + } + List _removeDuplicates(List assets) { final int before = assets.length; assets.sort(Asset.compareByOwnerChecksumCreatedModified); diff --git a/mobile/lib/utils/bootstrap.dart b/mobile/lib/utils/bootstrap.dart index 21231becf..dec48582b 100644 --- a/mobile/lib/utils/bootstrap.dart +++ b/mobile/lib/utils/bootstrap.dart @@ -10,6 +10,7 @@ import 'package:immich_mobile/entities/backup_album.entity.dart'; import 'package:immich_mobile/entities/duplicated_asset.entity.dart'; import 'package:immich_mobile/entities/etag.entity.dart'; import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; +import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'; import 'package:immich_mobile/infrastructure/entities/log.entity.dart'; import 'package:immich_mobile/infrastructure/entities/store.entity.dart'; @@ -39,6 +40,7 @@ abstract final class Bootstrap { ETagSchema, if (Platform.isAndroid) AndroidDeviceAssetSchema, if (Platform.isIOS) IOSDeviceAssetSchema, + DeviceAssetEntitySchema, ], directory: dir.path, maxSizeMiB: 1024, diff --git a/mobile/lib/utils/diff.dart b/mobile/lib/utils/diff.dart index a36902d8c..ea20de16c 100644 --- a/mobile/lib/utils/diff.dart +++ b/mobile/lib/utils/diff.dart @@ -75,3 +75,17 @@ bool diffSortedListsSync( } return diff; } + +int compareToNullable(T? a, T? b) { + if (a == null && b == null) { + return 0; + } + + if (a == null) { + return 1; + } + if (b == null) { + return -1; + } + return a.compareTo(b); +} diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 3e73ab445..bebd7a027 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -1,40 +1,51 @@ -import 'dart:async'; +// ignore_for_file: avoid-unsafe-collection-methods +import 'dart:async'; +import 'dart:io'; + +import 'package:flutter/widgets.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; import 'package:immich_mobile/entities/album.entity.dart'; +import 'package:immich_mobile/entities/android_device_asset.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; import 'package:immich_mobile/entities/etag.entity.dart'; +import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; import 'package:immich_mobile/entities/store.entity.dart'; +import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'; import 'package:immich_mobile/infrastructure/entities/store.entity.dart'; import 'package:immich_mobile/infrastructure/entities/user.entity.dart'; +import 'package:immich_mobile/utils/diff.dart'; import 'package:isar/isar.dart'; -const int targetVersion = 9; +const int targetVersion = 10; Future migrateDatabaseIfNeeded(Isar db) async { - final int version = Store.get(StoreKey.version, 1); + final int version = Store.get(StoreKey.version, targetVersion); if (version < 9) { - await Store.put(StoreKey.version, version); + await Store.put(StoreKey.version, targetVersion); final value = await db.storeValues.get(StoreKey.currentUser.id); if (value != null) { final id = value.intValue; - if (id == null) { - return; + if (id != null) { + await db.writeTxn(() async { + final user = await db.users.get(id); + await db.storeValues + .put(StoreValue(StoreKey.currentUser.id, strValue: user?.id)); + }); } - await db.writeTxn(() async { - final user = await db.users.get(id); - await db.storeValues - .put(StoreValue(StoreKey.currentUser.id, strValue: user?.id)); - }); } - // Do not clear other entities - return; } - if (version < targetVersion) { - _migrateTo(db, targetVersion); + if (version < 10) { + await Store.put(StoreKey.version, targetVersion); + await _migrateDeviceAsset(db); + } + + final shouldTruncate = version < 8 && version < targetVersion; + if (shouldTruncate) { + await _migrateTo(db, targetVersion); } } @@ -49,3 +60,59 @@ Future _migrateTo(Isar db, int version) async { }); await Store.put(StoreKey.version, version); } + +Future _migrateDeviceAsset(Isar db) async { + final ids = Platform.isAndroid + ? (await db.androidDeviceAssets.where().findAll()) + .map((a) => _DeviceAsset(assetId: a.id.toString(), hash: a.hash)) + .toList() + : (await db.iOSDeviceAssets.where().findAll()) + .map((i) => _DeviceAsset(assetId: i.id, hash: i.hash)) + .toList(); + final localAssets = (await db.assets + .where() + .anyOf(ids, (query, id) => query.localIdEqualTo(id.assetId)) + .findAll()) + .map((a) => _DeviceAsset(assetId: a.localId!, dateTime: a.fileModifiedAt)) + .toList(); + debugPrint("Device Asset Ids length - ${ids.length}"); + debugPrint("Local Asset Ids length - ${localAssets.length}"); + ids.sort((a, b) => a.assetId.compareTo(b.assetId)); + localAssets.sort((a, b) => a.assetId.compareTo(b.assetId)); + final List toAdd = []; + await diffSortedLists( + ids, + localAssets, + compare: (a, b) => a.assetId.compareTo(b.assetId), + both: (deviceAsset, asset) { + toAdd.add( + DeviceAssetEntity( + assetId: deviceAsset.assetId, + hash: deviceAsset.hash!, + modifiedTime: asset.dateTime!, + ), + ); + return false; + }, + onlyFirst: (deviceAsset) { + debugPrint( + 'DeviceAsset not found in local assets: ${deviceAsset.assetId}', + ); + }, + onlySecond: (asset) { + debugPrint('Local asset not found in DeviceAsset: ${asset.assetId}'); + }, + ); + debugPrint("Total number of device assets migrated - ${toAdd.length}"); + await db.writeTxn(() async { + await db.deviceAssetEntitys.putAll(toAdd); + }); +} + +class _DeviceAsset { + final String assetId; + final List? hash; + final DateTime? dateTime; + + const _DeviceAsset({required this.assetId, this.hash, this.dateTime}); +} diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index e79d9f408..7c8348726 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -463,7 +463,7 @@ packages: source: hosted version: "2.1.4" file: - dependency: transitive + dependency: "direct dev" description: name: file sha256: a3b4f84adafef897088c160faf7dfffb7696046cb13ae90b508c2cbc95d3b8d4 diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index d4ab110a3..e939c6583 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -102,6 +102,7 @@ dev_dependencies: immich_mobile_immich_lint: path: './immich_lint' fake_async: ^1.3.1 + file: ^7.0.1 # for MemoryFileSystem # Drift generator drift_dev: ^2.23.1 diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dart new file mode 100644 index 000000000..2da41cd70 --- /dev/null +++ b/mobile/test/domain/services/hash_service_test.dart @@ -0,0 +1,425 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:collection/collection.dart'; +import 'package:file/memory.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/domain/models/device_asset.model.dart'; +import 'package:immich_mobile/entities/asset.entity.dart'; +import 'package:immich_mobile/services/background.service.dart'; +import 'package:immich_mobile/services/hash.service.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:photo_manager/photo_manager.dart'; + +import '../../fixtures/asset.stub.dart'; +import '../../infrastructure/repository.mock.dart'; +import '../../service.mocks.dart'; + +class MockAsset extends Mock implements Asset {} + +class MockAssetEntity extends Mock implements AssetEntity {} + +void main() { + late HashService sut; + late BackgroundService mockBackgroundService; + late IDeviceAssetRepository mockDeviceAssetRepository; + + setUp(() { + mockBackgroundService = MockBackgroundService(); + mockDeviceAssetRepository = MockDeviceAssetRepository(); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + ); + + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?)?.call(); + }); + when(() => mockDeviceAssetRepository.updateAll(any())) + .thenAnswer((_) async => true); + when(() => mockDeviceAssetRepository.deleteIds(any())) + .thenAnswer((_) async => true); + }); + + group("HashService: No DeviceAsset entry", () { + test("hash successfully", () async { + final (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + // No DB entries for this asset + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => []); + + final result = await sut.hashAssets([mockAsset]); + + // Verify we stored the new hash in DB + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + verify( + () => mockDeviceAssetRepository.updateAll([ + deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), + ]), + ).called(1); + verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); + }); + expect( + result, + [AssetStub.image1.copyWith(checksum: base64.encode(hash))], + ); + }); + }); + + group("HashService: Has DeviceAsset entry", () { + test("when the asset is not modified", () async { + final hash = utf8.encode("image1-hash"); + + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer( + (_) async => [ + DeviceAsset( + assetId: AssetStub.image1.localId!, + hash: hash, + modifiedTime: AssetStub.image1.fileModifiedAt, + ), + ], + ); + final result = await sut.hashAssets([AssetStub.image1]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockBackgroundService.digestFile(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + + expect(result, [ + AssetStub.image1.copyWith(checksum: base64.encode(hash)), + ]); + }); + + test("hashed successful when asset is modified", () async { + final (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => [deviceAsset]); + + final result = await sut.hashAssets([mockAsset]); + + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + verify( + () => mockDeviceAssetRepository.updateAll([ + deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), + ]), + ).called(1); + verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); + }); + + verify(() => mockBackgroundService.digestFiles([file.path])).called(1); + + expect(result, [ + AssetStub.image1.copyWith(checksum: base64.encode(hash)), + ]); + }); + }); + + group("HashService: Cleanup", () { + late Asset mockAsset; + late Uint8List hash; + late DeviceAsset deviceAsset; + late File file; + + setUp(() async { + (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => [deviceAsset]); + }); + + test("cleanups DeviceAsset when local file cannot be obtained", () async { + when(() => mockAsset.local).thenThrow(Exception("File not found")); + final result = await sut.hashAssets([mockAsset]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockBackgroundService.digestFile(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verify( + () => mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), + ).called(1); + + expect(result, isEmpty); + }); + + test("cleanups DeviceAsset when hashing failed", () async { + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + + // Verify the callback inside the transaction because, doing it outside results + // in a small delay before the callback is invoked, resulting in other LOCs getting executed + // resulting in an incorrect state + // + // i.e, consider the following piece of code + // await _deviceAssetRepository.transaction(() async { + // await _deviceAssetRepository.updateAll(toBeAdded); + // await _deviceAssetRepository.deleteIds(toBeDeleted); + // }); + // toBeDeleted.clear(); + // since the transaction method is mocked, the callback is not invoked until it is captured + // and executed manually in the next event loop. However, the toBeDeleted.clear() is executed + // immediately once the transaction stub is executed, resulting in the deleteIds method being + // called with an empty list. + // + // To avoid this, we capture the callback and execute it within the transaction stub itself + // and verify the results inside the transaction stub + verify(() => mockDeviceAssetRepository.updateAll([])).called(1); + verify( + () => + mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), + ).called(1); + }); + + when(() => mockBackgroundService.digestFiles([file.path])).thenAnswer( + // Invalid hash, length != 20 + (_) async => [Uint8List.fromList(hash.slice(2).toList())], + ); + + final result = await sut.hashAssets([mockAsset]); + + verify(() => mockBackgroundService.digestFiles([file.path])).called(1); + expect(result, isEmpty); + }); + }); + + group("HashService: Batch processing", () { + test("processes assets in batches when size limit is reached", () async { + // Setup multiple assets with large file sizes + final (mock1, mock2, mock3) = await ( + _createAssetMock(AssetStub.image1), + _createAssetMock(AssetStub.image2), + _createAssetMock(AssetStub.image3), + ).wait; + + final (asset1, file1, deviceAsset1, hash1) = mock1; + final (asset2, file2, deviceAsset2, hash2) = mock2; + final (asset3, file3, deviceAsset3, hash3) = mock3; + + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + // Setup for multiple batch processing calls + when(() => mockBackgroundService.digestFiles([file1.path, file2.path])) + .thenAnswer((_) async => [hash1, hash2]); + when(() => mockBackgroundService.digestFiles([file3.path])) + .thenAnswer((_) async => [hash3]); + + final size = await file1.length() + await file2.length(); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + batchSizeLimit: size, + ); + final result = await sut.hashAssets([asset1, asset2, asset3]); + + // Verify multiple batch process calls + verify(() => mockBackgroundService.digestFiles([file1.path, file2.path])) + .called(1); + verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + + expect( + result, + [ + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ], + ); + }); + + test("processes assets in batches when file limit is reached", () async { + // Setup multiple assets with large file sizes + final (mock1, mock2, mock3) = await ( + _createAssetMock(AssetStub.image1), + _createAssetMock(AssetStub.image2), + _createAssetMock(AssetStub.image3), + ).wait; + + final (asset1, file1, deviceAsset1, hash1) = mock1; + final (asset2, file2, deviceAsset2, hash2) = mock2; + final (asset3, file3, deviceAsset3, hash3) = mock3; + + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + when(() => mockBackgroundService.digestFiles([file1.path])) + .thenAnswer((_) async => [hash1]); + when(() => mockBackgroundService.digestFiles([file2.path])) + .thenAnswer((_) async => [hash2]); + when(() => mockBackgroundService.digestFiles([file3.path])) + .thenAnswer((_) async => [hash3]); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + batchFileLimit: 1, + ); + final result = await sut.hashAssets([asset1, asset2, asset3]); + + // Verify multiple batch process calls + verify(() => mockBackgroundService.digestFiles([file1.path])).called(1); + verify(() => mockBackgroundService.digestFiles([file2.path])).called(1); + verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + + expect( + result, + [ + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ], + ); + }); + + test("HashService: Sort & Process different states", () async { + final (asset1, file1, deviceAsset1, hash1) = + await _createAssetMock(AssetStub.image1); // Will need rehashing + final (asset2, file2, deviceAsset2, hash2) = + await _createAssetMock(AssetStub.image2); // Will have matching hash + final (asset3, file3, deviceAsset3, hash3) = + await _createAssetMock(AssetStub.image3); // No DB entry + final asset4 = + AssetStub.image3.copyWith(localId: "image4"); // Cannot be hashed + + when(() => mockBackgroundService.digestFiles([file1.path, file3.path])) + .thenAnswer((_) async => [hash1, hash3]); + // DB entries are not sorted and a dummy entry added + when( + () => mockDeviceAssetRepository.getByIds([ + AssetStub.image1.localId!, + AssetStub.image2.localId!, + AssetStub.image3.localId!, + asset4.localId!, + ]), + ).thenAnswer( + (_) async => [ + // Same timestamp to reuse deviceAsset + deviceAsset2.copyWith(modifiedTime: asset2.fileModifiedAt), + deviceAsset1, + deviceAsset3.copyWith(assetId: asset4.localId!), + ], + ); + + final result = await sut.hashAssets([asset1, asset2, asset3, asset4]); + + // Verify correct processing of all assets + verify(() => mockBackgroundService.digestFiles([file1.path, file3.path])) + .called(1); + expect(result.length, 3); + expect(result, [ + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ]); + }); + + group("HashService: Edge cases", () { + test("handles empty list of assets", () async { + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + final result = await sut.hashAssets([]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + + expect(result, isEmpty); + }); + + test("handles all file access failures", () async { + // No DB entries + when( + () => mockDeviceAssetRepository.getByIds( + [AssetStub.image1.localId!, AssetStub.image2.localId!], + ), + ).thenAnswer((_) async => []); + + final result = await sut.hashAssets([ + AssetStub.image1, + AssetStub.image2, + ]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + expect(result, isEmpty); + }); + }); + }); +} + +Future<(Asset, File, DeviceAsset, Uint8List)> _createAssetMock( + Asset asset, +) async { + final random = Random(); + final hash = + Uint8List.fromList(List.generate(20, (i) => random.nextInt(255))); + final mockAsset = MockAsset(); + final mockAssetEntity = MockAssetEntity(); + final fs = MemoryFileSystem(); + final deviceAsset = DeviceAsset( + assetId: asset.localId!, + hash: Uint8List.fromList(hash), + modifiedTime: DateTime.now(), + ); + final tmp = await fs.systemTempDirectory.createTemp(); + final file = tmp.childFile("${asset.fileName}-path"); + await file.writeAsString("${asset.fileName}-content"); + + when(() => mockAsset.localId).thenReturn(asset.localId); + when(() => mockAsset.fileName).thenReturn(asset.fileName); + when(() => mockAsset.fileCreatedAt).thenReturn(asset.fileCreatedAt); + when(() => mockAsset.fileModifiedAt).thenReturn(asset.fileModifiedAt); + when(() => mockAsset.copyWith(checksum: any(named: "checksum"))) + .thenReturn(asset.copyWith(checksum: base64.encode(hash))); + when(() => mockAsset.local).thenAnswer((_) => mockAssetEntity); + when(() => mockAssetEntity.originFile).thenAnswer((_) async => file); + + return (mockAsset, file, deviceAsset, hash); +} diff --git a/mobile/test/fixtures/asset.stub.dart b/mobile/test/fixtures/asset.stub.dart index 26108d63b..b69b39212 100644 --- a/mobile/test/fixtures/asset.stub.dart +++ b/mobile/test/fixtures/asset.stub.dart @@ -8,8 +8,8 @@ final class AssetStub { localId: "image1", remoteId: 'image1-remote', ownerId: 1, - fileCreatedAt: DateTime.now(), - fileModifiedAt: DateTime.now(), + fileCreatedAt: DateTime(2019), + fileModifiedAt: DateTime(2020), updatedAt: DateTime.now(), durationInSeconds: 0, type: AssetType.image, @@ -34,4 +34,19 @@ final class AssetStub { isArchived: false, isTrashed: false, ); + + static final image3 = Asset( + checksum: "image3-checksum", + localId: "image3", + ownerId: 1, + fileCreatedAt: DateTime(2025), + fileModifiedAt: DateTime(2025), + updatedAt: DateTime.now(), + durationInSeconds: 60, + type: AssetType.image, + fileName: "image3.jpg", + isFavorite: true, + isArchived: false, + isTrashed: false, + ); } diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart index c9287bfb1..192858adf 100644 --- a/mobile/test/infrastructure/repository.mock.dart +++ b/mobile/test/infrastructure/repository.mock.dart @@ -1,3 +1,4 @@ +import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; import 'package:immich_mobile/domain/interfaces/log.interface.dart'; import 'package:immich_mobile/domain/interfaces/store.interface.dart'; import 'package:immich_mobile/domain/interfaces/user.interface.dart'; @@ -10,5 +11,8 @@ class MockLogRepository extends Mock implements ILogRepository {} class MockUserRepository extends Mock implements IUserRepository {} +class MockDeviceAssetRepository extends Mock + implements IDeviceAssetRepository {} + // API Repos class MockUserApiRepository extends Mock implements IUserApiRepository {} diff --git a/mobile/test/service.mocks.dart b/mobile/test/service.mocks.dart index 33c325b10..8ee1c5860 100644 --- a/mobile/test/service.mocks.dart +++ b/mobile/test/service.mocks.dart @@ -1,4 +1,5 @@ import 'package:immich_mobile/services/api.service.dart'; +import 'package:immich_mobile/services/background.service.dart'; import 'package:immich_mobile/services/entity.service.dart'; import 'package:immich_mobile/services/hash.service.dart'; import 'package:immich_mobile/services/network.service.dart'; @@ -17,3 +18,5 @@ class MockEntityService extends Mock implements EntityService {} class MockNetworkService extends Mock implements NetworkService {} class MockSearchApi extends Mock implements SearchApi {} + +class MockBackgroundService extends Mock implements BackgroundService {} diff --git a/mobile/test/test_utils.dart b/mobile/test/test_utils.dart index a5a89a244..c0f789795 100644 --- a/mobile/test/test_utils.dart +++ b/mobile/test/test_utils.dart @@ -12,6 +12,7 @@ import 'package:immich_mobile/entities/backup_album.entity.dart'; import 'package:immich_mobile/entities/duplicated_asset.entity.dart'; import 'package:immich_mobile/entities/etag.entity.dart'; import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; +import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'; import 'package:immich_mobile/infrastructure/entities/log.entity.dart'; import 'package:immich_mobile/infrastructure/entities/store.entity.dart'; @@ -52,6 +53,7 @@ abstract final class TestUtils { ETagSchema, AndroidDeviceAssetSchema, IOSDeviceAssetSchema, + DeviceAssetEntitySchema, ], directory: "test/", maxSizeMiB: 1024,