From 73075c64d167d4089c50413019879059d4633050 Mon Sep 17 00:00:00 2001 From: Fynn Petersen-Frey <10599762+fyfrey@users.noreply.github.com> Date: Sat, 10 Jun 2023 20:13:59 +0200 Subject: [PATCH] feature(mobile): hash assets & sync via checksum (#2592) * compare different sha1 implementations * remove openssl sha1 * sync via checksum * hash assets in batches * hash in background, show spinner in tab * undo tmp changes * migrate by clearing assets * ignore duplicate assets * error handling * trigger sync/merge after download and update view * review feedback improvements * hash in background isolate on iOS * rework linking assets with existing from DB * fine-grained errors on unique index violation * hash lenth validation * revert compute in background on iOS * ignore duplicate assets on device * fix bug with batching based on accumulated size --------- Co-authored-by: Fynn Petersen-Frey --- mobile/android/app/build.gradle | 1 + .../example/mobile/BackgroundServicePlugin.kt | 39 ++- mobile/android/build.gradle | 1 + mobile/lib/main.dart | 3 + .../image_viewer_page_state.provider.dart | 11 +- .../asset_viewer/ui/top_control_app_bar.dart | 10 +- .../asset_viewer/views/gallery_viewer.dart | 2 +- .../background.service.dart | 11 + mobile/lib/modules/home/views/home_page.dart | 14 +- mobile/lib/shared/models/album.dart | 21 +- .../shared/models/android_device_asset.dart | 10 + .../shared/models/android_device_asset.g.dart | Bin 0 -> 13561 bytes mobile/lib/shared/models/asset.dart | 126 ++++----- mobile/lib/shared/models/asset.g.dart | Bin 66051 -> 72553 bytes mobile/lib/shared/models/device_asset.dart | 8 + .../lib/shared/models/ios_device_asset.dart | 14 + .../lib/shared/models/ios_device_asset.g.dart | Bin 0 -> 21482 bytes .../lib/shared/providers/asset.provider.dart | 31 +-- mobile/lib/shared/services/hash.service.dart | 175 +++++++++++++ mobile/lib/shared/services/sync.service.dart | 243 ++++++++++-------- .../lib/shared/views/tab_controller_page.dart | 46 +++- mobile/lib/utils/builtin_extensions.dart | 21 +- mobile/lib/utils/migration.dart | 8 +- mobile/pubspec.lock | 10 +- mobile/pubspec.yaml | 1 + .../test/asset_grid_data_structure_test.dart | 3 +- ...text.dart => builtin_extensions_test.dart} | 7 +- mobile/test/sync_service_test.dart | 64 ++--- 28 files changed, 601 insertions(+), 279 deletions(-) create mode 100644 mobile/lib/shared/models/android_device_asset.dart create mode 100644 mobile/lib/shared/models/android_device_asset.g.dart create mode 100644 mobile/lib/shared/models/device_asset.dart create mode 100644 mobile/lib/shared/models/ios_device_asset.dart create mode 100644 mobile/lib/shared/models/ios_device_asset.g.dart create mode 100644 mobile/lib/shared/services/hash.service.dart rename mobile/test/{builtin_extensions_text.dart => builtin_extensions_test.dart} (88%) diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index 158a27208..38ef303b6 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -84,6 +84,7 @@ flutter { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$kotlin_coroutines_version" implementation "androidx.work:work-runtime-ktx:$work_version" implementation "androidx.concurrent:concurrent-futures:$concurrent_version" implementation "com.google.guava:guava:$guava_version" diff --git a/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt b/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt index 1e9b2502d..6541ad575 100644 --- a/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt +++ b/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt @@ -1,10 +1,15 @@ package app.alextran.immich import android.content.Context +import android.util.Log import io.flutter.embedding.engine.plugins.FlutterPlugin import io.flutter.plugin.common.BinaryMessenger import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodChannel +import java.security.MessageDigest +import java.io.File +import java.io.FileInputStream +import kotlinx.coroutines.* /** * Android plugin for Dart `BackgroundService` @@ -16,6 +21,7 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler { private var methodChannel: MethodChannel? = null private var context: Context? = null + private val sha1: MessageDigest = MessageDigest.getInstance("SHA-1") override fun onAttachedToEngine(binding: FlutterPlugin.FlutterPluginBinding) { onAttachedToEngine(binding.applicationContext, binding.binaryMessenger) @@ -70,9 +76,40 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler { "isIgnoringBatteryOptimizations" -> { result.success(BackupWorker.isIgnoringBatteryOptimizations(ctx)) } + "digestFiles" -> { + val args = call.arguments>()!! + GlobalScope.launch(Dispatchers.IO) { + val buf = ByteArray(BUFSIZE) + val digest: MessageDigest = MessageDigest.getInstance("SHA-1") + val hashes = arrayOfNulls(args.size) + for (i in args.indices) { + val path = args[i] + var len = 0 + try { + val file = FileInputStream(path) + try { + while (true) { + len = file.read(buf) + if (len != BUFSIZE) break + digest.update(buf) + } + } finally { + file.close() + } + digest.update(buf, 0, len) + hashes[i] = digest.digest() + } catch (e: Exception) { + // skip this file + Log.w(TAG, "Failed to hash file ${args[i]}: $e") + } + } + result.success(hashes.asList()) + } + } else -> result.notImplemented() } } } -private const val TAG = "BackgroundServicePlugin" \ No newline at end of file +private const val TAG = "BackgroundServicePlugin" +private const val BUFSIZE = 2*1024*1024; diff --git a/mobile/android/build.gradle b/mobile/android/build.gradle index ee01aa0f1..5d8f2bf6e 100644 --- a/mobile/android/build.gradle +++ b/mobile/android/build.gradle @@ -1,5 +1,6 @@ buildscript { ext.kotlin_version = '1.8.20' + ext.kotlin_coroutines_version = '1.7.1' ext.work_version = '2.7.1' ext.concurrent_version = '1.1.0' ext.guava_version = '31.0.1-android' diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 3c8cedc9d..1dfc630d4 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -19,9 +19,11 @@ import 'package:immich_mobile/modules/settings/providers/notification_permission import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/routing/tab_navigation_observer.dart'; import 'package:immich_mobile/shared/models/album.dart'; +import 'package:immich_mobile/shared/models/android_device_asset.dart'; import 'package:immich_mobile/shared/models/asset.dart'; import 'package:immich_mobile/shared/models/etag.dart'; import 'package:immich_mobile/shared/models/exif_info.dart'; +import 'package:immich_mobile/shared/models/ios_device_asset.dart'; import 'package:immich_mobile/shared/models/logger_message.model.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; @@ -91,6 +93,7 @@ Future loadDb() async { DuplicatedAssetSchema, LoggerMessageSchema, ETagSchema, + Platform.isAndroid ? AndroidDeviceAssetSchema : IOSDeviceAssetSchema, ], directory: dir.path, maxSizeMiB: 256, diff --git a/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart b/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart index ffb2e980e..b95a31c13 100644 --- a/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart +++ b/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart @@ -2,6 +2,7 @@ import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; import 'package:fluttertoast/fluttertoast.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/modules/album/services/album.service.dart'; import 'package:immich_mobile/modules/asset_viewer/models/image_viewer_page_state.model.dart'; import 'package:immich_mobile/modules/asset_viewer/services/image_viewer.service.dart'; import 'package:immich_mobile/shared/models/asset.dart'; @@ -12,9 +13,13 @@ import 'package:immich_mobile/shared/ui/share_dialog.dart'; class ImageViewerStateNotifier extends StateNotifier { final ImageViewerService _imageViewerService; final ShareService _shareService; + final AlbumService _albumService; - ImageViewerStateNotifier(this._imageViewerService, this._shareService) - : super( + ImageViewerStateNotifier( + this._imageViewerService, + this._shareService, + this._albumService, + ) : super( ImageViewerPageState( downloadAssetStatus: DownloadAssetStatus.idle, ), @@ -34,6 +39,7 @@ class ImageViewerStateNotifier extends StateNotifier { toastType: ToastType.success, gravity: ToastGravity.BOTTOM, ); + _albumService.refreshDeviceAlbums(); } else { state = state.copyWith(downloadAssetStatus: DownloadAssetStatus.error); ImmichToast.show( @@ -66,5 +72,6 @@ final imageViewerStateProvider = ((ref) => ImageViewerStateNotifier( ref.watch(imageViewerServiceProvider), ref.watch(shareServiceProvider), + ref.watch(albumServiceProvider), )), ); diff --git a/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart b/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart index b79f240bb..ddd1b40a8 100644 --- a/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart +++ b/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart @@ -72,15 +72,7 @@ class TopControlAppBar extends HookConsumerWidget { color: Colors.grey[200], ), ), - if (!asset.isLocal) - IconButton( - onPressed: onDownloadPressed, - icon: Icon( - Icons.cloud_download_outlined, - color: Colors.grey[200], - ), - ), - if (asset.storage == AssetState.merged) + if (asset.storage == AssetState.remote) IconButton( onPressed: onDownloadPressed, icon: Icon( diff --git a/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart b/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart index 29e932b35..f67fdf576 100644 --- a/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart +++ b/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart @@ -287,7 +287,7 @@ class GalleryViewerPage extends HookConsumerWidget { isFavorite: asset().isFavorite, onMoreInfoPressed: showInfo, onFavorite: asset().isRemote ? () => toggleFavorite(asset()) : null, - onDownloadPressed: asset().storage == AssetState.local + onDownloadPressed: asset().isLocal ? null : () => ref.watch(imageViewerStateProvider.notifier).downloadAsset( diff --git a/mobile/lib/modules/backup/background_service/background.service.dart b/mobile/lib/modules/backup/background_service/background.service.dart index 319c2890f..5817e1d2b 100644 --- a/mobile/lib/modules/backup/background_service/background.service.dart +++ b/mobile/lib/modules/backup/background_service/background.service.dart @@ -132,6 +132,17 @@ class BackgroundService { } } + Future digestFile(String path) { + return _foregroundChannel.invokeMethod("digestFile", [path]); + } + + Future?> digestFiles(List paths) { + return _foregroundChannel.invokeListMethod( + "digestFiles", + paths, + ); + } + /// Updates the notification shown by the background service Future _updateNotification({ String? title, diff --git a/mobile/lib/modules/home/views/home_page.dart b/mobile/lib/modules/home/views/home_page.dart index 957b15133..292389c05 100644 --- a/mobile/lib/modules/home/views/home_page.dart +++ b/mobile/lib/modules/home/views/home_page.dart @@ -47,11 +47,11 @@ class HomePage extends HookConsumerWidget { useEffect( () { - ref.watch(websocketProvider.notifier).connect(); - ref.watch(assetProvider.notifier).getAllAsset(); - ref.watch(albumProvider.notifier).getAllAlbums(); - ref.watch(sharedAlbumProvider.notifier).getAllSharedAlbums(); - ref.watch(serverInfoProvider.notifier).getServerVersion(); + ref.read(websocketProvider.notifier).connect(); + Future(() => ref.read(assetProvider.notifier).getAllAsset()); + ref.read(albumProvider.notifier).getAllAlbums(); + ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums(); + ref.read(serverInfoProvider.notifier).getServerVersion(); selectionEnabledHook.addListener(() { multiselectEnabled.state = selectionEnabledHook.value; @@ -144,7 +144,7 @@ class HomePage extends HookConsumerWidget { ); if (remoteAssets.isNotEmpty) { await ref - .watch(assetProvider.notifier) + .read(assetProvider.notifier) .toggleArchive(remoteAssets, true); final assetOrAssets = remoteAssets.length > 1 ? 'assets' : 'asset'; @@ -163,7 +163,7 @@ class HomePage extends HookConsumerWidget { void onDelete() async { processing.value = true; try { - await ref.watch(assetProvider.notifier).deleteAssets(selection.value); + await ref.read(assetProvider.notifier).deleteAssets(selection.value); selectionEnabledHook.value = false; } finally { processing.value = false; diff --git a/mobile/lib/shared/models/album.dart b/mobile/lib/shared/models/album.dart index a3cffa169..81a834b99 100644 --- a/mobile/lib/shared/models/album.dart +++ b/mobile/lib/shared/models/album.dart @@ -166,23 +166,10 @@ extension AssetsHelper on IsarCollection { } } -extension AssetPathEntityHelper on AssetPathEntity { - Future> getAssets({ - int start = 0, - int end = 0x7fffffffffffffff, - Set? excludedAssets, - }) async { - final assetEntities = await getAssetListRange(start: start, end: end); - if (excludedAssets != null) { - return assetEntities - .where((e) => !excludedAssets.contains(e.id)) - .map(Asset.local) - .toList(); - } - return assetEntities.map(Asset.local).toList(); - } -} - extension AlbumResponseDtoHelper on AlbumResponseDto { List getAssets() => assets.map(Asset.remote).toList(); } + +extension AssetPathEntityHelper on AssetPathEntity { + String get eTagKeyAssetCount => "device-album-$id-asset-count"; +} diff --git a/mobile/lib/shared/models/android_device_asset.dart b/mobile/lib/shared/models/android_device_asset.dart new file mode 100644 index 000000000..b6b2663fd --- /dev/null +++ b/mobile/lib/shared/models/android_device_asset.dart @@ -0,0 +1,10 @@ +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:isar/isar.dart'; + +part 'android_device_asset.g.dart'; + +@Collection() +class AndroidDeviceAsset extends DeviceAsset { + AndroidDeviceAsset({required this.id, required super.hash}); + Id id; +} diff --git a/mobile/lib/shared/models/android_device_asset.g.dart b/mobile/lib/shared/models/android_device_asset.g.dart new file mode 100644 index 0000000000000000000000000000000000000000..ca7c822ba09f07be2ee7221a84505a98388b2230 GIT binary patch literal 13561 zcmdU0ZExE)5dQ98!4CuMtgf5XXq&i+hrD=QfGo+D48srHTr$7J2 zj{ai59G;&v8WT5`EE=+=8~Sl1e8=ba!sCvcBwTiU=xx4lG+w=8Ki2j^X{U)BA4fsJ zJt?B_0}pxZ$|$aRAr$9D_aOU@_rzTo#r)+^1RUGdPiGj#4mS0epH2edxsp4otIfhF zbi61`q#H^{_&k(iD0u9I?wBWSki4)4qEsxtQ4MXnn z1W=eco}0)vo5XwwvPr1=H%>i(1QrTJ$P*`s!aEosM{elsZadz{#lDz-Nkz;Z8M^|; z-NDrH*o6)ef;b+6D~HgN!Vs`#n1%$n0}4i9D_}pb3(x_~9Pvbe2wTb6ojQq(VTD3R zm<9nT!@bYY@hK9a6LXoyp)(b7gfs#)#~?Gaa414I0BbxqfEn&RlI~OD#8DdhfRG4f z);x(4fjYwY+b9Y+SPlvT`jVsVZZLIciIYsYCx$ae&L%u*H2B|=hY8GLAGkbJtUe)9 zA0nr7)-f1@JY@eiSb>oH<#~N}$0gHIV#!O`K_BMhM~+0&@iJDow%#|sHX7tnRu*^# zpp9L2z;caLQwzi)#d<7mmZvq_&@KEPd-?X=o89j2yPe&g*WK5^-wv99KV-}_vrV)|t5k%MUSLb#n`I!LH(wr^$_bdL% z6Dh_JKUd-(810my8dw8xLc(JKVX|QKmFdNyK7V-dN#0)=jy6|nV@<&^I8FHH8h7C< zoPm;mR{(j6?PU~SzT11(ef?&qyVu>`e*1QB`}OWF%3TO-8j3F|fHj0z;AUjZCxPn~ zTWz=AXqiA;RHWLi3m3%>?NOXfE=6~n^|2pv62HPQlH$Io31jGT{CreXG6p~b`i!ho zbbw6fLqsc>vSx^)Qy^RgJs@09eUu18&XlC;QPhb8T`65}R4MF`x^NJrf6N9w*6en6 zI@>RHz_SpU4M;RqTG%ATNGwMni*7$bHb>bQ1^XbV4_oLEl1qY(zq*rs1ht)a_aRpW z7iP#7#pOL@9*m^q7&4!PJfKB%0@x4UgB+{##R2QG7kc3iI;#`#@J^0!oXWS>a7sIv zSF7yH!s=1uFgN}g3&^!>N?)s5tfX}4>qzI2mbUt4`(xX-Pvy=E@GB@`MKT(7>O&x- zu4usT-1WhLn7&pufVM8JB{;0C9WKup`UjFXE!n}Z3S+m}Pd~969YC0YmC9gt=1@g5 za^_`~CoQuv1MFY+$@(gzk?T5v%v6kFi-P_eD4W2Z?u}S07p17I@&()Ok@=Yct;G;% z^S79ruuHTVQ~c0P18HQ)Q5;Rlg@ZVb;#TvI@N*bV!^{qb1qQ&ze#q5n5`JHC+XWU> zP=#_#tL!-KRAr71D20DV_~Zf#bz~G~pCd?*`~D0zOX2nV3l^d@r5Lf?f;S%@j9it{ zlt5G@uNyK|pe7C4Ishu@IcH^N9xz$kn3qmvg?~$VJo|G5HRV+lOY#`hPthm(&feqZTDl15urT z&O>?2<&^VK5gIc<5KVb(G9flFmQJ9SwVR6AgYveom*-$kp&|_Dk*oy?G1P49OikFZ zg9C=)({U5e0E}j8kcoxb9oWoRk&<(rxYb#Fg_0miuHkst%29)3pckY*|3l$Ob>vol|TvH5{DIx0`GVhK&(a( z$pPpLWYwG%U-KR?oi=|97g%b@*29HG{rb-lrJ$w`*tvT)GepO0(SAdeR@!_^0}-iPkRTcJqS2HM=|Le zFK{SD)|I&nOScNYJEi1pu3hjs1u9t1-K5|1Y;FbZ%p({JD7)Sz7&FAGN)l83qPDda z90esyklN+1_Hs(C}v&Wh;oAL_Z<8+lE8xMB+aq%EzA=Clu5 zwKY6tu#SLKnp(YU3xrbb@<49U<%(r7?p$^$o`nuso5zpd=oL6c5|HiLO!{SJno6La z!+9Yo*qQ5$p6S>bUOnZhCB+ButV%7A-?)_$w_48_4$ld*6B>g2P;ae;`2!f8CWG-r z!X>OeSFUtRu@olRv+*Jv3p&<^1RhlWKQIc#^d5bxsk1If6u-+GktS}%RYd*!{%V{m zhM3N42G;cgSy@7D9F~tj#@OaC^Sj!&mm=~5+pyuRzDUS_+E)jrbsfZ>d&0FjXD9ux z5pJ;At=IZbPQoumJ6!52J=T&p;o)mx+pqJyv?(CZ?@b$r@Keu7n*#4#f;A5%RE~jP zs`9U3GaVGHgU|PHESiIy(s0i{24XT<=b>ns8@CI3agI)2HQu-_vogJl^M5RiPy3H% ztF5eLz}@09mg3;+5fAHFRLNQSQd8@%Pu%c!IVe}-D?3VjD{f*$) z3wT~V=TO-DDy;B9exWW}h5_`%?F3Hy5dJH^sAZ(ALV-+%_P?(+JDU9hm&Uc>XO=md z0Tbvt`U%io>KEi!i&uJc!-!fEI=Hoi7vpH=1-$Y^|1~4&!;5)FPHjLLf%r^N1x*1c ZF#geZPtMGX&s9nz`ev2qQoGv2{{j49x0L_@ literal 0 HcmV?d00001 diff --git a/mobile/lib/shared/models/asset.dart b/mobile/lib/shared/models/asset.dart index 840653554..701b6c2dd 100644 --- a/mobile/lib/shared/models/asset.dart +++ b/mobile/lib/shared/models/asset.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/utils/hash.dart'; @@ -14,7 +16,7 @@ part 'asset.g.dart'; class Asset { Asset.remote(AssetResponseDto remote) : remoteId = remote.id, - isLocal = false, + checksum = remote.checksum, fileCreatedAt = remote.fileCreatedAt, fileModifiedAt = remote.fileModifiedAt, updatedAt = remote.updatedAt, @@ -24,23 +26,20 @@ class Asset { height = remote.exifInfo?.exifImageHeight?.toInt(), width = remote.exifInfo?.exifImageWidth?.toInt(), livePhotoVideoId = remote.livePhotoVideoId, - localId = remote.deviceAssetId, - deviceId = fastHash(remote.deviceId), ownerId = fastHash(remote.ownerId), exifInfo = remote.exifInfo != null ? ExifInfo.fromDto(remote.exifInfo!) : null, isFavorite = remote.isFavorite, isArchived = remote.isArchived; - Asset.local(AssetEntity local) + Asset.local(AssetEntity local, List hash) : localId = local.id, - isLocal = true, + checksum = base64.encode(hash), durationInSeconds = local.duration, type = AssetType.values[local.typeInt], height = local.height, width = local.width, fileName = local.title!, - deviceId = Store.get(StoreKey.deviceIdHash), ownerId = Store.get(StoreKey.currentUser).isarId, fileModifiedAt = local.modifiedDateTime, updatedAt = local.modifiedDateTime, @@ -53,13 +52,15 @@ class Asset { if (local.latitude != null) { exifInfo = ExifInfo(lat: local.latitude, long: local.longitude); } + _local = local; + assert(hash.length == 20, "invalid SHA1 hash"); } Asset({ this.id = Isar.autoIncrement, + required this.checksum, this.remoteId, required this.localId, - required this.deviceId, required this.ownerId, required this.fileCreatedAt, required this.fileModifiedAt, @@ -72,7 +73,6 @@ class Asset { this.livePhotoVideoId, this.exifInfo, required this.isFavorite, - required this.isLocal, required this.isArchived, }); @@ -83,7 +83,7 @@ class Asset { AssetEntity? get local { if (isLocal && _local == null) { _local = AssetEntity( - id: localId, + id: localId!, typeInt: isImage ? 1 : 2, width: width ?? 0, height: height ?? 0, @@ -98,18 +98,21 @@ class Asset { Id id = Isar.autoIncrement; + /// stores the raw SHA1 bytes as a base64 String + /// because Isar cannot sort lists of byte arrays + @Index( + unique: true, + replace: false, + type: IndexType.hash, + composite: [CompositeIndex("ownerId")], + ) + String checksum; + @Index(unique: false, replace: false, type: IndexType.hash) String? remoteId; - @Index( - unique: false, - replace: false, - type: IndexType.hash, - composite: [CompositeIndex('deviceId')], - ) - String localId; - - int deviceId; + @Index(unique: false, replace: false, type: IndexType.hash) + String? localId; int ownerId; @@ -134,14 +137,15 @@ class Asset { bool isFavorite; - /// `true` if this [Asset] is present on the device - bool isLocal; - bool isArchived; @ignore ExifInfo? exifInfo; + /// `true` if this [Asset] is present on the device + @ignore + bool get isLocal => localId != null; + @ignore bool get isInDb => id != Isar.autoIncrement; @@ -175,9 +179,9 @@ class Asset { bool operator ==(other) { if (other is! Asset) return false; return id == other.id && + checksum == other.checksum && remoteId == other.remoteId && localId == other.localId && - deviceId == other.deviceId && ownerId == other.ownerId && fileCreatedAt.isAtSameMomentAs(other.fileCreatedAt) && fileModifiedAt.isAtSameMomentAs(other.fileModifiedAt) && @@ -197,9 +201,9 @@ class Asset { @ignore int get hashCode => id.hashCode ^ + checksum.hashCode ^ remoteId.hashCode ^ localId.hashCode ^ - deviceId.hashCode ^ ownerId.hashCode ^ fileCreatedAt.hashCode ^ fileModifiedAt.hashCode ^ @@ -217,8 +221,7 @@ class Asset { /// Returns `true` if this [Asset] can updated with values from parameter [a] bool canUpdate(Asset a) { assert(isInDb); - assert(localId == a.localId); - assert(deviceId == a.deviceId); + assert(checksum == a.checksum); assert(a.storage != AssetState.merged); return a.updatedAt.isAfter(updatedAt) || a.isRemote && !isRemote || @@ -239,11 +242,18 @@ class Asset { if (a.isRemote) { return a._copyWith( id: id, - isLocal: isLocal, + localId: localId, width: a.width ?? width, height: a.height ?? height, exifInfo: a.exifInfo?.copyWith(id: id) ?? exifInfo, ); + } else if (isRemote) { + return _copyWith( + localId: localId ?? a.localId, + width: width ?? a.width, + height: height ?? a.height, + exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id), + ); } else { return a._copyWith( id: id, @@ -270,7 +280,7 @@ class Asset { } else { // add only missing values (and set isLocal to true) return _copyWith( - isLocal: true, + localId: localId ?? a.localId, width: width ?? a.width, height: height ?? a.height, exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id), @@ -281,9 +291,9 @@ class Asset { Asset _copyWith({ Id? id, + String? checksum, String? remoteId, String? localId, - int? deviceId, int? ownerId, DateTime? fileCreatedAt, DateTime? fileModifiedAt, @@ -295,15 +305,14 @@ class Asset { String? fileName, String? livePhotoVideoId, bool? isFavorite, - bool? isLocal, bool? isArchived, ExifInfo? exifInfo, }) => Asset( id: id ?? this.id, + checksum: checksum ?? this.checksum, remoteId: remoteId ?? this.remoteId, localId: localId ?? this.localId, - deviceId: deviceId ?? this.deviceId, ownerId: ownerId ?? this.ownerId, fileCreatedAt: fileCreatedAt ?? this.fileCreatedAt, fileModifiedAt: fileModifiedAt ?? this.fileModifiedAt, @@ -315,7 +324,6 @@ class Asset { fileName: fileName ?? this.fileName, livePhotoVideoId: livePhotoVideoId ?? this.livePhotoVideoId, isFavorite: isFavorite ?? this.isFavorite, - isLocal: isLocal ?? this.isLocal, isArchived: isArchived ?? this.isArchived, exifInfo: exifInfo ?? this.exifInfo, ); @@ -328,39 +336,36 @@ class Asset { } } - /// compares assets by [ownerId], [deviceId], [localId] - static int compareByOwnerDeviceLocalId(Asset a, Asset b) { - final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); - if (ownerIdOrder != 0) { - return ownerIdOrder; - } - final int deviceIdOrder = a.deviceId.compareTo(b.deviceId); - if (deviceIdOrder != 0) { - return deviceIdOrder; - } - final int localIdOrder = a.localId.compareTo(b.localId); - return localIdOrder; - } - - /// compares assets by [ownerId], [deviceId], [localId], [fileModifiedAt] - static int compareByOwnerDeviceLocalIdModified(Asset a, Asset b) { - final int order = compareByOwnerDeviceLocalId(a, b); - return order != 0 ? order : a.fileModifiedAt.compareTo(b.fileModifiedAt); - } - static int compareById(Asset a, Asset b) => a.id.compareTo(b.id); - static int compareByLocalId(Asset a, Asset b) => - a.localId.compareTo(b.localId); + static int compareByChecksum(Asset a, Asset b) => + a.checksum.compareTo(b.checksum); + + static int compareByOwnerChecksum(Asset a, Asset b) { + final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); + if (ownerIdOrder != 0) return ownerIdOrder; + return compareByChecksum(a, b); + } + + static int compareByOwnerChecksumCreatedModified(Asset a, Asset b) { + final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); + if (ownerIdOrder != 0) return ownerIdOrder; + final int checksumOrder = compareByChecksum(a, b); + if (checksumOrder != 0) return checksumOrder; + final int createdOrder = a.fileCreatedAt.compareTo(b.fileCreatedAt); + if (createdOrder != 0) return createdOrder; + return a.fileModifiedAt.compareTo(b.fileModifiedAt); + } @override String toString() { return """ { + "id": ${id == Isar.autoIncrement ? '"N/A"' : id}, "remoteId": "${remoteId ?? "N/A"}", - "localId": "$localId", - "deviceId": "$deviceId", - "ownerId": "$ownerId", + "localId": "${localId ?? "N/A"}", + "checksum": "$checksum", + "ownerId": $ownerId, "livePhotoVideoId": "${livePhotoVideoId ?? "N/A"}", "fileCreatedAt": "$fileCreatedAt", "fileModifiedAt": "$fileModifiedAt", @@ -369,9 +374,8 @@ class Asset { "type": "$type", "fileName": "$fileName", "isFavorite": $isFavorite, - "isLocal": $isLocal, "isRemote: $isRemote, - "storage": $storage, + "storage": "$storage", "width": ${width ?? "N/A"}, "height": ${height ?? "N/A"}, "isArchived": $isArchived @@ -424,10 +428,6 @@ extension AssetsHelper on IsarCollection { QueryBuilder _remote(Iterable ids) => where().anyOf(ids, (q, String e) => q.remoteIdEqualTo(e)); QueryBuilder _local(Iterable ids) { - return where().anyOf( - ids, - (q, String e) => - q.localIdDeviceIdEqualTo(e, Store.get(StoreKey.deviceIdHash)), - ); + return where().anyOf(ids, (q, String e) => q.localIdEqualTo(e)); } } diff --git a/mobile/lib/shared/models/asset.g.dart b/mobile/lib/shared/models/asset.g.dart index e7085bad4cb0ea2c2cab2cf20c0ab0e97daa52c3..713c26885b95a41168d52c4c1b8ca7d3433b6f37 100644 GIT binary patch delta 5376 zcmcgwdu&rx7|#`??dsNc-PU#O=GlVo!5(cl#-5!6hgPT4L5R8)xU^T+(e+|`3(kpG zqab34j*l20iNu6N(a6J__&}o~ipEDY8jUe3OEkVB5&lpl;o-TD({o$8X>q!w>pkZ? z=l6Z*_dU*cd+&X7e|~Z9{#WN1LnB;hT_QOKmBwGLOTzb##xky|*9*Bjx+Yui_fdm_0!bhX{L=isL)tg9gf*(y_7cET+$vCIZ^!U+Q zPFOs?M2n~a*}s8s%`zb#iLFJ1ih2}@2}p5%AY7}UAQa~?PE6gesI3vR`5X;V6b&I* z+llSil!yN*cJ*X6V%c~sl$#+UK8a$9vNd;OG-lTDeAJJ^T$B^IoTw2qxgi)TJ-Hhs zTQZ#?S?CeRW^z=TG5?OC{zyU~0i;AwZ#0UT5;LMms|U(zHKP2fkXpe;lU$-6DMB1w zeiB})fCSmCkr*3AIFV6z##Sx6y@H69O7xH*iQ8;ZOqCFL?*99o0&tBBYw3t?ou$iq+w9TXL!*!-56mL`YNG$cDo;soYR49*W3iNX^02jG8T4zUb3VAzdF@qI@Z+ zHCnE=nj)HPdbi0br+O{jX-Z&RpyNc@rR1DBd2V(bCtk4UX2)@2?w*@I57l6!&eMvM zk}7H{pXWYNF5MUCyV-D^h~59cC!Nl9;(wiVI^OkpN77D|@6IfFCtsertr?f?zsb0C zuZcO-%Gt@d^n??Y*!yR-Qs=f=^O_jUUMqFnD{jj<=>lOtTA$3)Cn@cV);QK8QEsnN z65t^v6_H z*W;T`gXo}-)F7?mD7OY?K%wBgnj@DSdawAAsA8wC0j*2KR-JcUk+ezUBSG~O#WF3~ z^eX4m>y}lQg@gpT>N9u(>uyIs!TOs(&`v>+5amZl6-@M0NM|bc7(JxL?Yk|>Mwj!w zvGD<|lh(>{MnesUIyl(XRAC|;SHXwiG^N@BIM8`bGm%}-3pwDnfv5F>%r`d1=K{Xb zN#9Q%Zft?m4TD+M>DQFP@wWM}%D43&uCq;pnW=liLe8eq1Sjf>i&71qu8RfM(*bu6 zTESn}nNF59MV8hw!M@IG7i)AAc#Str3ng-f_X@1>@QlaBJf_Sis%`bzC^~OhrO!|4 z%}#5!J*k5SVbZ(i0*q^g^*$4P)Y$Esfgc#_D033x)WFxh^}#)hBiIS$JG?enP?Mi& z#EGur!qi^U!Ukb#+okYqB+r^$zn+VC&{sh@zEkh%dMCl%<0WM@5EOHBvZ9Vy5u~}B zshxPOse4^^BldQev=AK1M3mpi#kEkPvkEu~XC}#!f<<~bnW=@vx`?Kz(9~T8 zS4h3!smzCuYmL(0>aYR!R&Rqh{08`7vJeJq2DG0v*yM7ZdToIj&UP8$#@f5U+*AO6 z)>&cL(?fD91#+}0q{%+Upk^rW6~NKv0t7fZFInQl){r+exeg;Qd?IsmOc;Ge3`Au+ecpaT+2)<7TY0#4^ zk4z&uvFe>^BsZ`+$x{zMlt-KwNWiYAV_@A_PIr2rX6c~fzMXV{CDK^mmkrYO!q-+< zDeQpXHd|mc+y|aX6L85#bYlb5ZMMVq$6tV{?T^BiEe<&T#4pNo#nvMHybq?fekZwi zF$|t8#GW<4!0;Cs$RNG^;15m4CFG~U$=3&U798%Ul0$E~^ZD9L}+|*K51(uDiai?YN<^1o~82@^Ljip~)5gAtNx81V}xnoS}vAv%0XOn^}DdM{g@ka%D2f4|>z zzvuTnzyJO3_Gh`T4d+gLnWKxw_V@0N1*5o2xiOo=ka7T{S_h8i7VxWb*&MQWZz!?5 zE1m)Mkc|5LJ88$1wg5HcT3jQ|I7miNuQOxBMB|J$AMcVD6zNuCT*qL$QHlvgF=aI1 zPs$0FVvsg6_)gbARKP+Vfl1GRr#grL^7Zrll(C%9?##4=3r4r4s?uNW_f=H{yx#Sd zfpwL>%790Y<0cC!fnr!=`Wi(>pGJD;)jSS+jkSE*R7Rka23Xob#ziTD;uyUehnT&0c(fKj#- z_l5iSY>6P`0b51*;kWn%8QqrmY_g}(8ZH^@ipG*<;w_ZmpRgtgO7^yf!=Tfev$tbC zj@W7hhD){;n72|Q%RSp_!iSai3Su3evu_a?M(iPh;f{T+z~FFPMWd^THxzzB;IPw; z+s+qo$H5`ya)}9i;9@ZAU|?`@aJdp8Ps@9(fS@3gsm*a8hO)x$6 zCcbSU)whb9FRAO9DkpwzX-7JEAPXbKKT>hU%?SZRfo{t~y4(jYOzM;dG1q8eIeEq& zuD5T1lP7w-`t_`;LcffPQ$~b~{$RbVQBKMxCkow%~$Nldp|kzZ{9y`e8l$z$NPtsSaui}>@i`D2)A4GC6yosW?Y z4i392HUnJb?qKa$R!TQNFi`@f~l zcu=P^|4WC+(sE@v0`)<$x>gz*`ronmmv$ac9Ec^Av3gWN`}+HpFa~R_xLg0pe<`Cq zLd8V|M0o*whuYAhGE!Gh!`(JaN97Mm*+ZD?I;P3QKumUPr8MG`v&vG4E}YUkqt_O| z3*qyz-{kPbW)`MTClHP@{MD{+))vUei$b?6k%(io=dgO5%?w9z=oHPLJ~mCGB%Ftf zC$xB&oCJO56js27$iVH)y6JsZSf9o5x2w@Np2AOK1$gT~TF9Akelsbs?t$sx9z@QP z{H+gflq=pdH`~>p%2#vSlyKz64n<+)m+gG{4{1sp%=~6By7gHM{z$1UcFl%JDHX@i z^&Og!c!2YB*X#94v22b;G(XfTsVttLnBj-#pD{d|t3kvp<%92!ZT$4TzbU0KbF(Qm zK@oYBGSLzczOvNExdeM|TtgT%izq~3zBbeMBGfrOu~C~EV+cDbZ;I3;OGL5?t~w{= nALgbuT8KhPcm|`B$0}A^mFl(;uGB> hash; +} diff --git a/mobile/lib/shared/models/ios_device_asset.dart b/mobile/lib/shared/models/ios_device_asset.dart new file mode 100644 index 000000000..0c55c74eb --- /dev/null +++ b/mobile/lib/shared/models/ios_device_asset.dart @@ -0,0 +1,14 @@ +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:immich_mobile/utils/hash.dart'; +import 'package:isar/isar.dart'; + +part 'ios_device_asset.g.dart'; + +@Collection() +class IOSDeviceAsset extends DeviceAsset { + IOSDeviceAsset({required this.id, required super.hash}); + + @Index(replace: true, unique: true, type: IndexType.hash) + String id; + Id get isarId => fastHash(id); +} diff --git a/mobile/lib/shared/models/ios_device_asset.g.dart b/mobile/lib/shared/models/ios_device_asset.g.dart new file mode 100644 index 0000000000000000000000000000000000000000..f10c3decdaa9c28011b45164a12b65da808b9bc4 GIT binary patch literal 21482 zcmdU1ZByI07XF@Jp$~V4uuW5F+5%~t4iwsDhL-Na-p&rw@wl-<)EYar9oXC*{`)=W zNU|l_gg~4S@}V?(J4cTmoflcovuEt%!RLcNcTW%Y+56-D1NM~dAG6QLr|i@5{^5te z+1}sm_ubF?jmF4NG8PZmnut>`;FqG$JwHu()()U%?RBH^>>2y5whxYWnEJ{4I1G6| z6LIvBM?CSfIH_3>4%d$_VeE6>73Wc$@TUV2ax9l$y+NFKSkz^~cod4hpK(uCwOABK zUO$e~%#Sip1U$;bK=8zi{0pA8VB{OCM0DwgBJjp3_u_#!O5zM=$^-9$Uwr3Di;bg* z_jw8^OuW9IW-T^K_yESHLFK>6xDSwkpio3S^};whhX&cukG$;-uRrv$F5$n%BH><^ z_yXFUL)VK7A1Xuy!|@1~asV}BXachtj3WZv0|jH4D`0Qy0#tx$4tXkI2vf<#zw**7 z0SQh;G!8>hhWCF$#gAA-Uc$3+5_wl58zPMW%?lVapW#46eh9Pa`yur3FJr*zN9rZ< zI0^tE5oLMtC{6|H2<^YeamZojpdg^1ah$s!Uip*MOGmsf1`{uvjCk5;@P9HMrO=Cg z#rNvjRvhB zvxhhVcrJYQmKE}4Q4@qCnYt`lv%9Rdph^T?_H^^r*5=C>ueM%xwx4hAyx898Y+xNw zYQ&RF@U%;FU@*kcPlsz=_C;1r3K0;oyV z>+(;sCW5IVHuY#)1hm&hjsQs~H9QEp#{cp(6Bn=vPsIN~8PCT3z^KH=DNh9Kvx=4{ zx(1sC{N{%HMV-?WVJvB2G{<`EB7)Cq2?Y@9P)=6>m;W)>hG09y((d9k(GdAWrn2tf zsv4plf+u|q;~14s-m(sRq6D{b<=P>S&a>gvb|O%%*l((p) zkG|(?BEXNaEBZ4pCUTTE{n=BNyQy$F+xXTp5rDO0O04W~LXJIezF9OEfplxmIy!7J ztRl@i0HcT#!dd>w{Qzc}(4To{NwI^P3aq=W6^dpIml*shiEHDZ3N+T)yLaqNStE3e zM)MLDWUUA?MFL4;BET1S^$jG@#J>5`BS5Nfz}=(F&_wQ5ERjoghs}^eoYfz)W-*l9 zB+?|ajV_^*Gq`C?)j>v~vd;XJeL;$-AvUKBVWG8db`AKzABUOln+=oriU@I#ByrMQ z`z!hx#aB@d0mFC^c*Fiv%w&yVzs9)gu(Z+0$Te*WC~8OqtmSG;=S-_cwgAZKjwYI}o@&)8N{~yicmPc~(=bF^ZMKyhUmR$rEYxcT+zx{t z8;xlLPBULQcVt_0($dov!~yf91|;$JZdYci#r~zOmHL;EK0*Ww9L7xLTJu{mwUA~` zMFof~@7*o_Tj-&PA$c6Y-h#m_YS@}Vj7f24zQqa^%9}+u#cc)lRz<1`OC^LfuGmGn zAoSn(nHh>gk1`r?(lG^&DL&zkF;6C6hj0vZ5+@m{1jim!x;GXfgk_Ega6M0p{XyNO z)3{s5-2nu*s<$6a(CvX^aY$ee+4e`HaMHwMl@@FMLfv)ioT60P_Je>1ejoa9z}8ep zE@oL9BsSNTX}ng%nF*;mP?GA*hm{^fMk@!u#(sDj%fJD=D?|qbqsB$?xE2%-E&RUo z?27Y9OJ*#LuXxf`@-@8~8;{@=$}M(*2TV;BZzo;B9fgP5PIC62<4glHHzH!7sMfyH z`-HuH%W%K6{j{%v#=We-CJHKju$VEIk5(nFnn6}bE7SB8j&Peca&TPR561!jOX5i< z1nQVX3Mq~W ztAy2T99uY0%53kMMjc=xQy?9)wCGSqW=WN)txn;b(vj?his)n?vzDxvQM(XY+HAbZ zQI+O~2FsiTI4x;%X-XxJX5x@<1l2Rlc`cisP7zFp1h(iCCP&tod7e;D}NT<5&+k zD`OfRXr+m`)i_$n%Rag7QI5RA_AJ6YPSt;!niQc_TJd75=H!x7)D&B~iCpd^>o=2A z^>;un(4V$?(@0irDsMZe`XEBc=4tMCKF^bV-^WovH>>V_TIG(TRsw-I2svK((v})e z%E=7YWt|<-P3L>N6tU_Dig?Y+Szl$J>5mRj^xm1zJUd$T$^e-OKGo((L6%PiW^ScZ zeblnHyvk9s1f`kls|n=BGf$v2ZuzCO$%+A;9}TQ1%K0h5pwC;1?Q}h@fO7ZbVzHznLY zaQR+lV>j;D2REJY92#o`S8jSPxbMQG8rRyk&q{mjt1r3=Lba@UL-d4oMJAn z;5u~QPvBKBXBI1AGip6@g4@&1=~n63|IW$6TKE$vC{1Dzr%&R z+SXeO*qea)q?>e(^|>=g^@#AR4!O!;5!8||;Rr!1KvTjKME94htOE>Zy0;nIFY1@r zXgYU}4QRLPA!gtM+_QogwyD=Kyp>D8?w7*vOAxE|IJI#>4~TC9p)-|x#{uXzJ{-ZK i)2~2PDUJlwQWmB5eMl^iyQ(x6!J_pAhB^>mH~tS?8YTk( literal 0 HcmV?d00001 diff --git a/mobile/lib/shared/providers/asset.provider.dart b/mobile/lib/shared/providers/asset.provider.dart index c1384f402..54e969426 100644 --- a/mobile/lib/shared/providers/asset.provider.dart +++ b/mobile/lib/shared/providers/asset.provider.dart @@ -18,11 +18,7 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; import 'package:photo_manager/photo_manager.dart'; -/// State does not contain archived assets. -/// Use database provider if you want to access the isArchived assets -class AssetsState {} - -class AssetNotifier extends StateNotifier { +class AssetNotifier extends StateNotifier { final AssetService _assetService; final AlbumService _albumService; final UserService _userService; @@ -38,7 +34,7 @@ class AssetNotifier extends StateNotifier { this._userService, this._syncService, this._db, - ) : super(AssetsState()); + ) : super(false); Future getAllAsset({bool clear = false}) async { if (_getAllAssetInProgress || _deleteInProgress) { @@ -48,14 +44,15 @@ class AssetNotifier extends StateNotifier { final stopwatch = Stopwatch()..start(); try { _getAllAssetInProgress = true; + state = true; if (clear) { await clearAssetsAndAlbums(_db); log.info("Manual refresh requested, cleared assets and albums from db"); } + await _userService.refreshUsers(); final bool newRemote = await _assetService.refreshRemoteAssets(); final bool newLocal = await _albumService.refreshDeviceAlbums(); debugPrint("newRemote: $newRemote, newLocal: $newLocal"); - await _userService.refreshUsers(); final List partners = await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll(); for (User u in partners) { @@ -64,6 +61,7 @@ class AssetNotifier extends StateNotifier { log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms"); } finally { _getAllAssetInProgress = false; + state = false; } } @@ -79,6 +77,7 @@ class AssetNotifier extends StateNotifier { Future deleteAssets(Set deleteAssets) async { _deleteInProgress = true; + state = true; try { final localDeleted = await _deleteLocalAssets(deleteAssets); final remoteDeleted = await _deleteRemoteAssets(deleteAssets); @@ -91,24 +90,14 @@ class AssetNotifier extends StateNotifier { } } finally { _deleteInProgress = false; + state = false; } } Future> _deleteLocalAssets(Set assetsToDelete) async { - final int deviceId = Store.get(StoreKey.deviceIdHash); - final List local = []; + final List local = + assetsToDelete.where((a) => a.isLocal).map((a) => a.localId!).toList(); // Delete asset from device - for (final Asset asset in assetsToDelete) { - if (asset.isLocal) { - local.add(asset.localId); - } else if (asset.deviceId == deviceId) { - // Delete asset on device if it is still present - var localAsset = await AssetEntity.fromId(asset.localId); - if (localAsset != null) { - local.add(localAsset.id); - } - } - } if (local.isNotEmpty) { try { return await PhotoManager.editor.deleteWithIds(local); @@ -153,7 +142,7 @@ class AssetNotifier extends StateNotifier { } } -final assetProvider = StateNotifierProvider((ref) { +final assetProvider = StateNotifierProvider((ref) { return AssetNotifier( ref.watch(assetServiceProvider), ref.watch(albumServiceProvider), diff --git a/mobile/lib/shared/services/hash.service.dart b/mobile/lib/shared/services/hash.service.dart new file mode 100644 index 000000000..ee272cf5f --- /dev/null +++ b/mobile/lib/shared/services/hash.service.dart @@ -0,0 +1,175 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/modules/backup/background_service/background.service.dart'; +import 'package:immich_mobile/shared/models/android_device_asset.dart'; +import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:immich_mobile/shared/models/ios_device_asset.dart'; +import 'package:immich_mobile/shared/providers/db.provider.dart'; +import 'package:immich_mobile/utils/builtin_extensions.dart'; +import 'package:isar/isar.dart'; +import 'package:logging/logging.dart'; +import 'package:photo_manager/photo_manager.dart'; + +class HashService { + HashService(this._db, this._backgroundService); + final Isar _db; + final BackgroundService _backgroundService; + final _log = Logger('HashService'); + + /// Returns all assets that were successfully hashed + Future> getHashedAssets( + AssetPathEntity album, { + int start = 0, + int end = 0x7fffffffffffffff, + Set? excludedAssets, + }) async { + final entities = await album.getAssetListRange(start: start, end: end); + final filtered = excludedAssets == null + ? entities + : entities.where((e) => !excludedAssets.contains(e.id)).toList(); + return _hashAssets(filtered); + } + + /// Converts a list of [AssetEntity]s to [Asset]s including only those + /// that were successfully hashed. Hashes are looked up in a DB table + /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing + /// entries are newly hashed and added to the DB table. + Future> _hashAssets(List assetEntities) async { + const int batchFileCount = 128; + const int batchDataSize = 1024 * 1024 * 1024; // 1GB + + final ids = assetEntities + .map(Platform.isAndroid ? (a) => a.id.toInt() : (a) => a.id) + .toList(); + final List hashes = await _lookupHashes(ids); + final List toAdd = []; + final List toHash = []; + + int bytes = 0; + + for (int i = 0; i < assetEntities.length; i++) { + if (hashes[i] != null) { + continue; + } + final file = await assetEntities[i].originFile; + if (file == null) { + _log.warning( + "Failed to get file for asset ${assetEntities[i].id}, skipping", + ); + continue; + } + bytes += await file.length(); + toHash.add(file.path); + final deviceAsset = Platform.isAndroid + ? AndroidDeviceAsset(id: ids[i] as int, hash: const []) + : IOSDeviceAsset(id: ids[i] as String, hash: const []); + toAdd.add(deviceAsset); + hashes[i] = deviceAsset; + if (toHash.length == batchFileCount || bytes >= batchDataSize) { + await _processBatch(toHash, toAdd); + toAdd.clear(); + toHash.clear(); + bytes = 0; + } + } + if (toHash.isNotEmpty) { + await _processBatch(toHash, toAdd); + } + return _mapAllHashedAssets(assetEntities, hashes); + } + + /// Lookup hashes of assets by their local ID + Future> _lookupHashes(List ids) => + Platform.isAndroid + ? _db.androidDeviceAssets.getAll(ids.cast()) + : _db.iOSDeviceAssets.getAllById(ids.cast()); + + /// Processes a batch of files and saves any successfully hashed + /// values to the DB table. + Future _processBatch( + final List toHash, + final List toAdd, + ) async { + final hashes = await _hashFiles(toHash); + bool anyNull = false; + for (int j = 0; j < hashes.length; j++) { + if (hashes[j]?.length == 20) { + toAdd[j].hash = hashes[j]!; + } else { + _log.warning("Failed to hash file ${toHash[j]}, skipping"); + anyNull = true; + } + } + final validHashes = anyNull + ? toAdd.where((e) => e.hash.length == 20).toList(growable: false) + : toAdd; + await _db.writeTxn( + () => Platform.isAndroid + ? _db.androidDeviceAssets.putAll(validHashes.cast()) + : _db.iOSDeviceAssets.putAll(validHashes.cast()), + ); + _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); + } + + /// Hashes the given files and returns a list of the same length + /// files that could not be hashed have a `null` value + Future> _hashFiles(List paths) async { + if (Platform.isAndroid) { + final List? hashes = + await _backgroundService.digestFiles(paths); + if (hashes == null) { + throw Exception("Hashing ${paths.length} files failed"); + } + return hashes; + } else if (Platform.isIOS) { + final List result = List.filled(paths.length, null); + for (int i = 0; i < paths.length; i++) { + result[i] = await _hashAssetDart(File(paths[i])); + } + return result; + } else { + throw Exception("_hashFiles implementation missing"); + } + } + + /// Hashes a single file using Dart's crypto package + Future _hashAssetDart(File f) async { + late Digest output; + final sink = sha1.startChunkedConversion( + ChunkedConversionSink.withCallback((accumulated) { + output = accumulated.first; + }), + ); + await for (final chunk in f.openRead()) { + sink.add(chunk); + } + sink.close(); + return Uint8List.fromList(output.bytes); + } + + /// Converts [AssetEntity]s that were successfully hashed to [Asset]s + List _mapAllHashedAssets( + List assets, + List hashes, + ) { + final List result = []; + for (int i = 0; i < assets.length; i++) { + if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { + result.add(Asset.local(assets[i], hashes[i]!.hash)); + } + } + return result; + } +} + +final hashServiceProvider = Provider( + (ref) => HashService( + ref.watch(dbProvider), + ref.watch(backgroundServiceProvider), + ), +); diff --git a/mobile/lib/shared/services/sync.service.dart b/mobile/lib/shared/services/sync.service.dart index f5471d154..0bb237836 100644 --- a/mobile/lib/shared/services/sync.service.dart +++ b/mobile/lib/shared/services/sync.service.dart @@ -4,10 +4,12 @@ import 'package:collection/collection.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/shared/models/album.dart'; import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/etag.dart'; import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; import 'package:immich_mobile/shared/providers/db.provider.dart'; +import 'package:immich_mobile/shared/services/hash.service.dart'; import 'package:immich_mobile/utils/async_mutex.dart'; import 'package:immich_mobile/utils/builtin_extensions.dart'; import 'package:immich_mobile/utils/diff.dart'; @@ -16,15 +18,17 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; import 'package:photo_manager/photo_manager.dart'; -final syncServiceProvider = - Provider((ref) => SyncService(ref.watch(dbProvider))); +final syncServiceProvider = Provider( + (ref) => SyncService(ref.watch(dbProvider), ref.watch(hashServiceProvider)), +); class SyncService { final Isar _db; + final HashService _hashService; final AsyncMutex _lock = AsyncMutex(); final Logger _log = Logger('SyncService'); - SyncService(this._db); + SyncService(this._db, this._hashService); // public methods: @@ -33,6 +37,7 @@ class SyncService { Future syncUsersFromServer(List users) async { users.sortBy((u) => u.id); final dbUsers = await _db.users.where().sortById().findAll(); + assert(dbUsers.isSortedBy((u) => u.id), "dbUsers not sorted!"); final List toDelete = []; final List toUpsert = []; final changes = diffSortedListsSync( @@ -108,40 +113,16 @@ class SyncService { // private methods: /// Syncs a new asset to the db. Returns `true` if successful - Future _syncNewAssetToDb(Asset newAsset) async { - final List inDb = await _db.assets - .where() - .localIdDeviceIdEqualTo(newAsset.localId, newAsset.deviceId) - .findAll(); - Asset? match; - if (inDb.length == 1) { - // exactly one match: trivial case - match = inDb.first; - } else if (inDb.length > 1) { - // TODO instead of this heuristics: match by checksum once available - for (Asset a in inDb) { - if (a.ownerId == newAsset.ownerId && - a.fileModifiedAt.isAtSameMomentAs(newAsset.fileModifiedAt)) { - assert(match == null); - match = a; - } - } - if (match == null) { - for (Asset a in inDb) { - if (a.ownerId == newAsset.ownerId) { - assert(match == null); - match = a; - } - } - } - } - if (match != null) { + Future _syncNewAssetToDb(Asset a) async { + final Asset? inDb = + await _db.assets.getByChecksumOwnerId(a.checksum, a.ownerId); + if (inDb != null) { // unify local/remote assets by replacing the // local-only asset in the DB with a local&remote asset - newAsset = match.updatedCopy(newAsset); + a = inDb.updatedCopy(a); } try { - await _db.writeTxn(() => newAsset.put(_db)); + await _db.writeTxn(() => a.put(_db)); } on IsarError catch (e) { _log.severe("Failed to put new asset into db: $e"); return false; @@ -162,11 +143,11 @@ class SyncService { final List inDb = await _db.assets .filter() .ownerIdEqualTo(user.isarId) - .sortByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() + .sortByChecksum() .findAll(); - remote.sort(Asset.compareByOwnerDeviceLocalIdModified); + assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + + remote.sort(Asset.compareByChecksum); final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true); if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) { return false; @@ -199,6 +180,7 @@ class SyncService { query = baseQuery.owner((q) => q.isarIdEqualTo(me.isarId)); } final List dbAlbums = await query.sortByRemoteId().findAll(); + assert(dbAlbums.isSortedBy((e) => e.remoteId!), "dbAlbums not sorted!"); final List toDelete = []; final List existing = []; @@ -245,16 +227,16 @@ class SyncService { if (dto.assetCount != dto.assets.length) { return false; } - final assetsInDb = await album.assets - .filter() - .sortByOwnerId() - .thenByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() - .findAll(); + final assetsInDb = + await album.assets.filter().sortByOwnerId().thenByChecksum().findAll(); + assert(assetsInDb.isSorted(Asset.compareByOwnerChecksum), "inDb unsorted!"); final List assetsOnRemote = dto.getAssets(); - assetsOnRemote.sort(Asset.compareByOwnerDeviceLocalIdModified); - final (toAdd, toUpdate, toUnlink) = _diffAssets(assetsOnRemote, assetsInDb); + assetsOnRemote.sort(Asset.compareByOwnerChecksum); + final (toAdd, toUpdate, toUnlink) = _diffAssets( + assetsOnRemote, + assetsInDb, + compare: Asset.compareByOwnerChecksum, + ); // update shared users final List sharedUsers = album.sharedUsers.toList(growable: false); @@ -297,6 +279,7 @@ class SyncService { await album.assets.update(link: assetsToLink, unlink: toUnlink.cast()); await _db.albums.put(album); }); + _log.info("Synced changes of remote album ${album.name} to DB"); } on IsarError catch (e) { _log.severe("Failed to sync remote album to database $e"); } @@ -382,10 +365,11 @@ class SyncService { Set? excludedAssets, ]) async { onDevice.sort((a, b) => a.id.compareTo(b.id)); - final List inDb = + final inDb = await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll(); final List deleteCandidates = []; final List existing = []; + assert(inDb.isSorted((a, b) => a.localId!.compareTo(b.localId!)), "sort!"); final bool anyChanges = await diffSortedLists( onDevice, inDb, @@ -447,14 +431,15 @@ class SyncService { final inDb = await album.assets .filter() .ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId) - .deviceIdEqualTo(Store.get(StoreKey.deviceIdHash)) - .sortByLocalId() + .sortByChecksum() .findAll(); + assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + final int assetCountOnDevice = await ape.assetCountAsync; final List onDevice = - await ape.getAssets(excludedAssets: excludedAssets); - onDevice.sort(Asset.compareByLocalId); - final (toAdd, toUpdate, toDelete) = - _diffAssets(onDevice, inDb, compare: Asset.compareByLocalId); + await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + _removeDuplicates(onDevice); + // _removeDuplicates sorts `onDevice` by checksum + final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); if (toAdd.isEmpty && toUpdate.isEmpty && toDelete.isEmpty && @@ -491,6 +476,9 @@ class SyncService { await _db.albums.put(album); album.thumbnail.value ??= await album.assets.filter().findFirst(); await album.thumbnail.save(); + await _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, value: assetCountOnDevice.toString()), + ); }); _log.info("Synced changes of local album ${ape.name} to DB"); } on IsarError catch (e) { @@ -503,8 +491,13 @@ class SyncService { /// fast path for common case: only new assets were added to device album /// returns `true` if successfull, else `false` Future _syncDeviceAlbumFast(AssetPathEntity ape, Album album) async { + if (!(ape.lastModified ?? DateTime.now()).isAfter(album.modifiedAt)) { + return false; + } final int totalOnDevice = await ape.assetCountAsync; - final AssetPathEntity? modified = totalOnDevice > album.assetCount + final int lastKnownTotal = + (await _db.eTags.getById(ape.eTagKeyAssetCount))?.value?.toInt() ?? 0; + final AssetPathEntity? modified = totalOnDevice > lastKnownTotal ? await ape.fetchPathProperties( filterOptionGroup: FilterOptionGroup( updateTimeCond: DateTimeCond( @@ -517,17 +510,22 @@ class SyncService { if (modified == null) { return false; } - final List newAssets = await modified.getAssets(); - if (totalOnDevice != album.assets.length + newAssets.length) { + final List newAssets = await _hashService.getHashedAssets(modified); + + if (totalOnDevice != lastKnownTotal + newAssets.length) { return false; } album.modifiedAt = ape.lastModified ?? DateTime.now(); + _removeDuplicates(newAssets); final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); try { await _db.writeTxn(() async { await _db.assets.putAll(updated); await album.assets.update(link: existingInDb + updated); await _db.albums.put(album); + await _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, value: totalOnDevice.toString()), + ); }); _log.info("Fast synced local album ${ape.name} to DB"); } on IsarError catch (e) { @@ -547,7 +545,9 @@ class SyncService { ]) async { _log.info("Syncing a new local album to DB: ${ape.name}"); final Album a = Album.local(ape); - final assets = await ape.getAssets(excludedAssets: excludedAssets); + final assets = + await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + _removeDuplicates(assets); final (existingInDb, updated) = await _linkWithExistingFromDb(assets); _log.info( "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", @@ -570,44 +570,29 @@ class SyncService { Future<(List existing, List updated)> _linkWithExistingFromDb( List assets, ) async { - if (assets.isEmpty) { - return ([].cast(), [].cast()); - } - final List inDb = await _db.assets - .where() - .anyOf( - assets, - (q, Asset e) => q.localIdDeviceIdEqualTo(e.localId, e.deviceId), - ) - .sortByOwnerId() - .thenByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() - .findAll(); - assets.sort(Asset.compareByOwnerDeviceLocalIdModified); - final List existing = [], toUpsert = []; - diffSortedListsSync( - inDb, - assets, - // do not compare by modified date because for some assets dates differ on - // client and server, thus never reaching "both" case below - compare: Asset.compareByOwnerDeviceLocalId, - both: (Asset a, Asset b) { - if (a.canUpdate(b)) { - toUpsert.add(a.updatedCopy(b)); - return true; - } else { - existing.add(a); - return false; - } - }, - onlyFirst: (Asset a) => _log.finer( - "_linkWithExistingFromDb encountered asset only in DB: $a", - null, - StackTrace.current, - ), - onlySecond: (Asset b) => toUpsert.add(b), + if (assets.isEmpty) return ([].cast(), [].cast()); + + final List inDb = await _db.assets.getAllByChecksumOwnerId( + assets.map((a) => a.checksum).toList(growable: false), + assets.map((a) => a.ownerId).toInt64List(), ); + assert(inDb.length == assets.length); + final List existing = [], toUpsert = []; + for (int i = 0; i < assets.length; i++) { + final Asset? b = inDb[i]; + if (b == null) { + toUpsert.add(assets[i]); + continue; + } + if (b.canUpdate(assets[i])) { + final updated = b.updatedCopy(assets[i]); + assert(updated.id != Isar.autoIncrement); + toUpsert.add(updated); + } else { + existing.add(b); + } + } + assert(existing.length + toUpsert.length == assets.length); return (existing, toUpsert); } @@ -627,11 +612,63 @@ class SyncService { }); _log.info("Upserted ${assets.length} assets into the DB"); } on IsarError catch (e) { - _log.warning( + _log.severe( "Failed to upsert ${assets.length} assets into the DB: ${e.toString()}", ); + // give details on the errors + assets.sort(Asset.compareByOwnerChecksum); + final inDb = await _db.assets.getAllByChecksumOwnerId( + assets.map((e) => e.checksum).toList(growable: false), + assets.map((e) => e.ownerId).toInt64List(), + ); + for (int i = 0; i < assets.length; i++) { + final Asset a = assets[i]; + final Asset? b = inDb[i]; + if (b == null) { + if (a.id != Isar.autoIncrement) { + _log.warning( + "Trying to update an asset that does not exist in DB:\n$a", + ); + } + } else if (a.id != b.id) { + _log.warning( + "Trying to insert another asset with the same checksum+owner. In DB:\n$b\nTo insert:\n$a", + ); + } + } + for (int i = 1; i < assets.length; i++) { + if (Asset.compareByOwnerChecksum(assets[i - 1], assets[i]) == 0) { + _log.warning( + "Trying to insert duplicate assets:\n${assets[i - 1]}\n${assets[i]}", + ); + } + } } } + + List _removeDuplicates(List assets) { + final int before = assets.length; + assets.sort(Asset.compareByOwnerChecksumCreatedModified); + assets.uniqueConsecutive( + compare: Asset.compareByOwnerChecksum, + onDuplicate: (a, b) => + _log.info("Ignoring duplicate assets on device:\n$a\n$b"), + ); + final int duplicates = before - assets.length; + if (duplicates > 0) { + _log.warning("Ignored $duplicates duplicate assets on device"); + } + return assets; + } + + /// returns `true` if the albums differ on the surface + Future _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async { + return a.name != b.name || + a.lastModified == null || + !a.lastModified!.isAtSameMomentAs(b.modifiedAt) || + await a.assetCountAsync != + (await _db.eTags.getById(a.eTagKeyAssetCount))?.value?.toInt(); + } } /// Returns a triple(toAdd, toUpdate, toRemove) @@ -639,7 +676,7 @@ class SyncService { List assets, List inDb, { bool? remote, - int Function(Asset, Asset) compare = Asset.compareByOwnerDeviceLocalId, + int Function(Asset, Asset) compare = Asset.compareByChecksum, }) { final List toAdd = []; final List toUpdate = []; @@ -663,7 +700,7 @@ class SyncService { } } else if (remote == false && a.isRemote) { if (a.isLocal) { - a.isLocal = false; + a.localId = null; toUpdate.add(a); } } else { @@ -685,9 +722,9 @@ class SyncService { return const ([], []); } deleteCandidates.sort(Asset.compareById); - deleteCandidates.uniqueConsecutive((a) => a.id); + deleteCandidates.uniqueConsecutive(compare: Asset.compareById); existing.sort(Asset.compareById); - existing.uniqueConsecutive((a) => a.id); + existing.uniqueConsecutive(compare: Asset.compareById); final (tooAdd, toUpdate, toRemove) = _diffAssets( existing, deleteCandidates, @@ -698,14 +735,6 @@ class SyncService { return (toRemove.map((e) => e.id).toList(), toUpdate); } -/// returns `true` if the albums differ on the surface -Future _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async { - return a.name != b.name || - a.lastModified == null || - !a.lastModified!.isAtSameMomentAs(b.modifiedAt) || - await a.assetCountAsync != b.assetCount; -} - /// returns `true` if the albums differ on the surface bool _hasAlbumResponseDtoChanged(AlbumResponseDto dto, Album a) { return dto.assetCount != a.assetCount || diff --git a/mobile/lib/shared/views/tab_controller_page.dart b/mobile/lib/shared/views/tab_controller_page.dart index 4d245d034..e14f53514 100644 --- a/mobile/lib/shared/views/tab_controller_page.dart +++ b/mobile/lib/shared/views/tab_controller_page.dart @@ -6,12 +6,39 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/modules/asset_viewer/providers/scroll_notifier.provider.dart'; import 'package:immich_mobile/modules/home/providers/multiselect.provider.dart'; import 'package:immich_mobile/routing/router.dart'; +import 'package:immich_mobile/shared/providers/asset.provider.dart'; -class TabControllerPage extends ConsumerWidget { +class TabControllerPage extends HookConsumerWidget { const TabControllerPage({Key? key}) : super(key: key); @override Widget build(BuildContext context, WidgetRef ref) { + final refreshing = ref.watch(assetProvider); + + Widget buildIcon(Widget icon) { + if (!refreshing) return icon; + return Stack( + alignment: Alignment.center, + clipBehavior: Clip.none, + children: [ + icon, + Positioned( + right: -14, + child: SizedBox( + height: 12, + width: 12, + child: CircularProgressIndicator( + strokeWidth: 2, + valueColor: AlwaysStoppedAnimation( + Theme.of(context).primaryColor, + ), + ), + ), + ), + ], + ); + } + navigationRail(TabsRouter tabsRouter) { return NavigationRail( labelType: NavigationRailLabelType.all, @@ -83,9 +110,12 @@ class TabControllerPage extends ConsumerWidget { icon: const Icon( Icons.photo_library_outlined, ), - selectedIcon: Icon( - Icons.photo_library, - color: Theme.of(context).primaryColor, + selectedIcon: buildIcon( + Icon( + size: 24, + Icons.photo_library, + color: Theme.of(context).primaryColor, + ), ), ), NavigationDestination( @@ -113,9 +143,11 @@ class TabControllerPage extends ConsumerWidget { icon: const Icon( Icons.photo_album_outlined, ), - selectedIcon: Icon( - Icons.photo_album_rounded, - color: Theme.of(context).primaryColor, + selectedIcon: buildIcon( + Icon( + Icons.photo_album_rounded, + color: Theme.of(context).primaryColor, + ), ), ) ], diff --git a/mobile/lib/utils/builtin_extensions.dart b/mobile/lib/utils/builtin_extensions.dart index 3a3a723dc..5b769f26f 100644 --- a/mobile/lib/utils/builtin_extensions.dart +++ b/mobile/lib/utils/builtin_extensions.dart @@ -1,3 +1,5 @@ +import 'dart:typed_data'; + import 'package:collection/collection.dart'; extension DurationExtension on String { @@ -22,15 +24,20 @@ extension DurationExtension on String { } extension ListExtension on List { - List uniqueConsecutive([T Function(E element)? key]) { - key ??= (E e) => e as T; + List uniqueConsecutive({ + int Function(E a, E b)? compare, + void Function(E a, E b)? onDuplicate, + }) { + compare ??= (E a, E b) => a == b ? 0 : 1; int i = 1, j = 1; for (; i < length; i++) { - if (key(this[i]) != key(this[i - 1])) { + if (compare(this[i - 1], this[i]) != 0) { if (i != j) { this[j] = this[i]; } j++; + } else if (onDuplicate != null) { + onDuplicate(this[i - 1], this[i]); } } length = length == 0 ? 0 : j; @@ -45,3 +52,11 @@ extension ListExtension on List { return ListSlice(this, start, end); } } + +extension IntListExtension on Iterable { + Int64List toInt64List() { + final list = Int64List(length); + list.setAll(0, this); + return list; + } +} diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 56e9ba8df..724f3a872 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -8,11 +8,13 @@ Future migrateDatabaseIfNeeded(Isar db) async { final int version = Store.get(StoreKey.version, 1); switch (version) { case 1: - await _migrateV1ToV2(db); + await _migrateTo(db, 2); + case 2: + await _migrateTo(db, 3); } } -Future _migrateV1ToV2(Isar db) async { +Future _migrateTo(Isar db, int version) async { await clearAssetsAndAlbums(db); - await Store.put(StoreKey.version, 2); + await Store.put(StoreKey.version, version); } diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 736b7a4f5..71585f92e 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -242,13 +242,13 @@ packages: source: hosted version: "0.3.3+4" crypto: - dependency: transitive + dependency: "direct main" description: name: crypto - sha256: aa274aa7774f8964e4f4f38cc994db7b6158dd36e9187aaceaddc994b35c6c67 + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab url: "https://pub.dev" source: hosted - version: "3.0.2" + version: "3.0.3" csslib: dependency: transitive description: @@ -333,10 +333,10 @@ packages: dependency: transitive description: name: ffi - sha256: a38574032c5f1dd06c4aee541789906c12ccaab8ba01446e800d9c5b79c4a978 + sha256: ed5337a5660c506388a9f012be0288fb38b49020ce2b45fe1f8b8323fe429f99 url: "https://pub.dev" source: hosted - version: "2.0.1" + version: "2.0.2" file: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 8dfc06389..0a5aa1350 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -45,6 +45,7 @@ dependencies: isar_flutter_libs: *isar_version # contains Isar Core permission_handler: ^10.2.0 device_info_plus: ^8.1.0 + crypto: ^3.0.3 # TODO remove once native crypto is used on iOS openapi: path: openapi diff --git a/mobile/test/asset_grid_data_structure_test.dart b/mobile/test/asset_grid_data_structure_test.dart index 78c553f5a..6522bec3d 100644 --- a/mobile/test/asset_grid_data_structure_test.dart +++ b/mobile/test/asset_grid_data_structure_test.dart @@ -13,8 +13,8 @@ void main() { testAssets.add( Asset( + checksum: "", localId: '$i', - deviceId: 1, ownerId: 1, fileCreatedAt: date, fileModifiedAt: date, @@ -23,7 +23,6 @@ void main() { type: AssetType.image, fileName: '', isFavorite: false, - isLocal: false, isArchived: false, ), ); diff --git a/mobile/test/builtin_extensions_text.dart b/mobile/test/builtin_extensions_test.dart similarity index 88% rename from mobile/test/builtin_extensions_text.dart rename to mobile/test/builtin_extensions_test.dart index 9e4924e44..875a20fb0 100644 --- a/mobile/test/builtin_extensions_text.dart +++ b/mobile/test/builtin_extensions_test.dart @@ -43,7 +43,12 @@ void main() { test('withKey', () { final a = ["a", "bb", "cc", "ddd"]; - expect(a.uniqueConsecutive((s) => s.length), ["a", "bb", "ddd"]); + expect( + a.uniqueConsecutive( + compare: (s1, s2) => s1.length.compareTo(s2.length), + ), + ["a", "bb", "ddd"], + ); }); }); } diff --git a/mobile/test/sync_service_test.dart b/mobile/test/sync_service_test.dart index 0d1045c72..63b5614bd 100644 --- a/mobile/test/sync_service_test.dart +++ b/mobile/test/sync_service_test.dart @@ -6,32 +6,33 @@ import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/logger_message.model.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; +import 'package:immich_mobile/shared/services/hash.service.dart'; import 'package:immich_mobile/shared/services/immich_logger.service.dart'; import 'package:immich_mobile/shared/services/sync.service.dart'; import 'package:isar/isar.dart'; +import 'package:mockito/mockito.dart'; void main() { Asset makeAsset({ - required String localId, + required String checksum, + String? localId, String? remoteId, int deviceId = 1, int ownerId = 590700560494856554, // hash of "1" - bool isLocal = false, }) { final DateTime date = DateTime(2000); return Asset( + checksum: checksum, localId: localId, remoteId: remoteId, - deviceId: deviceId, ownerId: ownerId, fileCreatedAt: date, fileModifiedAt: date, updatedAt: date, durationInSeconds: 0, type: AssetType.image, - fileName: localId, + fileName: localId ?? remoteId ?? "", isFavorite: false, - isLocal: isLocal, isArchived: false, ); } @@ -53,6 +54,7 @@ void main() { group('Test SyncService grouped', () { late final Isar db; + final MockHashService hs = MockHashService(); final owner = User( id: "1", updatedAt: DateTime.now(), @@ -71,11 +73,11 @@ void main() { await Store.put(StoreKey.currentUser, owner); }); final List initialAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1", isLocal: true), - makeAsset(localId: "2", isLocal: true), - makeAsset(localId: "3", isLocal: true), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", localId: "1", remoteId: "1-1"), + makeAsset(checksum: "d", localId: "2"), + makeAsset(checksum: "e", localId: "3"), ]; setUp(() { db.writeTxnSync(() { @@ -84,11 +86,11 @@ void main() { }); }); test('test inserting existing assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1"), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", remoteId: "1-1"), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -97,14 +99,14 @@ void main() { }); test('test inserting new assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1"), - makeAsset(localId: "2", remoteId: "1-2"), - makeAsset(localId: "4", remoteId: "1-4"), - makeAsset(localId: "1", remoteId: "3-1", deviceId: 3), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", remoteId: "1-1"), + makeAsset(checksum: "d", remoteId: "1-2"), + makeAsset(checksum: "f", remoteId: "1-4"), + makeAsset(checksum: "g", remoteId: "3-1", deviceId: 3), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -113,14 +115,14 @@ void main() { }); test('test syncing duplicate assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "1-1"), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1b", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1c", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1d", deviceId: 2), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "1-1"), + makeAsset(checksum: "c", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "h", remoteId: "2-1b", deviceId: 2), + makeAsset(checksum: "i", remoteId: "2-1c", deviceId: 2), + makeAsset(checksum: "j", remoteId: "2-1d", deviceId: 2), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -133,11 +135,13 @@ void main() { final bool c3 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); expect(c3, true); expect(db.assets.countSync(), 7); - remoteAssets.add(makeAsset(localId: "1", remoteId: "2-1e", deviceId: 2)); - remoteAssets.add(makeAsset(localId: "2", remoteId: "2-2", deviceId: 2)); + remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e", deviceId: 2)); + remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2", deviceId: 2)); final bool c4 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); expect(c4, true); expect(db.assets.countSync(), 9); }); }); } + +class MockHashService extends Mock implements HashService {}