diff --git a/mobile/android/app/build.gradle b/mobile/android/app/build.gradle index 158a27208..38ef303b6 100644 --- a/mobile/android/app/build.gradle +++ b/mobile/android/app/build.gradle @@ -84,6 +84,7 @@ flutter { dependencies { implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk8:$kotlin_version" + implementation "org.jetbrains.kotlinx:kotlinx-coroutines-android:$kotlin_coroutines_version" implementation "androidx.work:work-runtime-ktx:$work_version" implementation "androidx.concurrent:concurrent-futures:$concurrent_version" implementation "com.google.guava:guava:$guava_version" diff --git a/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt b/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt index 1e9b2502d..6541ad575 100644 --- a/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt +++ b/mobile/android/app/src/main/kotlin/com/example/mobile/BackgroundServicePlugin.kt @@ -1,10 +1,15 @@ package app.alextran.immich import android.content.Context +import android.util.Log import io.flutter.embedding.engine.plugins.FlutterPlugin import io.flutter.plugin.common.BinaryMessenger import io.flutter.plugin.common.MethodCall import io.flutter.plugin.common.MethodChannel +import java.security.MessageDigest +import java.io.File +import java.io.FileInputStream +import kotlinx.coroutines.* /** * Android plugin for Dart `BackgroundService` @@ -16,6 +21,7 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler { private var methodChannel: MethodChannel? = null private var context: Context? = null + private val sha1: MessageDigest = MessageDigest.getInstance("SHA-1") override fun onAttachedToEngine(binding: FlutterPlugin.FlutterPluginBinding) { onAttachedToEngine(binding.applicationContext, binding.binaryMessenger) @@ -70,9 +76,40 @@ class BackgroundServicePlugin : FlutterPlugin, MethodChannel.MethodCallHandler { "isIgnoringBatteryOptimizations" -> { result.success(BackupWorker.isIgnoringBatteryOptimizations(ctx)) } + "digestFiles" -> { + val args = call.arguments>()!! + GlobalScope.launch(Dispatchers.IO) { + val buf = ByteArray(BUFSIZE) + val digest: MessageDigest = MessageDigest.getInstance("SHA-1") + val hashes = arrayOfNulls(args.size) + for (i in args.indices) { + val path = args[i] + var len = 0 + try { + val file = FileInputStream(path) + try { + while (true) { + len = file.read(buf) + if (len != BUFSIZE) break + digest.update(buf) + } + } finally { + file.close() + } + digest.update(buf, 0, len) + hashes[i] = digest.digest() + } catch (e: Exception) { + // skip this file + Log.w(TAG, "Failed to hash file ${args[i]}: $e") + } + } + result.success(hashes.asList()) + } + } else -> result.notImplemented() } } } -private const val TAG = "BackgroundServicePlugin" \ No newline at end of file +private const val TAG = "BackgroundServicePlugin" +private const val BUFSIZE = 2*1024*1024; diff --git a/mobile/android/build.gradle b/mobile/android/build.gradle index ee01aa0f1..5d8f2bf6e 100644 --- a/mobile/android/build.gradle +++ b/mobile/android/build.gradle @@ -1,5 +1,6 @@ buildscript { ext.kotlin_version = '1.8.20' + ext.kotlin_coroutines_version = '1.7.1' ext.work_version = '2.7.1' ext.concurrent_version = '1.1.0' ext.guava_version = '31.0.1-android' diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 3c8cedc9d..1dfc630d4 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -19,9 +19,11 @@ import 'package:immich_mobile/modules/settings/providers/notification_permission import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/routing/tab_navigation_observer.dart'; import 'package:immich_mobile/shared/models/album.dart'; +import 'package:immich_mobile/shared/models/android_device_asset.dart'; import 'package:immich_mobile/shared/models/asset.dart'; import 'package:immich_mobile/shared/models/etag.dart'; import 'package:immich_mobile/shared/models/exif_info.dart'; +import 'package:immich_mobile/shared/models/ios_device_asset.dart'; import 'package:immich_mobile/shared/models/logger_message.model.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; @@ -91,6 +93,7 @@ Future loadDb() async { DuplicatedAssetSchema, LoggerMessageSchema, ETagSchema, + Platform.isAndroid ? AndroidDeviceAssetSchema : IOSDeviceAssetSchema, ], directory: dir.path, maxSizeMiB: 256, diff --git a/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart b/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart index ffb2e980e..b95a31c13 100644 --- a/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart +++ b/mobile/lib/modules/asset_viewer/providers/image_viewer_page_state.provider.dart @@ -2,6 +2,7 @@ import 'package:easy_localization/easy_localization.dart'; import 'package:flutter/material.dart'; import 'package:fluttertoast/fluttertoast.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/modules/album/services/album.service.dart'; import 'package:immich_mobile/modules/asset_viewer/models/image_viewer_page_state.model.dart'; import 'package:immich_mobile/modules/asset_viewer/services/image_viewer.service.dart'; import 'package:immich_mobile/shared/models/asset.dart'; @@ -12,9 +13,13 @@ import 'package:immich_mobile/shared/ui/share_dialog.dart'; class ImageViewerStateNotifier extends StateNotifier { final ImageViewerService _imageViewerService; final ShareService _shareService; + final AlbumService _albumService; - ImageViewerStateNotifier(this._imageViewerService, this._shareService) - : super( + ImageViewerStateNotifier( + this._imageViewerService, + this._shareService, + this._albumService, + ) : super( ImageViewerPageState( downloadAssetStatus: DownloadAssetStatus.idle, ), @@ -34,6 +39,7 @@ class ImageViewerStateNotifier extends StateNotifier { toastType: ToastType.success, gravity: ToastGravity.BOTTOM, ); + _albumService.refreshDeviceAlbums(); } else { state = state.copyWith(downloadAssetStatus: DownloadAssetStatus.error); ImmichToast.show( @@ -66,5 +72,6 @@ final imageViewerStateProvider = ((ref) => ImageViewerStateNotifier( ref.watch(imageViewerServiceProvider), ref.watch(shareServiceProvider), + ref.watch(albumServiceProvider), )), ); diff --git a/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart b/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart index b79f240bb..ddd1b40a8 100644 --- a/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart +++ b/mobile/lib/modules/asset_viewer/ui/top_control_app_bar.dart @@ -72,15 +72,7 @@ class TopControlAppBar extends HookConsumerWidget { color: Colors.grey[200], ), ), - if (!asset.isLocal) - IconButton( - onPressed: onDownloadPressed, - icon: Icon( - Icons.cloud_download_outlined, - color: Colors.grey[200], - ), - ), - if (asset.storage == AssetState.merged) + if (asset.storage == AssetState.remote) IconButton( onPressed: onDownloadPressed, icon: Icon( diff --git a/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart b/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart index 29e932b35..f67fdf576 100644 --- a/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart +++ b/mobile/lib/modules/asset_viewer/views/gallery_viewer.dart @@ -287,7 +287,7 @@ class GalleryViewerPage extends HookConsumerWidget { isFavorite: asset().isFavorite, onMoreInfoPressed: showInfo, onFavorite: asset().isRemote ? () => toggleFavorite(asset()) : null, - onDownloadPressed: asset().storage == AssetState.local + onDownloadPressed: asset().isLocal ? null : () => ref.watch(imageViewerStateProvider.notifier).downloadAsset( diff --git a/mobile/lib/modules/backup/background_service/background.service.dart b/mobile/lib/modules/backup/background_service/background.service.dart index 319c2890f..5817e1d2b 100644 --- a/mobile/lib/modules/backup/background_service/background.service.dart +++ b/mobile/lib/modules/backup/background_service/background.service.dart @@ -132,6 +132,17 @@ class BackgroundService { } } + Future digestFile(String path) { + return _foregroundChannel.invokeMethod("digestFile", [path]); + } + + Future?> digestFiles(List paths) { + return _foregroundChannel.invokeListMethod( + "digestFiles", + paths, + ); + } + /// Updates the notification shown by the background service Future _updateNotification({ String? title, diff --git a/mobile/lib/modules/home/views/home_page.dart b/mobile/lib/modules/home/views/home_page.dart index 957b15133..292389c05 100644 --- a/mobile/lib/modules/home/views/home_page.dart +++ b/mobile/lib/modules/home/views/home_page.dart @@ -47,11 +47,11 @@ class HomePage extends HookConsumerWidget { useEffect( () { - ref.watch(websocketProvider.notifier).connect(); - ref.watch(assetProvider.notifier).getAllAsset(); - ref.watch(albumProvider.notifier).getAllAlbums(); - ref.watch(sharedAlbumProvider.notifier).getAllSharedAlbums(); - ref.watch(serverInfoProvider.notifier).getServerVersion(); + ref.read(websocketProvider.notifier).connect(); + Future(() => ref.read(assetProvider.notifier).getAllAsset()); + ref.read(albumProvider.notifier).getAllAlbums(); + ref.read(sharedAlbumProvider.notifier).getAllSharedAlbums(); + ref.read(serverInfoProvider.notifier).getServerVersion(); selectionEnabledHook.addListener(() { multiselectEnabled.state = selectionEnabledHook.value; @@ -144,7 +144,7 @@ class HomePage extends HookConsumerWidget { ); if (remoteAssets.isNotEmpty) { await ref - .watch(assetProvider.notifier) + .read(assetProvider.notifier) .toggleArchive(remoteAssets, true); final assetOrAssets = remoteAssets.length > 1 ? 'assets' : 'asset'; @@ -163,7 +163,7 @@ class HomePage extends HookConsumerWidget { void onDelete() async { processing.value = true; try { - await ref.watch(assetProvider.notifier).deleteAssets(selection.value); + await ref.read(assetProvider.notifier).deleteAssets(selection.value); selectionEnabledHook.value = false; } finally { processing.value = false; diff --git a/mobile/lib/shared/models/album.dart b/mobile/lib/shared/models/album.dart index a3cffa169..81a834b99 100644 --- a/mobile/lib/shared/models/album.dart +++ b/mobile/lib/shared/models/album.dart @@ -166,23 +166,10 @@ extension AssetsHelper on IsarCollection { } } -extension AssetPathEntityHelper on AssetPathEntity { - Future> getAssets({ - int start = 0, - int end = 0x7fffffffffffffff, - Set? excludedAssets, - }) async { - final assetEntities = await getAssetListRange(start: start, end: end); - if (excludedAssets != null) { - return assetEntities - .where((e) => !excludedAssets.contains(e.id)) - .map(Asset.local) - .toList(); - } - return assetEntities.map(Asset.local).toList(); - } -} - extension AlbumResponseDtoHelper on AlbumResponseDto { List getAssets() => assets.map(Asset.remote).toList(); } + +extension AssetPathEntityHelper on AssetPathEntity { + String get eTagKeyAssetCount => "device-album-$id-asset-count"; +} diff --git a/mobile/lib/shared/models/android_device_asset.dart b/mobile/lib/shared/models/android_device_asset.dart new file mode 100644 index 000000000..b6b2663fd --- /dev/null +++ b/mobile/lib/shared/models/android_device_asset.dart @@ -0,0 +1,10 @@ +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:isar/isar.dart'; + +part 'android_device_asset.g.dart'; + +@Collection() +class AndroidDeviceAsset extends DeviceAsset { + AndroidDeviceAsset({required this.id, required super.hash}); + Id id; +} diff --git a/mobile/lib/shared/models/android_device_asset.g.dart b/mobile/lib/shared/models/android_device_asset.g.dart new file mode 100644 index 000000000..ca7c822ba Binary files /dev/null and b/mobile/lib/shared/models/android_device_asset.g.dart differ diff --git a/mobile/lib/shared/models/asset.dart b/mobile/lib/shared/models/asset.dart index 840653554..701b6c2dd 100644 --- a/mobile/lib/shared/models/asset.dart +++ b/mobile/lib/shared/models/asset.dart @@ -1,3 +1,5 @@ +import 'dart:convert'; + import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/utils/hash.dart'; @@ -14,7 +16,7 @@ part 'asset.g.dart'; class Asset { Asset.remote(AssetResponseDto remote) : remoteId = remote.id, - isLocal = false, + checksum = remote.checksum, fileCreatedAt = remote.fileCreatedAt, fileModifiedAt = remote.fileModifiedAt, updatedAt = remote.updatedAt, @@ -24,23 +26,20 @@ class Asset { height = remote.exifInfo?.exifImageHeight?.toInt(), width = remote.exifInfo?.exifImageWidth?.toInt(), livePhotoVideoId = remote.livePhotoVideoId, - localId = remote.deviceAssetId, - deviceId = fastHash(remote.deviceId), ownerId = fastHash(remote.ownerId), exifInfo = remote.exifInfo != null ? ExifInfo.fromDto(remote.exifInfo!) : null, isFavorite = remote.isFavorite, isArchived = remote.isArchived; - Asset.local(AssetEntity local) + Asset.local(AssetEntity local, List hash) : localId = local.id, - isLocal = true, + checksum = base64.encode(hash), durationInSeconds = local.duration, type = AssetType.values[local.typeInt], height = local.height, width = local.width, fileName = local.title!, - deviceId = Store.get(StoreKey.deviceIdHash), ownerId = Store.get(StoreKey.currentUser).isarId, fileModifiedAt = local.modifiedDateTime, updatedAt = local.modifiedDateTime, @@ -53,13 +52,15 @@ class Asset { if (local.latitude != null) { exifInfo = ExifInfo(lat: local.latitude, long: local.longitude); } + _local = local; + assert(hash.length == 20, "invalid SHA1 hash"); } Asset({ this.id = Isar.autoIncrement, + required this.checksum, this.remoteId, required this.localId, - required this.deviceId, required this.ownerId, required this.fileCreatedAt, required this.fileModifiedAt, @@ -72,7 +73,6 @@ class Asset { this.livePhotoVideoId, this.exifInfo, required this.isFavorite, - required this.isLocal, required this.isArchived, }); @@ -83,7 +83,7 @@ class Asset { AssetEntity? get local { if (isLocal && _local == null) { _local = AssetEntity( - id: localId, + id: localId!, typeInt: isImage ? 1 : 2, width: width ?? 0, height: height ?? 0, @@ -98,18 +98,21 @@ class Asset { Id id = Isar.autoIncrement; + /// stores the raw SHA1 bytes as a base64 String + /// because Isar cannot sort lists of byte arrays + @Index( + unique: true, + replace: false, + type: IndexType.hash, + composite: [CompositeIndex("ownerId")], + ) + String checksum; + @Index(unique: false, replace: false, type: IndexType.hash) String? remoteId; - @Index( - unique: false, - replace: false, - type: IndexType.hash, - composite: [CompositeIndex('deviceId')], - ) - String localId; - - int deviceId; + @Index(unique: false, replace: false, type: IndexType.hash) + String? localId; int ownerId; @@ -134,14 +137,15 @@ class Asset { bool isFavorite; - /// `true` if this [Asset] is present on the device - bool isLocal; - bool isArchived; @ignore ExifInfo? exifInfo; + /// `true` if this [Asset] is present on the device + @ignore + bool get isLocal => localId != null; + @ignore bool get isInDb => id != Isar.autoIncrement; @@ -175,9 +179,9 @@ class Asset { bool operator ==(other) { if (other is! Asset) return false; return id == other.id && + checksum == other.checksum && remoteId == other.remoteId && localId == other.localId && - deviceId == other.deviceId && ownerId == other.ownerId && fileCreatedAt.isAtSameMomentAs(other.fileCreatedAt) && fileModifiedAt.isAtSameMomentAs(other.fileModifiedAt) && @@ -197,9 +201,9 @@ class Asset { @ignore int get hashCode => id.hashCode ^ + checksum.hashCode ^ remoteId.hashCode ^ localId.hashCode ^ - deviceId.hashCode ^ ownerId.hashCode ^ fileCreatedAt.hashCode ^ fileModifiedAt.hashCode ^ @@ -217,8 +221,7 @@ class Asset { /// Returns `true` if this [Asset] can updated with values from parameter [a] bool canUpdate(Asset a) { assert(isInDb); - assert(localId == a.localId); - assert(deviceId == a.deviceId); + assert(checksum == a.checksum); assert(a.storage != AssetState.merged); return a.updatedAt.isAfter(updatedAt) || a.isRemote && !isRemote || @@ -239,11 +242,18 @@ class Asset { if (a.isRemote) { return a._copyWith( id: id, - isLocal: isLocal, + localId: localId, width: a.width ?? width, height: a.height ?? height, exifInfo: a.exifInfo?.copyWith(id: id) ?? exifInfo, ); + } else if (isRemote) { + return _copyWith( + localId: localId ?? a.localId, + width: width ?? a.width, + height: height ?? a.height, + exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id), + ); } else { return a._copyWith( id: id, @@ -270,7 +280,7 @@ class Asset { } else { // add only missing values (and set isLocal to true) return _copyWith( - isLocal: true, + localId: localId ?? a.localId, width: width ?? a.width, height: height ?? a.height, exifInfo: exifInfo ?? a.exifInfo?.copyWith(id: id), @@ -281,9 +291,9 @@ class Asset { Asset _copyWith({ Id? id, + String? checksum, String? remoteId, String? localId, - int? deviceId, int? ownerId, DateTime? fileCreatedAt, DateTime? fileModifiedAt, @@ -295,15 +305,14 @@ class Asset { String? fileName, String? livePhotoVideoId, bool? isFavorite, - bool? isLocal, bool? isArchived, ExifInfo? exifInfo, }) => Asset( id: id ?? this.id, + checksum: checksum ?? this.checksum, remoteId: remoteId ?? this.remoteId, localId: localId ?? this.localId, - deviceId: deviceId ?? this.deviceId, ownerId: ownerId ?? this.ownerId, fileCreatedAt: fileCreatedAt ?? this.fileCreatedAt, fileModifiedAt: fileModifiedAt ?? this.fileModifiedAt, @@ -315,7 +324,6 @@ class Asset { fileName: fileName ?? this.fileName, livePhotoVideoId: livePhotoVideoId ?? this.livePhotoVideoId, isFavorite: isFavorite ?? this.isFavorite, - isLocal: isLocal ?? this.isLocal, isArchived: isArchived ?? this.isArchived, exifInfo: exifInfo ?? this.exifInfo, ); @@ -328,39 +336,36 @@ class Asset { } } - /// compares assets by [ownerId], [deviceId], [localId] - static int compareByOwnerDeviceLocalId(Asset a, Asset b) { - final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); - if (ownerIdOrder != 0) { - return ownerIdOrder; - } - final int deviceIdOrder = a.deviceId.compareTo(b.deviceId); - if (deviceIdOrder != 0) { - return deviceIdOrder; - } - final int localIdOrder = a.localId.compareTo(b.localId); - return localIdOrder; - } - - /// compares assets by [ownerId], [deviceId], [localId], [fileModifiedAt] - static int compareByOwnerDeviceLocalIdModified(Asset a, Asset b) { - final int order = compareByOwnerDeviceLocalId(a, b); - return order != 0 ? order : a.fileModifiedAt.compareTo(b.fileModifiedAt); - } - static int compareById(Asset a, Asset b) => a.id.compareTo(b.id); - static int compareByLocalId(Asset a, Asset b) => - a.localId.compareTo(b.localId); + static int compareByChecksum(Asset a, Asset b) => + a.checksum.compareTo(b.checksum); + + static int compareByOwnerChecksum(Asset a, Asset b) { + final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); + if (ownerIdOrder != 0) return ownerIdOrder; + return compareByChecksum(a, b); + } + + static int compareByOwnerChecksumCreatedModified(Asset a, Asset b) { + final int ownerIdOrder = a.ownerId.compareTo(b.ownerId); + if (ownerIdOrder != 0) return ownerIdOrder; + final int checksumOrder = compareByChecksum(a, b); + if (checksumOrder != 0) return checksumOrder; + final int createdOrder = a.fileCreatedAt.compareTo(b.fileCreatedAt); + if (createdOrder != 0) return createdOrder; + return a.fileModifiedAt.compareTo(b.fileModifiedAt); + } @override String toString() { return """ { + "id": ${id == Isar.autoIncrement ? '"N/A"' : id}, "remoteId": "${remoteId ?? "N/A"}", - "localId": "$localId", - "deviceId": "$deviceId", - "ownerId": "$ownerId", + "localId": "${localId ?? "N/A"}", + "checksum": "$checksum", + "ownerId": $ownerId, "livePhotoVideoId": "${livePhotoVideoId ?? "N/A"}", "fileCreatedAt": "$fileCreatedAt", "fileModifiedAt": "$fileModifiedAt", @@ -369,9 +374,8 @@ class Asset { "type": "$type", "fileName": "$fileName", "isFavorite": $isFavorite, - "isLocal": $isLocal, "isRemote: $isRemote, - "storage": $storage, + "storage": "$storage", "width": ${width ?? "N/A"}, "height": ${height ?? "N/A"}, "isArchived": $isArchived @@ -424,10 +428,6 @@ extension AssetsHelper on IsarCollection { QueryBuilder _remote(Iterable ids) => where().anyOf(ids, (q, String e) => q.remoteIdEqualTo(e)); QueryBuilder _local(Iterable ids) { - return where().anyOf( - ids, - (q, String e) => - q.localIdDeviceIdEqualTo(e, Store.get(StoreKey.deviceIdHash)), - ); + return where().anyOf(ids, (q, String e) => q.localIdEqualTo(e)); } } diff --git a/mobile/lib/shared/models/asset.g.dart b/mobile/lib/shared/models/asset.g.dart index e7085bad4..713c26885 100644 Binary files a/mobile/lib/shared/models/asset.g.dart and b/mobile/lib/shared/models/asset.g.dart differ diff --git a/mobile/lib/shared/models/device_asset.dart b/mobile/lib/shared/models/device_asset.dart new file mode 100644 index 000000000..0973dd4ff --- /dev/null +++ b/mobile/lib/shared/models/device_asset.dart @@ -0,0 +1,8 @@ +import 'package:isar/isar.dart'; + +class DeviceAsset { + DeviceAsset({required this.hash}); + + @Index(unique: false, type: IndexType.hash) + List hash; +} diff --git a/mobile/lib/shared/models/ios_device_asset.dart b/mobile/lib/shared/models/ios_device_asset.dart new file mode 100644 index 000000000..0c55c74eb --- /dev/null +++ b/mobile/lib/shared/models/ios_device_asset.dart @@ -0,0 +1,14 @@ +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:immich_mobile/utils/hash.dart'; +import 'package:isar/isar.dart'; + +part 'ios_device_asset.g.dart'; + +@Collection() +class IOSDeviceAsset extends DeviceAsset { + IOSDeviceAsset({required this.id, required super.hash}); + + @Index(replace: true, unique: true, type: IndexType.hash) + String id; + Id get isarId => fastHash(id); +} diff --git a/mobile/lib/shared/models/ios_device_asset.g.dart b/mobile/lib/shared/models/ios_device_asset.g.dart new file mode 100644 index 000000000..f10c3decd Binary files /dev/null and b/mobile/lib/shared/models/ios_device_asset.g.dart differ diff --git a/mobile/lib/shared/providers/asset.provider.dart b/mobile/lib/shared/providers/asset.provider.dart index c1384f402..54e969426 100644 --- a/mobile/lib/shared/providers/asset.provider.dart +++ b/mobile/lib/shared/providers/asset.provider.dart @@ -18,11 +18,7 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; import 'package:photo_manager/photo_manager.dart'; -/// State does not contain archived assets. -/// Use database provider if you want to access the isArchived assets -class AssetsState {} - -class AssetNotifier extends StateNotifier { +class AssetNotifier extends StateNotifier { final AssetService _assetService; final AlbumService _albumService; final UserService _userService; @@ -38,7 +34,7 @@ class AssetNotifier extends StateNotifier { this._userService, this._syncService, this._db, - ) : super(AssetsState()); + ) : super(false); Future getAllAsset({bool clear = false}) async { if (_getAllAssetInProgress || _deleteInProgress) { @@ -48,14 +44,15 @@ class AssetNotifier extends StateNotifier { final stopwatch = Stopwatch()..start(); try { _getAllAssetInProgress = true; + state = true; if (clear) { await clearAssetsAndAlbums(_db); log.info("Manual refresh requested, cleared assets and albums from db"); } + await _userService.refreshUsers(); final bool newRemote = await _assetService.refreshRemoteAssets(); final bool newLocal = await _albumService.refreshDeviceAlbums(); debugPrint("newRemote: $newRemote, newLocal: $newLocal"); - await _userService.refreshUsers(); final List partners = await _db.users.filter().isPartnerSharedWithEqualTo(true).findAll(); for (User u in partners) { @@ -64,6 +61,7 @@ class AssetNotifier extends StateNotifier { log.info("Load assets: ${stopwatch.elapsedMilliseconds}ms"); } finally { _getAllAssetInProgress = false; + state = false; } } @@ -79,6 +77,7 @@ class AssetNotifier extends StateNotifier { Future deleteAssets(Set deleteAssets) async { _deleteInProgress = true; + state = true; try { final localDeleted = await _deleteLocalAssets(deleteAssets); final remoteDeleted = await _deleteRemoteAssets(deleteAssets); @@ -91,24 +90,14 @@ class AssetNotifier extends StateNotifier { } } finally { _deleteInProgress = false; + state = false; } } Future> _deleteLocalAssets(Set assetsToDelete) async { - final int deviceId = Store.get(StoreKey.deviceIdHash); - final List local = []; + final List local = + assetsToDelete.where((a) => a.isLocal).map((a) => a.localId!).toList(); // Delete asset from device - for (final Asset asset in assetsToDelete) { - if (asset.isLocal) { - local.add(asset.localId); - } else if (asset.deviceId == deviceId) { - // Delete asset on device if it is still present - var localAsset = await AssetEntity.fromId(asset.localId); - if (localAsset != null) { - local.add(localAsset.id); - } - } - } if (local.isNotEmpty) { try { return await PhotoManager.editor.deleteWithIds(local); @@ -153,7 +142,7 @@ class AssetNotifier extends StateNotifier { } } -final assetProvider = StateNotifierProvider((ref) { +final assetProvider = StateNotifierProvider((ref) { return AssetNotifier( ref.watch(assetServiceProvider), ref.watch(albumServiceProvider), diff --git a/mobile/lib/shared/services/hash.service.dart b/mobile/lib/shared/services/hash.service.dart new file mode 100644 index 000000000..ee272cf5f --- /dev/null +++ b/mobile/lib/shared/services/hash.service.dart @@ -0,0 +1,175 @@ +import 'dart:convert'; +import 'dart:io'; + +import 'package:crypto/crypto.dart'; +import 'package:flutter/foundation.dart'; +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/modules/backup/background_service/background.service.dart'; +import 'package:immich_mobile/shared/models/android_device_asset.dart'; +import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/device_asset.dart'; +import 'package:immich_mobile/shared/models/ios_device_asset.dart'; +import 'package:immich_mobile/shared/providers/db.provider.dart'; +import 'package:immich_mobile/utils/builtin_extensions.dart'; +import 'package:isar/isar.dart'; +import 'package:logging/logging.dart'; +import 'package:photo_manager/photo_manager.dart'; + +class HashService { + HashService(this._db, this._backgroundService); + final Isar _db; + final BackgroundService _backgroundService; + final _log = Logger('HashService'); + + /// Returns all assets that were successfully hashed + Future> getHashedAssets( + AssetPathEntity album, { + int start = 0, + int end = 0x7fffffffffffffff, + Set? excludedAssets, + }) async { + final entities = await album.getAssetListRange(start: start, end: end); + final filtered = excludedAssets == null + ? entities + : entities.where((e) => !excludedAssets.contains(e.id)).toList(); + return _hashAssets(filtered); + } + + /// Converts a list of [AssetEntity]s to [Asset]s including only those + /// that were successfully hashed. Hashes are looked up in a DB table + /// [AndroidDeviceAsset] / [IOSDeviceAsset] by local id. Only missing + /// entries are newly hashed and added to the DB table. + Future> _hashAssets(List assetEntities) async { + const int batchFileCount = 128; + const int batchDataSize = 1024 * 1024 * 1024; // 1GB + + final ids = assetEntities + .map(Platform.isAndroid ? (a) => a.id.toInt() : (a) => a.id) + .toList(); + final List hashes = await _lookupHashes(ids); + final List toAdd = []; + final List toHash = []; + + int bytes = 0; + + for (int i = 0; i < assetEntities.length; i++) { + if (hashes[i] != null) { + continue; + } + final file = await assetEntities[i].originFile; + if (file == null) { + _log.warning( + "Failed to get file for asset ${assetEntities[i].id}, skipping", + ); + continue; + } + bytes += await file.length(); + toHash.add(file.path); + final deviceAsset = Platform.isAndroid + ? AndroidDeviceAsset(id: ids[i] as int, hash: const []) + : IOSDeviceAsset(id: ids[i] as String, hash: const []); + toAdd.add(deviceAsset); + hashes[i] = deviceAsset; + if (toHash.length == batchFileCount || bytes >= batchDataSize) { + await _processBatch(toHash, toAdd); + toAdd.clear(); + toHash.clear(); + bytes = 0; + } + } + if (toHash.isNotEmpty) { + await _processBatch(toHash, toAdd); + } + return _mapAllHashedAssets(assetEntities, hashes); + } + + /// Lookup hashes of assets by their local ID + Future> _lookupHashes(List ids) => + Platform.isAndroid + ? _db.androidDeviceAssets.getAll(ids.cast()) + : _db.iOSDeviceAssets.getAllById(ids.cast()); + + /// Processes a batch of files and saves any successfully hashed + /// values to the DB table. + Future _processBatch( + final List toHash, + final List toAdd, + ) async { + final hashes = await _hashFiles(toHash); + bool anyNull = false; + for (int j = 0; j < hashes.length; j++) { + if (hashes[j]?.length == 20) { + toAdd[j].hash = hashes[j]!; + } else { + _log.warning("Failed to hash file ${toHash[j]}, skipping"); + anyNull = true; + } + } + final validHashes = anyNull + ? toAdd.where((e) => e.hash.length == 20).toList(growable: false) + : toAdd; + await _db.writeTxn( + () => Platform.isAndroid + ? _db.androidDeviceAssets.putAll(validHashes.cast()) + : _db.iOSDeviceAssets.putAll(validHashes.cast()), + ); + _log.fine("Hashed ${validHashes.length}/${toHash.length} assets"); + } + + /// Hashes the given files and returns a list of the same length + /// files that could not be hashed have a `null` value + Future> _hashFiles(List paths) async { + if (Platform.isAndroid) { + final List? hashes = + await _backgroundService.digestFiles(paths); + if (hashes == null) { + throw Exception("Hashing ${paths.length} files failed"); + } + return hashes; + } else if (Platform.isIOS) { + final List result = List.filled(paths.length, null); + for (int i = 0; i < paths.length; i++) { + result[i] = await _hashAssetDart(File(paths[i])); + } + return result; + } else { + throw Exception("_hashFiles implementation missing"); + } + } + + /// Hashes a single file using Dart's crypto package + Future _hashAssetDart(File f) async { + late Digest output; + final sink = sha1.startChunkedConversion( + ChunkedConversionSink.withCallback((accumulated) { + output = accumulated.first; + }), + ); + await for (final chunk in f.openRead()) { + sink.add(chunk); + } + sink.close(); + return Uint8List.fromList(output.bytes); + } + + /// Converts [AssetEntity]s that were successfully hashed to [Asset]s + List _mapAllHashedAssets( + List assets, + List hashes, + ) { + final List result = []; + for (int i = 0; i < assets.length; i++) { + if (hashes[i] != null && hashes[i]!.hash.isNotEmpty) { + result.add(Asset.local(assets[i], hashes[i]!.hash)); + } + } + return result; + } +} + +final hashServiceProvider = Provider( + (ref) => HashService( + ref.watch(dbProvider), + ref.watch(backgroundServiceProvider), + ), +); diff --git a/mobile/lib/shared/services/sync.service.dart b/mobile/lib/shared/services/sync.service.dart index f5471d154..0bb237836 100644 --- a/mobile/lib/shared/services/sync.service.dart +++ b/mobile/lib/shared/services/sync.service.dart @@ -4,10 +4,12 @@ import 'package:collection/collection.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/shared/models/album.dart'; import 'package:immich_mobile/shared/models/asset.dart'; +import 'package:immich_mobile/shared/models/etag.dart'; import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; import 'package:immich_mobile/shared/providers/db.provider.dart'; +import 'package:immich_mobile/shared/services/hash.service.dart'; import 'package:immich_mobile/utils/async_mutex.dart'; import 'package:immich_mobile/utils/builtin_extensions.dart'; import 'package:immich_mobile/utils/diff.dart'; @@ -16,15 +18,17 @@ import 'package:logging/logging.dart'; import 'package:openapi/api.dart'; import 'package:photo_manager/photo_manager.dart'; -final syncServiceProvider = - Provider((ref) => SyncService(ref.watch(dbProvider))); +final syncServiceProvider = Provider( + (ref) => SyncService(ref.watch(dbProvider), ref.watch(hashServiceProvider)), +); class SyncService { final Isar _db; + final HashService _hashService; final AsyncMutex _lock = AsyncMutex(); final Logger _log = Logger('SyncService'); - SyncService(this._db); + SyncService(this._db, this._hashService); // public methods: @@ -33,6 +37,7 @@ class SyncService { Future syncUsersFromServer(List users) async { users.sortBy((u) => u.id); final dbUsers = await _db.users.where().sortById().findAll(); + assert(dbUsers.isSortedBy((u) => u.id), "dbUsers not sorted!"); final List toDelete = []; final List toUpsert = []; final changes = diffSortedListsSync( @@ -108,40 +113,16 @@ class SyncService { // private methods: /// Syncs a new asset to the db. Returns `true` if successful - Future _syncNewAssetToDb(Asset newAsset) async { - final List inDb = await _db.assets - .where() - .localIdDeviceIdEqualTo(newAsset.localId, newAsset.deviceId) - .findAll(); - Asset? match; - if (inDb.length == 1) { - // exactly one match: trivial case - match = inDb.first; - } else if (inDb.length > 1) { - // TODO instead of this heuristics: match by checksum once available - for (Asset a in inDb) { - if (a.ownerId == newAsset.ownerId && - a.fileModifiedAt.isAtSameMomentAs(newAsset.fileModifiedAt)) { - assert(match == null); - match = a; - } - } - if (match == null) { - for (Asset a in inDb) { - if (a.ownerId == newAsset.ownerId) { - assert(match == null); - match = a; - } - } - } - } - if (match != null) { + Future _syncNewAssetToDb(Asset a) async { + final Asset? inDb = + await _db.assets.getByChecksumOwnerId(a.checksum, a.ownerId); + if (inDb != null) { // unify local/remote assets by replacing the // local-only asset in the DB with a local&remote asset - newAsset = match.updatedCopy(newAsset); + a = inDb.updatedCopy(a); } try { - await _db.writeTxn(() => newAsset.put(_db)); + await _db.writeTxn(() => a.put(_db)); } on IsarError catch (e) { _log.severe("Failed to put new asset into db: $e"); return false; @@ -162,11 +143,11 @@ class SyncService { final List inDb = await _db.assets .filter() .ownerIdEqualTo(user.isarId) - .sortByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() + .sortByChecksum() .findAll(); - remote.sort(Asset.compareByOwnerDeviceLocalIdModified); + assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + + remote.sort(Asset.compareByChecksum); final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true); if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) { return false; @@ -199,6 +180,7 @@ class SyncService { query = baseQuery.owner((q) => q.isarIdEqualTo(me.isarId)); } final List dbAlbums = await query.sortByRemoteId().findAll(); + assert(dbAlbums.isSortedBy((e) => e.remoteId!), "dbAlbums not sorted!"); final List toDelete = []; final List existing = []; @@ -245,16 +227,16 @@ class SyncService { if (dto.assetCount != dto.assets.length) { return false; } - final assetsInDb = await album.assets - .filter() - .sortByOwnerId() - .thenByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() - .findAll(); + final assetsInDb = + await album.assets.filter().sortByOwnerId().thenByChecksum().findAll(); + assert(assetsInDb.isSorted(Asset.compareByOwnerChecksum), "inDb unsorted!"); final List assetsOnRemote = dto.getAssets(); - assetsOnRemote.sort(Asset.compareByOwnerDeviceLocalIdModified); - final (toAdd, toUpdate, toUnlink) = _diffAssets(assetsOnRemote, assetsInDb); + assetsOnRemote.sort(Asset.compareByOwnerChecksum); + final (toAdd, toUpdate, toUnlink) = _diffAssets( + assetsOnRemote, + assetsInDb, + compare: Asset.compareByOwnerChecksum, + ); // update shared users final List sharedUsers = album.sharedUsers.toList(growable: false); @@ -297,6 +279,7 @@ class SyncService { await album.assets.update(link: assetsToLink, unlink: toUnlink.cast()); await _db.albums.put(album); }); + _log.info("Synced changes of remote album ${album.name} to DB"); } on IsarError catch (e) { _log.severe("Failed to sync remote album to database $e"); } @@ -382,10 +365,11 @@ class SyncService { Set? excludedAssets, ]) async { onDevice.sort((a, b) => a.id.compareTo(b.id)); - final List inDb = + final inDb = await _db.albums.where().localIdIsNotNull().sortByLocalId().findAll(); final List deleteCandidates = []; final List existing = []; + assert(inDb.isSorted((a, b) => a.localId!.compareTo(b.localId!)), "sort!"); final bool anyChanges = await diffSortedLists( onDevice, inDb, @@ -447,14 +431,15 @@ class SyncService { final inDb = await album.assets .filter() .ownerIdEqualTo(Store.get(StoreKey.currentUser).isarId) - .deviceIdEqualTo(Store.get(StoreKey.deviceIdHash)) - .sortByLocalId() + .sortByChecksum() .findAll(); + assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!"); + final int assetCountOnDevice = await ape.assetCountAsync; final List onDevice = - await ape.getAssets(excludedAssets: excludedAssets); - onDevice.sort(Asset.compareByLocalId); - final (toAdd, toUpdate, toDelete) = - _diffAssets(onDevice, inDb, compare: Asset.compareByLocalId); + await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + _removeDuplicates(onDevice); + // _removeDuplicates sorts `onDevice` by checksum + final (toAdd, toUpdate, toDelete) = _diffAssets(onDevice, inDb); if (toAdd.isEmpty && toUpdate.isEmpty && toDelete.isEmpty && @@ -491,6 +476,9 @@ class SyncService { await _db.albums.put(album); album.thumbnail.value ??= await album.assets.filter().findFirst(); await album.thumbnail.save(); + await _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, value: assetCountOnDevice.toString()), + ); }); _log.info("Synced changes of local album ${ape.name} to DB"); } on IsarError catch (e) { @@ -503,8 +491,13 @@ class SyncService { /// fast path for common case: only new assets were added to device album /// returns `true` if successfull, else `false` Future _syncDeviceAlbumFast(AssetPathEntity ape, Album album) async { + if (!(ape.lastModified ?? DateTime.now()).isAfter(album.modifiedAt)) { + return false; + } final int totalOnDevice = await ape.assetCountAsync; - final AssetPathEntity? modified = totalOnDevice > album.assetCount + final int lastKnownTotal = + (await _db.eTags.getById(ape.eTagKeyAssetCount))?.value?.toInt() ?? 0; + final AssetPathEntity? modified = totalOnDevice > lastKnownTotal ? await ape.fetchPathProperties( filterOptionGroup: FilterOptionGroup( updateTimeCond: DateTimeCond( @@ -517,17 +510,22 @@ class SyncService { if (modified == null) { return false; } - final List newAssets = await modified.getAssets(); - if (totalOnDevice != album.assets.length + newAssets.length) { + final List newAssets = await _hashService.getHashedAssets(modified); + + if (totalOnDevice != lastKnownTotal + newAssets.length) { return false; } album.modifiedAt = ape.lastModified ?? DateTime.now(); + _removeDuplicates(newAssets); final (existingInDb, updated) = await _linkWithExistingFromDb(newAssets); try { await _db.writeTxn(() async { await _db.assets.putAll(updated); await album.assets.update(link: existingInDb + updated); await _db.albums.put(album); + await _db.eTags.put( + ETag(id: ape.eTagKeyAssetCount, value: totalOnDevice.toString()), + ); }); _log.info("Fast synced local album ${ape.name} to DB"); } on IsarError catch (e) { @@ -547,7 +545,9 @@ class SyncService { ]) async { _log.info("Syncing a new local album to DB: ${ape.name}"); final Album a = Album.local(ape); - final assets = await ape.getAssets(excludedAssets: excludedAssets); + final assets = + await _hashService.getHashedAssets(ape, excludedAssets: excludedAssets); + _removeDuplicates(assets); final (existingInDb, updated) = await _linkWithExistingFromDb(assets); _log.info( "${existingInDb.length} assets already existed in DB, to upsert ${updated.length}", @@ -570,44 +570,29 @@ class SyncService { Future<(List existing, List updated)> _linkWithExistingFromDb( List assets, ) async { - if (assets.isEmpty) { - return ([].cast(), [].cast()); - } - final List inDb = await _db.assets - .where() - .anyOf( - assets, - (q, Asset e) => q.localIdDeviceIdEqualTo(e.localId, e.deviceId), - ) - .sortByOwnerId() - .thenByDeviceId() - .thenByLocalId() - .thenByFileModifiedAt() - .findAll(); - assets.sort(Asset.compareByOwnerDeviceLocalIdModified); - final List existing = [], toUpsert = []; - diffSortedListsSync( - inDb, - assets, - // do not compare by modified date because for some assets dates differ on - // client and server, thus never reaching "both" case below - compare: Asset.compareByOwnerDeviceLocalId, - both: (Asset a, Asset b) { - if (a.canUpdate(b)) { - toUpsert.add(a.updatedCopy(b)); - return true; - } else { - existing.add(a); - return false; - } - }, - onlyFirst: (Asset a) => _log.finer( - "_linkWithExistingFromDb encountered asset only in DB: $a", - null, - StackTrace.current, - ), - onlySecond: (Asset b) => toUpsert.add(b), + if (assets.isEmpty) return ([].cast(), [].cast()); + + final List inDb = await _db.assets.getAllByChecksumOwnerId( + assets.map((a) => a.checksum).toList(growable: false), + assets.map((a) => a.ownerId).toInt64List(), ); + assert(inDb.length == assets.length); + final List existing = [], toUpsert = []; + for (int i = 0; i < assets.length; i++) { + final Asset? b = inDb[i]; + if (b == null) { + toUpsert.add(assets[i]); + continue; + } + if (b.canUpdate(assets[i])) { + final updated = b.updatedCopy(assets[i]); + assert(updated.id != Isar.autoIncrement); + toUpsert.add(updated); + } else { + existing.add(b); + } + } + assert(existing.length + toUpsert.length == assets.length); return (existing, toUpsert); } @@ -627,11 +612,63 @@ class SyncService { }); _log.info("Upserted ${assets.length} assets into the DB"); } on IsarError catch (e) { - _log.warning( + _log.severe( "Failed to upsert ${assets.length} assets into the DB: ${e.toString()}", ); + // give details on the errors + assets.sort(Asset.compareByOwnerChecksum); + final inDb = await _db.assets.getAllByChecksumOwnerId( + assets.map((e) => e.checksum).toList(growable: false), + assets.map((e) => e.ownerId).toInt64List(), + ); + for (int i = 0; i < assets.length; i++) { + final Asset a = assets[i]; + final Asset? b = inDb[i]; + if (b == null) { + if (a.id != Isar.autoIncrement) { + _log.warning( + "Trying to update an asset that does not exist in DB:\n$a", + ); + } + } else if (a.id != b.id) { + _log.warning( + "Trying to insert another asset with the same checksum+owner. In DB:\n$b\nTo insert:\n$a", + ); + } + } + for (int i = 1; i < assets.length; i++) { + if (Asset.compareByOwnerChecksum(assets[i - 1], assets[i]) == 0) { + _log.warning( + "Trying to insert duplicate assets:\n${assets[i - 1]}\n${assets[i]}", + ); + } + } } } + + List _removeDuplicates(List assets) { + final int before = assets.length; + assets.sort(Asset.compareByOwnerChecksumCreatedModified); + assets.uniqueConsecutive( + compare: Asset.compareByOwnerChecksum, + onDuplicate: (a, b) => + _log.info("Ignoring duplicate assets on device:\n$a\n$b"), + ); + final int duplicates = before - assets.length; + if (duplicates > 0) { + _log.warning("Ignored $duplicates duplicate assets on device"); + } + return assets; + } + + /// returns `true` if the albums differ on the surface + Future _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async { + return a.name != b.name || + a.lastModified == null || + !a.lastModified!.isAtSameMomentAs(b.modifiedAt) || + await a.assetCountAsync != + (await _db.eTags.getById(a.eTagKeyAssetCount))?.value?.toInt(); + } } /// Returns a triple(toAdd, toUpdate, toRemove) @@ -639,7 +676,7 @@ class SyncService { List assets, List inDb, { bool? remote, - int Function(Asset, Asset) compare = Asset.compareByOwnerDeviceLocalId, + int Function(Asset, Asset) compare = Asset.compareByChecksum, }) { final List toAdd = []; final List toUpdate = []; @@ -663,7 +700,7 @@ class SyncService { } } else if (remote == false && a.isRemote) { if (a.isLocal) { - a.isLocal = false; + a.localId = null; toUpdate.add(a); } } else { @@ -685,9 +722,9 @@ class SyncService { return const ([], []); } deleteCandidates.sort(Asset.compareById); - deleteCandidates.uniqueConsecutive((a) => a.id); + deleteCandidates.uniqueConsecutive(compare: Asset.compareById); existing.sort(Asset.compareById); - existing.uniqueConsecutive((a) => a.id); + existing.uniqueConsecutive(compare: Asset.compareById); final (tooAdd, toUpdate, toRemove) = _diffAssets( existing, deleteCandidates, @@ -698,14 +735,6 @@ class SyncService { return (toRemove.map((e) => e.id).toList(), toUpdate); } -/// returns `true` if the albums differ on the surface -Future _hasAssetPathEntityChanged(AssetPathEntity a, Album b) async { - return a.name != b.name || - a.lastModified == null || - !a.lastModified!.isAtSameMomentAs(b.modifiedAt) || - await a.assetCountAsync != b.assetCount; -} - /// returns `true` if the albums differ on the surface bool _hasAlbumResponseDtoChanged(AlbumResponseDto dto, Album a) { return dto.assetCount != a.assetCount || diff --git a/mobile/lib/shared/views/tab_controller_page.dart b/mobile/lib/shared/views/tab_controller_page.dart index 4d245d034..e14f53514 100644 --- a/mobile/lib/shared/views/tab_controller_page.dart +++ b/mobile/lib/shared/views/tab_controller_page.dart @@ -6,12 +6,39 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; import 'package:immich_mobile/modules/asset_viewer/providers/scroll_notifier.provider.dart'; import 'package:immich_mobile/modules/home/providers/multiselect.provider.dart'; import 'package:immich_mobile/routing/router.dart'; +import 'package:immich_mobile/shared/providers/asset.provider.dart'; -class TabControllerPage extends ConsumerWidget { +class TabControllerPage extends HookConsumerWidget { const TabControllerPage({Key? key}) : super(key: key); @override Widget build(BuildContext context, WidgetRef ref) { + final refreshing = ref.watch(assetProvider); + + Widget buildIcon(Widget icon) { + if (!refreshing) return icon; + return Stack( + alignment: Alignment.center, + clipBehavior: Clip.none, + children: [ + icon, + Positioned( + right: -14, + child: SizedBox( + height: 12, + width: 12, + child: CircularProgressIndicator( + strokeWidth: 2, + valueColor: AlwaysStoppedAnimation( + Theme.of(context).primaryColor, + ), + ), + ), + ), + ], + ); + } + navigationRail(TabsRouter tabsRouter) { return NavigationRail( labelType: NavigationRailLabelType.all, @@ -83,9 +110,12 @@ class TabControllerPage extends ConsumerWidget { icon: const Icon( Icons.photo_library_outlined, ), - selectedIcon: Icon( - Icons.photo_library, - color: Theme.of(context).primaryColor, + selectedIcon: buildIcon( + Icon( + size: 24, + Icons.photo_library, + color: Theme.of(context).primaryColor, + ), ), ), NavigationDestination( @@ -113,9 +143,11 @@ class TabControllerPage extends ConsumerWidget { icon: const Icon( Icons.photo_album_outlined, ), - selectedIcon: Icon( - Icons.photo_album_rounded, - color: Theme.of(context).primaryColor, + selectedIcon: buildIcon( + Icon( + Icons.photo_album_rounded, + color: Theme.of(context).primaryColor, + ), ), ) ], diff --git a/mobile/lib/utils/builtin_extensions.dart b/mobile/lib/utils/builtin_extensions.dart index 3a3a723dc..5b769f26f 100644 --- a/mobile/lib/utils/builtin_extensions.dart +++ b/mobile/lib/utils/builtin_extensions.dart @@ -1,3 +1,5 @@ +import 'dart:typed_data'; + import 'package:collection/collection.dart'; extension DurationExtension on String { @@ -22,15 +24,20 @@ extension DurationExtension on String { } extension ListExtension on List { - List uniqueConsecutive([T Function(E element)? key]) { - key ??= (E e) => e as T; + List uniqueConsecutive({ + int Function(E a, E b)? compare, + void Function(E a, E b)? onDuplicate, + }) { + compare ??= (E a, E b) => a == b ? 0 : 1; int i = 1, j = 1; for (; i < length; i++) { - if (key(this[i]) != key(this[i - 1])) { + if (compare(this[i - 1], this[i]) != 0) { if (i != j) { this[j] = this[i]; } j++; + } else if (onDuplicate != null) { + onDuplicate(this[i - 1], this[i]); } } length = length == 0 ? 0 : j; @@ -45,3 +52,11 @@ extension ListExtension on List { return ListSlice(this, start, end); } } + +extension IntListExtension on Iterable { + Int64List toInt64List() { + final list = Int64List(length); + list.setAll(0, this); + return list; + } +} diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 56e9ba8df..724f3a872 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -8,11 +8,13 @@ Future migrateDatabaseIfNeeded(Isar db) async { final int version = Store.get(StoreKey.version, 1); switch (version) { case 1: - await _migrateV1ToV2(db); + await _migrateTo(db, 2); + case 2: + await _migrateTo(db, 3); } } -Future _migrateV1ToV2(Isar db) async { +Future _migrateTo(Isar db, int version) async { await clearAssetsAndAlbums(db); - await Store.put(StoreKey.version, 2); + await Store.put(StoreKey.version, version); } diff --git a/mobile/pubspec.lock b/mobile/pubspec.lock index 736b7a4f5..71585f92e 100644 --- a/mobile/pubspec.lock +++ b/mobile/pubspec.lock @@ -242,13 +242,13 @@ packages: source: hosted version: "0.3.3+4" crypto: - dependency: transitive + dependency: "direct main" description: name: crypto - sha256: aa274aa7774f8964e4f4f38cc994db7b6158dd36e9187aaceaddc994b35c6c67 + sha256: ff625774173754681d66daaf4a448684fb04b78f902da9cb3d308c19cc5e8bab url: "https://pub.dev" source: hosted - version: "3.0.2" + version: "3.0.3" csslib: dependency: transitive description: @@ -333,10 +333,10 @@ packages: dependency: transitive description: name: ffi - sha256: a38574032c5f1dd06c4aee541789906c12ccaab8ba01446e800d9c5b79c4a978 + sha256: ed5337a5660c506388a9f012be0288fb38b49020ce2b45fe1f8b8323fe429f99 url: "https://pub.dev" source: hosted - version: "2.0.1" + version: "2.0.2" file: dependency: transitive description: diff --git a/mobile/pubspec.yaml b/mobile/pubspec.yaml index 8dfc06389..0a5aa1350 100644 --- a/mobile/pubspec.yaml +++ b/mobile/pubspec.yaml @@ -45,6 +45,7 @@ dependencies: isar_flutter_libs: *isar_version # contains Isar Core permission_handler: ^10.2.0 device_info_plus: ^8.1.0 + crypto: ^3.0.3 # TODO remove once native crypto is used on iOS openapi: path: openapi diff --git a/mobile/test/asset_grid_data_structure_test.dart b/mobile/test/asset_grid_data_structure_test.dart index 78c553f5a..6522bec3d 100644 --- a/mobile/test/asset_grid_data_structure_test.dart +++ b/mobile/test/asset_grid_data_structure_test.dart @@ -13,8 +13,8 @@ void main() { testAssets.add( Asset( + checksum: "", localId: '$i', - deviceId: 1, ownerId: 1, fileCreatedAt: date, fileModifiedAt: date, @@ -23,7 +23,6 @@ void main() { type: AssetType.image, fileName: '', isFavorite: false, - isLocal: false, isArchived: false, ), ); diff --git a/mobile/test/builtin_extensions_text.dart b/mobile/test/builtin_extensions_test.dart similarity index 88% rename from mobile/test/builtin_extensions_text.dart rename to mobile/test/builtin_extensions_test.dart index 9e4924e44..875a20fb0 100644 --- a/mobile/test/builtin_extensions_text.dart +++ b/mobile/test/builtin_extensions_test.dart @@ -43,7 +43,12 @@ void main() { test('withKey', () { final a = ["a", "bb", "cc", "ddd"]; - expect(a.uniqueConsecutive((s) => s.length), ["a", "bb", "ddd"]); + expect( + a.uniqueConsecutive( + compare: (s1, s2) => s1.length.compareTo(s2.length), + ), + ["a", "bb", "ddd"], + ); }); }); } diff --git a/mobile/test/sync_service_test.dart b/mobile/test/sync_service_test.dart index 0d1045c72..63b5614bd 100644 --- a/mobile/test/sync_service_test.dart +++ b/mobile/test/sync_service_test.dart @@ -6,32 +6,33 @@ import 'package:immich_mobile/shared/models/exif_info.dart'; import 'package:immich_mobile/shared/models/logger_message.model.dart'; import 'package:immich_mobile/shared/models/store.dart'; import 'package:immich_mobile/shared/models/user.dart'; +import 'package:immich_mobile/shared/services/hash.service.dart'; import 'package:immich_mobile/shared/services/immich_logger.service.dart'; import 'package:immich_mobile/shared/services/sync.service.dart'; import 'package:isar/isar.dart'; +import 'package:mockito/mockito.dart'; void main() { Asset makeAsset({ - required String localId, + required String checksum, + String? localId, String? remoteId, int deviceId = 1, int ownerId = 590700560494856554, // hash of "1" - bool isLocal = false, }) { final DateTime date = DateTime(2000); return Asset( + checksum: checksum, localId: localId, remoteId: remoteId, - deviceId: deviceId, ownerId: ownerId, fileCreatedAt: date, fileModifiedAt: date, updatedAt: date, durationInSeconds: 0, type: AssetType.image, - fileName: localId, + fileName: localId ?? remoteId ?? "", isFavorite: false, - isLocal: isLocal, isArchived: false, ); } @@ -53,6 +54,7 @@ void main() { group('Test SyncService grouped', () { late final Isar db; + final MockHashService hs = MockHashService(); final owner = User( id: "1", updatedAt: DateTime.now(), @@ -71,11 +73,11 @@ void main() { await Store.put(StoreKey.currentUser, owner); }); final List initialAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1", isLocal: true), - makeAsset(localId: "2", isLocal: true), - makeAsset(localId: "3", isLocal: true), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", localId: "1", remoteId: "1-1"), + makeAsset(checksum: "d", localId: "2"), + makeAsset(checksum: "e", localId: "3"), ]; setUp(() { db.writeTxnSync(() { @@ -84,11 +86,11 @@ void main() { }); }); test('test inserting existing assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1"), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", remoteId: "1-1"), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -97,14 +99,14 @@ void main() { }); test('test inserting new assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "1-1"), - makeAsset(localId: "2", remoteId: "1-2"), - makeAsset(localId: "4", remoteId: "1-4"), - makeAsset(localId: "1", remoteId: "3-1", deviceId: 3), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "c", remoteId: "1-1"), + makeAsset(checksum: "d", remoteId: "1-2"), + makeAsset(checksum: "f", remoteId: "1-4"), + makeAsset(checksum: "g", remoteId: "3-1", deviceId: 3), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -113,14 +115,14 @@ void main() { }); test('test syncing duplicate assets', () async { - SyncService s = SyncService(db); + SyncService s = SyncService(db, hs); final List remoteAssets = [ - makeAsset(localId: "1", remoteId: "0-1", deviceId: 0), - makeAsset(localId: "1", remoteId: "1-1"), - makeAsset(localId: "1", remoteId: "2-1", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1b", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1c", deviceId: 2), - makeAsset(localId: "1", remoteId: "2-1d", deviceId: 2), + makeAsset(checksum: "a", remoteId: "0-1", deviceId: 0), + makeAsset(checksum: "b", remoteId: "1-1"), + makeAsset(checksum: "c", remoteId: "2-1", deviceId: 2), + makeAsset(checksum: "h", remoteId: "2-1b", deviceId: 2), + makeAsset(checksum: "i", remoteId: "2-1c", deviceId: 2), + makeAsset(checksum: "j", remoteId: "2-1d", deviceId: 2), ]; expect(db.assets.countSync(), 5); final bool c1 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); @@ -133,11 +135,13 @@ void main() { final bool c3 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); expect(c3, true); expect(db.assets.countSync(), 7); - remoteAssets.add(makeAsset(localId: "1", remoteId: "2-1e", deviceId: 2)); - remoteAssets.add(makeAsset(localId: "2", remoteId: "2-2", deviceId: 2)); + remoteAssets.add(makeAsset(checksum: "k", remoteId: "2-1e", deviceId: 2)); + remoteAssets.add(makeAsset(checksum: "l", remoteId: "2-2", deviceId: 2)); final bool c4 = await s.syncRemoteAssetsToDb(owner, () => remoteAssets); expect(c4, true); expect(db.assets.countSync(), 9); }); }); } + +class MockHashService extends Mock implements HashService {}