diff --git a/mobile/analysis_options.yaml b/mobile/analysis_options.yaml index dc81c10de..4c06edc8c 100644 --- a/mobile/analysis_options.yaml +++ b/mobile/analysis_options.yaml @@ -56,6 +56,7 @@ custom_lint: allowed: # required / wanted - 'lib/infrastructure/repositories/album_media.repository.dart' + - 'lib/infrastructure/repositories/storage.repository.dart' - 'lib/repositories/{album,asset,file}_media.repository.dart' # acceptable exceptions for the time being - lib/entities/asset.entity.dart # to provide local AssetEntity for now diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt index f4dbda730..18a788903 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/Messages.g.kt @@ -247,6 +247,7 @@ interface NativeSyncApi { fun getAlbums(): List fun getAssetsCountSince(albumId: String, timestamp: Long): Long fun getAssetsForAlbum(albumId: String, updatedTimeCond: Long?): List + fun hashPaths(paths: List): List companion object { /** The codec used by NativeSyncApi. */ @@ -388,6 +389,23 @@ interface NativeSyncApi { channel.setMessageHandler(null) } } + run { + val channel = BasicMessageChannel(binaryMessenger, "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths$separatedMessageChannelSuffix", codec, taskQueue) + if (api != null) { + channel.setMessageHandler { message, reply -> + val args = message as List + val pathsArg = args[0] as List + val wrapped: List = try { + listOf(api.hashPaths(pathsArg)) + } catch (exception: Throwable) { + MessagesPigeonUtils.wrapError(exception) + } + reply.reply(wrapped) + } + } else { + channel.setMessageHandler(null) + } + } } } } diff --git a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt index 232285530..70fc045d5 100644 --- a/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt +++ b/mobile/android/app/src/main/kotlin/app/alextran/immich/sync/MessagesImplBase.kt @@ -4,7 +4,10 @@ import android.annotation.SuppressLint import android.content.Context import android.database.Cursor import android.provider.MediaStore +import android.util.Log import java.io.File +import java.io.FileInputStream +import java.security.MessageDigest sealed class AssetResult { data class ValidAsset(val asset: PlatformAsset, val albumId: String) : AssetResult() @@ -16,6 +19,8 @@ open class NativeSyncApiImplBase(context: Context) { private val ctx: Context = context.applicationContext companion object { + private const val TAG = "NativeSyncApiImplBase" + const val MEDIA_SELECTION = "(${MediaStore.Files.FileColumns.MEDIA_TYPE} = ? OR ${MediaStore.Files.FileColumns.MEDIA_TYPE} = ?)" val MEDIA_SELECTION_ARGS = arrayOf( @@ -34,6 +39,8 @@ open class NativeSyncApiImplBase(context: Context) { MediaStore.MediaColumns.BUCKET_ID, MediaStore.MediaColumns.DURATION ) + + const val HASH_BUFFER_SIZE = 2 * 1024 * 1024 } protected fun getCursor( @@ -174,4 +181,24 @@ open class NativeSyncApiImplBase(context: Context) { .mapNotNull { result -> (result as? AssetResult.ValidAsset)?.asset } .toList() } + + fun hashPaths(paths: List): List { + val buffer = ByteArray(HASH_BUFFER_SIZE) + val digest = MessageDigest.getInstance("SHA-1") + + return paths.map { path -> + try { + FileInputStream(path).use { file -> + var bytesRead: Int + while (file.read(buffer).also { bytesRead = it } > 0) { + digest.update(buffer, 0, bytesRead) + } + } + digest.digest() + } catch (e: Exception) { + Log.w(TAG, "Failed to hash file $path: $e") + null + } + } + } } diff --git a/mobile/drift_schemas/main/drift_schema_v1.json b/mobile/drift_schemas/main/drift_schema_v1.json index 5cdec3d92..ee8e41aea 100644 Binary files a/mobile/drift_schemas/main/drift_schema_v1.json and b/mobile/drift_schemas/main/drift_schema_v1.json differ diff --git a/mobile/ios/Runner/Sync/Messages.g.swift b/mobile/ios/Runner/Sync/Messages.g.swift index 0d7a30268..eb765337c 100644 --- a/mobile/ios/Runner/Sync/Messages.g.swift +++ b/mobile/ios/Runner/Sync/Messages.g.swift @@ -307,6 +307,7 @@ protocol NativeSyncApi { func getAlbums() throws -> [PlatformAlbum] func getAssetsCountSince(albumId: String, timestamp: Int64) throws -> Int64 func getAssetsForAlbum(albumId: String, updatedTimeCond: Int64?) throws -> [PlatformAsset] + func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] } /// Generated setup class from Pigeon to handle messages through the `binaryMessenger`. @@ -442,5 +443,22 @@ class NativeSyncApiSetup { } else { getAssetsForAlbumChannel.setMessageHandler(nil) } + let hashPathsChannel = taskQueue == nil + ? FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec) + : FlutterBasicMessageChannel(name: "dev.flutter.pigeon.immich_mobile.NativeSyncApi.hashPaths\(channelSuffix)", binaryMessenger: binaryMessenger, codec: codec, taskQueue: taskQueue) + if let api = api { + hashPathsChannel.setMessageHandler { message, reply in + let args = message as! [Any?] + let pathsArg = args[0] as! [String] + do { + let result = try api.hashPaths(paths: pathsArg) + reply(wrapResult(result)) + } catch { + reply(wrapError(error)) + } + } + } else { + hashPathsChannel.setMessageHandler(nil) + } } } diff --git a/mobile/ios/Runner/Sync/MessagesImpl.swift b/mobile/ios/Runner/Sync/MessagesImpl.swift index 5d2f08691..06c958b88 100644 --- a/mobile/ios/Runner/Sync/MessagesImpl.swift +++ b/mobile/ios/Runner/Sync/MessagesImpl.swift @@ -1,4 +1,5 @@ import Photos +import CryptoKit struct AssetWrapper: Hashable, Equatable { let asset: PlatformAsset @@ -34,6 +35,8 @@ class NativeSyncApiImpl: NativeSyncApi { private let changeTokenKey = "immich:changeToken" private let albumTypes: [PHAssetCollectionType] = [.album, .smartAlbum] + private let hashBufferSize = 2 * 1024 * 1024 + init(with defaults: UserDefaults = .standard) { self.defaults = defaults } @@ -243,4 +246,24 @@ class NativeSyncApiImpl: NativeSyncApi { } return assets } + + func hashPaths(paths: [String]) throws -> [FlutterStandardTypedData?] { + return paths.map { path in + guard let file = FileHandle(forReadingAtPath: path) else { + print("Cannot open file: \(path)") + return nil + } + + var hasher = Insecure.SHA1() + while autoreleasepool(invoking: { + let chunk = file.readData(ofLength: hashBufferSize) + guard !chunk.isEmpty else { return false } + hasher.update(data: chunk) + return true + }) { } + + let digest = hasher.finalize() + return FlutterStandardTypedData(bytes: Data(digest)) + } + } } diff --git a/mobile/lib/domain/interfaces/local_album.interface.dart b/mobile/lib/domain/interfaces/local_album.interface.dart index 35cfad445..d7b38c567 100644 --- a/mobile/lib/domain/interfaces/local_album.interface.dart +++ b/mobile/lib/domain/interfaces/local_album.interface.dart @@ -29,6 +29,8 @@ abstract interface class ILocalAlbumRepository implements IDatabaseRepository { String albumId, Iterable assetIdsToKeep, ); + + Future> getAssetsToHash(String albumId); } enum SortLocalAlbumsBy { id } diff --git a/mobile/lib/domain/interfaces/local_asset.interface.dart b/mobile/lib/domain/interfaces/local_asset.interface.dart new file mode 100644 index 000000000..5792ebe5d --- /dev/null +++ b/mobile/lib/domain/interfaces/local_asset.interface.dart @@ -0,0 +1,6 @@ +import 'package:immich_mobile/domain/interfaces/db.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; + +abstract interface class ILocalAssetRepository implements IDatabaseRepository { + Future updateHashes(Iterable hashes); +} diff --git a/mobile/lib/domain/interfaces/storage.interface.dart b/mobile/lib/domain/interfaces/storage.interface.dart new file mode 100644 index 000000000..ea6513e7f --- /dev/null +++ b/mobile/lib/domain/interfaces/storage.interface.dart @@ -0,0 +1,7 @@ +import 'dart:io'; + +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; + +abstract interface class IStorageRepository { + Future getFileForAsset(LocalAsset asset); +} diff --git a/mobile/lib/domain/models/local_album.model.dart b/mobile/lib/domain/models/local_album.model.dart index 95c56627b..ee27d91a7 100644 --- a/mobile/lib/domain/models/local_album.model.dart +++ b/mobile/lib/domain/models/local_album.model.dart @@ -1,13 +1,19 @@ enum BackupSelection { - none, - selected, - excluded, + none._(1), + selected._(0), + excluded._(2); + + // Used to sort albums based on the backupSelection + // selected -> none -> excluded + final int sortOrder; + const BackupSelection._(this.sortOrder); } class LocalAlbum { final String id; final String name; final DateTime updatedAt; + final bool isIosSharedAlbum; final int assetCount; final BackupSelection backupSelection; @@ -18,6 +24,7 @@ class LocalAlbum { required this.updatedAt, this.assetCount = 0, this.backupSelection = BackupSelection.none, + this.isIosSharedAlbum = false, }); LocalAlbum copyWith({ @@ -26,6 +33,7 @@ class LocalAlbum { DateTime? updatedAt, int? assetCount, BackupSelection? backupSelection, + bool? isIosSharedAlbum, }) { return LocalAlbum( id: id ?? this.id, @@ -33,6 +41,7 @@ class LocalAlbum { updatedAt: updatedAt ?? this.updatedAt, assetCount: assetCount ?? this.assetCount, backupSelection: backupSelection ?? this.backupSelection, + isIosSharedAlbum: isIosSharedAlbum ?? this.isIosSharedAlbum, ); } @@ -45,7 +54,8 @@ class LocalAlbum { other.name == name && other.updatedAt == updatedAt && other.assetCount == assetCount && - other.backupSelection == backupSelection; + other.backupSelection == backupSelection && + other.isIosSharedAlbum == isIosSharedAlbum; } @override @@ -54,7 +64,8 @@ class LocalAlbum { name.hashCode ^ updatedAt.hashCode ^ assetCount.hashCode ^ - backupSelection.hashCode; + backupSelection.hashCode ^ + isIosSharedAlbum.hashCode; } @override @@ -65,6 +76,7 @@ name: $name, updatedAt: $updatedAt, assetCount: $assetCount, backupSelection: $backupSelection, +isIosSharedAlbum: $isIosSharedAlbum }'''; } } diff --git a/mobile/lib/domain/services/hash.service.dart b/mobile/lib/domain/services/hash.service.dart new file mode 100644 index 000000000..9820453db --- /dev/null +++ b/mobile/lib/domain/services/hash.service.dart @@ -0,0 +1,121 @@ +import 'dart:convert'; + +import 'package:immich_mobile/constants/constants.dart'; +import 'package:immich_mobile/domain/interfaces/local_album.interface.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; +import 'package:immich_mobile/presentation/pages/dev/dev_logger.dart'; +import 'package:logging/logging.dart'; + +class HashService { + final int batchSizeLimit; + final int batchFileLimit; + final ILocalAlbumRepository _localAlbumRepository; + final ILocalAssetRepository _localAssetRepository; + final IStorageRepository _storageRepository; + final NativeSyncApi _nativeSyncApi; + final _log = Logger('HashService'); + + HashService({ + required ILocalAlbumRepository localAlbumRepository, + required ILocalAssetRepository localAssetRepository, + required IStorageRepository storageRepository, + required NativeSyncApi nativeSyncApi, + this.batchSizeLimit = kBatchHashSizeLimit, + this.batchFileLimit = kBatchHashFileLimit, + }) : _localAlbumRepository = localAlbumRepository, + _localAssetRepository = localAssetRepository, + _storageRepository = storageRepository, + _nativeSyncApi = nativeSyncApi; + + Future hashAssets() async { + final Stopwatch stopwatch = Stopwatch()..start(); + // Sorted by backupSelection followed by isCloud + final localAlbums = await _localAlbumRepository.getAll(); + localAlbums.sort((a, b) { + final backupComparison = + a.backupSelection.sortOrder.compareTo(b.backupSelection.sortOrder); + + if (backupComparison != 0) { + return backupComparison; + } + + // Local albums come before iCloud albums + return (a.isIosSharedAlbum ? 1 : 0).compareTo(b.isIosSharedAlbum ? 1 : 0); + }); + + for (final album in localAlbums) { + final assetsToHash = + await _localAlbumRepository.getAssetsToHash(album.id); + if (assetsToHash.isNotEmpty) { + await _hashAssets(assetsToHash); + } + } + + stopwatch.stop(); + _log.info("Hashing took - ${stopwatch.elapsedMilliseconds}ms"); + DLog.log("Hashing took - ${stopwatch.elapsedMilliseconds}ms"); + } + + /// Processes a list of [LocalAsset]s, storing their hash and updating the assets in the DB + /// with hash for those that were successfully hashed. Hashes are looked up in a table + /// [LocalAssetHashEntity] by local id. Only missing entries are newly hashed and added to the DB. + Future _hashAssets(List assetsToHash) async { + int bytesProcessed = 0; + final toHash = <_AssetToPath>[]; + + for (final asset in assetsToHash) { + final file = await _storageRepository.getFileForAsset(asset); + if (file == null) { + continue; + } + + bytesProcessed += await file.length(); + toHash.add(_AssetToPath(asset: asset, path: file.path)); + + if (toHash.length >= batchFileLimit || bytesProcessed >= batchSizeLimit) { + await _processBatch(toHash); + toHash.clear(); + bytesProcessed = 0; + } + } + + await _processBatch(toHash); + } + + /// Processes a batch of assets. + Future _processBatch(List<_AssetToPath> toHash) async { + if (toHash.isEmpty) { + return; + } + + _log.fine("Hashing ${toHash.length} files"); + + final hashed = []; + final hashes = + await _nativeSyncApi.hashPaths(toHash.map((e) => e.path).toList()); + + for (final (index, hash) in hashes.indexed) { + final asset = toHash[index].asset; + if (hash?.length == 20) { + hashed.add(asset.copyWith(checksum: base64.encode(hash!))); + } else { + _log.warning("Failed to hash file ${asset.id}"); + } + } + + _log.fine("Hashed ${hashed.length}/${toHash.length} assets"); + DLog.log("Hashed ${hashed.length}/${toHash.length} assets"); + + await _localAssetRepository.updateHashes(hashed); + } +} + +class _AssetToPath { + final LocalAsset asset; + final String path; + + const _AssetToPath({required this.asset, required this.path}); +} diff --git a/mobile/lib/domain/services/local_sync.service.dart b/mobile/lib/domain/services/local_sync.service.dart index e07595b6d..e39999f22 100644 --- a/mobile/lib/domain/services/local_sync.service.dart +++ b/mobile/lib/domain/services/local_sync.service.dart @@ -365,6 +365,7 @@ extension on Iterable { (e) => LocalAsset( id: e.id, name: e.name, + checksum: null, type: AssetType.values.elementAtOrNull(e.type) ?? AssetType.other, createdAt: e.createdAt == null ? DateTime.now() diff --git a/mobile/lib/domain/utils/background_sync.dart b/mobile/lib/domain/utils/background_sync.dart index 6a694ee44..c8d2e2b62 100644 --- a/mobile/lib/domain/utils/background_sync.dart +++ b/mobile/lib/domain/utils/background_sync.dart @@ -7,6 +7,7 @@ import 'package:worker_manager/worker_manager.dart'; class BackgroundSyncManager { Cancelable? _syncTask; Cancelable? _deviceAlbumSyncTask; + Cancelable? _hashTask; BackgroundSyncManager(); @@ -45,6 +46,20 @@ class BackgroundSyncManager { }); } +// No need to cancel the task, as it can also be run when the user logs out + Future hashAssets() { + if (_hashTask != null) { + return _hashTask!.future; + } + + _hashTask = runInIsolateGentle( + computation: (ref) => ref.read(hashServiceProvider).hashAssets(), + ); + return _hashTask!.whenComplete(() { + _hashTask = null; + }); + } + Future syncRemote() { if (_syncTask != null) { return _syncTask!.future; diff --git a/mobile/lib/infrastructure/entities/local_album.entity.dart b/mobile/lib/infrastructure/entities/local_album.entity.dart index 74c3e7a8f..9657173c3 100644 --- a/mobile/lib/infrastructure/entities/local_album.entity.dart +++ b/mobile/lib/infrastructure/entities/local_album.entity.dart @@ -9,6 +9,8 @@ class LocalAlbumEntity extends Table with DriftDefaultsMixin { TextColumn get name => text()(); DateTimeColumn get updatedAt => dateTime().withDefault(currentDateAndTime)(); IntColumn get backupSelection => intEnum()(); + BoolColumn get isIosSharedAlbum => + boolean().withDefault(const Constant(false))(); // Used for mark & sweep BoolColumn get marker_ => boolean().nullable()(); diff --git a/mobile/lib/infrastructure/entities/local_album.entity.drift.dart b/mobile/lib/infrastructure/entities/local_album.entity.drift.dart index 5955742ec..ff6226ba3 100644 Binary files a/mobile/lib/infrastructure/entities/local_album.entity.drift.dart and b/mobile/lib/infrastructure/entities/local_album.entity.drift.dart differ diff --git a/mobile/lib/infrastructure/repositories/local_album.repository.dart b/mobile/lib/infrastructure/repositories/local_album.repository.dart index 650b7a1aa..5100b7a19 100644 --- a/mobile/lib/infrastructure/repositories/local_album.repository.dart +++ b/mobile/lib/infrastructure/repositories/local_album.repository.dart @@ -98,12 +98,24 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository name: localAlbum.name, updatedAt: Value(localAlbum.updatedAt), backupSelection: localAlbum.backupSelection, + isIosSharedAlbum: Value(localAlbum.isIosSharedAlbum), ); return _db.transaction(() async { await _db.localAlbumEntity .insertOne(companion, onConflict: DoUpdate((_) => companion)); - await _addAssets(localAlbum.id, toUpsert); + if (toUpsert.isNotEmpty) { + await _upsertAssets(toUpsert); + await _db.localAlbumAssetEntity.insertAll( + toUpsert.map( + (a) => LocalAlbumAssetEntityCompanion.insert( + assetId: a.id, + albumId: localAlbum.id, + ), + ), + mode: InsertMode.insertOrIgnore, + ); + } await _removeAssets(localAlbum.id, toDelete); }); } @@ -122,6 +134,7 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository name: album.name, updatedAt: Value(album.updatedAt), backupSelection: album.backupSelection, + isIosSharedAlbum: Value(album.isIosSharedAlbum), marker_: const Value(null), ); @@ -226,21 +239,52 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository }); } - Future _addAssets(String albumId, Iterable assets) { - if (assets.isEmpty) { + @override + Future> getAssetsToHash(String albumId) { + final query = _db.localAlbumAssetEntity.select().join( + [ + innerJoin( + _db.localAssetEntity, + _db.localAlbumAssetEntity.assetId.equalsExp(_db.localAssetEntity.id), + ), + ], + ) + ..where( + _db.localAlbumAssetEntity.albumId.equals(albumId) & + _db.localAssetEntity.checksum.isNull(), + ) + ..orderBy([OrderingTerm.asc(_db.localAssetEntity.id)]); + + return query + .map((row) => row.readTable(_db.localAssetEntity).toDto()) + .get(); + } + + Future _upsertAssets(Iterable localAssets) { + if (localAssets.isEmpty) { return Future.value(); } - return transaction(() async { - await _upsertAssets(assets); - await _db.localAlbumAssetEntity.insertAll( - assets.map( - (a) => LocalAlbumAssetEntityCompanion.insert( - assetId: a.id, - albumId: albumId, + + return _db.batch((batch) async { + for (final asset in localAssets) { + final companion = LocalAssetEntityCompanion.insert( + name: asset.name, + type: asset.type, + createdAt: Value(asset.createdAt), + updatedAt: Value(asset.updatedAt), + durationInSeconds: Value.absentIfNull(asset.durationInSeconds), + id: asset.id, + checksum: const Value(null), + ); + batch.insert<$LocalAssetEntityTable, LocalAssetEntityData>( + _db.localAssetEntity, + companion, + onConflict: DoUpdate( + (_) => companion, + where: (old) => old.updatedAt.isNotValue(asset.updatedAt), ), - ), - mode: InsertMode.insertOrIgnore, - ); + ); + } }); } @@ -301,40 +345,14 @@ class DriftLocalAlbumRepository extends DriftDatabaseRepository return query.map((row) => row.read(assetId)!).get(); } - Future _upsertAssets(Iterable localAssets) { - if (localAssets.isEmpty) { - return Future.value(); - } - - return _db.batch((batch) async { - batch.insertAllOnConflictUpdate( - _db.localAssetEntity, - localAssets.map( - (a) => LocalAssetEntityCompanion.insert( - name: a.name, - type: a.type, - createdAt: Value(a.createdAt), - updatedAt: Value(a.updatedAt), - durationInSeconds: Value.absentIfNull(a.durationInSeconds), - id: a.id, - checksum: Value.absentIfNull(a.checksum), - ), - ), - ); - }); - } - Future _deleteAssets(Iterable ids) { if (ids.isEmpty) { return Future.value(); } - return _db.batch( - (batch) => batch.deleteWhere( - _db.localAssetEntity, - (f) => f.id.isIn(ids), - ), - ); + return _db.batch((batch) { + batch.deleteWhere(_db.localAssetEntity, (f) => f.id.isIn(ids)); + }); } } diff --git a/mobile/lib/infrastructure/repositories/local_asset.repository.dart b/mobile/lib/infrastructure/repositories/local_asset.repository.dart new file mode 100644 index 000000000..350a8dcd3 --- /dev/null +++ b/mobile/lib/infrastructure/repositories/local_asset.repository.dart @@ -0,0 +1,28 @@ +import 'package:drift/drift.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; + +class DriftLocalAssetRepository extends DriftDatabaseRepository + implements ILocalAssetRepository { + final Drift _db; + const DriftLocalAssetRepository(this._db) : super(_db); + + @override + Future updateHashes(Iterable hashes) { + if (hashes.isEmpty) { + return Future.value(); + } + + return _db.batch((batch) async { + for (final asset in hashes) { + batch.update( + _db.localAssetEntity, + LocalAssetEntityCompanion(checksum: Value(asset.checksum)), + where: (e) => e.id.equals(asset.id), + ); + } + }); + } +} diff --git a/mobile/lib/infrastructure/repositories/storage.repository.dart b/mobile/lib/infrastructure/repositories/storage.repository.dart new file mode 100644 index 000000000..57dfc4213 --- /dev/null +++ b/mobile/lib/infrastructure/repositories/storage.repository.dart @@ -0,0 +1,31 @@ +import 'dart:io'; + +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:logging/logging.dart'; +import 'package:photo_manager/photo_manager.dart'; + +class StorageRepository implements IStorageRepository { + final _log = Logger('StorageRepository'); + + @override + Future getFileForAsset(LocalAsset asset) async { + File? file; + try { + final entity = await AssetEntity.fromId(asset.id); + file = await entity?.originFile; + if (file == null) { + _log.warning( + "Cannot get file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt}", + ); + } + } catch (error, stackTrace) { + _log.warning( + "Error getting file for asset ${asset.id}, name: ${asset.name}, created on: ${asset.createdAt}", + error, + stackTrace, + ); + } + return file; + } +} diff --git a/mobile/lib/main.dart b/mobile/lib/main.dart index 32bb02591..469554042 100644 --- a/mobile/lib/main.dart +++ b/mobile/lib/main.dart @@ -18,8 +18,8 @@ import 'package:immich_mobile/providers/db.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; import 'package:immich_mobile/providers/locale_provider.dart'; import 'package:immich_mobile/providers/theme.provider.dart'; -import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/routing/app_navigation_observer.dart'; +import 'package:immich_mobile/routing/router.dart'; import 'package:immich_mobile/services/background.service.dart'; import 'package:immich_mobile/services/local_notification.service.dart'; import 'package:immich_mobile/theme/dynamic_theme.dart'; diff --git a/mobile/lib/platform/native_sync_api.g.dart b/mobile/lib/platform/native_sync_api.g.dart index c4e4c467d..ffcef6796 100644 Binary files a/mobile/lib/platform/native_sync_api.g.dart and b/mobile/lib/platform/native_sync_api.g.dart differ diff --git a/mobile/lib/presentation/pages/dev/dev_logger.dart b/mobile/lib/presentation/pages/dev/dev_logger.dart index 6d179241a..ab9849f87 100644 --- a/mobile/lib/presentation/pages/dev/dev_logger.dart +++ b/mobile/lib/presentation/pages/dev/dev_logger.dart @@ -1,4 +1,5 @@ import 'dart:async'; +import 'dart:io'; import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/models/log.model.dart'; @@ -15,7 +16,6 @@ abstract final class DLog { static Stream> watchLog() { final db = Isar.getInstance(); if (db == null) { - debugPrint('Isar is not initialized'); return const Stream.empty(); } @@ -30,7 +30,6 @@ abstract final class DLog { static void clearLog() { final db = Isar.getInstance(); if (db == null) { - debugPrint('Isar is not initialized'); return; } @@ -40,7 +39,9 @@ abstract final class DLog { } static void log(String message, [Object? error, StackTrace? stackTrace]) { - debugPrint('[$kDevLoggerTag] [${DateTime.now()}] $message'); + if (!Platform.environment.containsKey('FLUTTER_TEST')) { + debugPrint('[$kDevLoggerTag] [${DateTime.now()}] $message'); + } if (error != null) { debugPrint('Error: $error'); } @@ -50,7 +51,6 @@ abstract final class DLog { final isar = Isar.getInstance(); if (isar == null) { - debugPrint('Isar is not initialized'); return; } diff --git a/mobile/lib/presentation/pages/dev/feat_in_development.page.dart b/mobile/lib/presentation/pages/dev/feat_in_development.page.dart index 3ff0b12b9..edbbd2379 100644 --- a/mobile/lib/presentation/pages/dev/feat_in_development.page.dart +++ b/mobile/lib/presentation/pages/dev/feat_in_development.page.dart @@ -26,6 +26,11 @@ final _features = [ icon: Icons.photo_library_rounded, onTap: (_, ref) => ref.read(backgroundSyncProvider).syncLocal(full: true), ), + _Feature( + name: 'Hash Local Assets', + icon: Icons.numbers_outlined, + onTap: (_, ref) => ref.read(backgroundSyncProvider).hashAssets(), + ), _Feature( name: 'Sync Remote', icon: Icons.refresh_rounded, diff --git a/mobile/lib/presentation/pages/dev/media_stat.page.dart b/mobile/lib/presentation/pages/dev/media_stat.page.dart index 5debeff31..c074e524b 100644 --- a/mobile/lib/presentation/pages/dev/media_stat.page.dart +++ b/mobile/lib/presentation/pages/dev/media_stat.page.dart @@ -4,7 +4,6 @@ import 'package:auto_route/auto_route.dart'; import 'package:collection/collection.dart'; import 'package:flutter/material.dart'; import 'package:hooks_riverpod/hooks_riverpod.dart'; -import 'package:immich_mobile/domain/models/local_album.model.dart'; import 'package:immich_mobile/extensions/build_context_extensions.dart'; import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:immich_mobile/providers/infrastructure/album.provider.dart'; @@ -94,9 +93,8 @@ class LocalMediaSummaryPage extends StatelessWidget { ), FutureBuilder( future: albumsFuture, - initialData: [], builder: (_, snap) { - final albums = snap.data!; + final albums = snap.data ?? []; if (albums.isEmpty) { return const SliverToBoxAdapter(child: SizedBox.shrink()); } diff --git a/mobile/lib/providers/infrastructure/asset.provider.dart b/mobile/lib/providers/infrastructure/asset.provider.dart new file mode 100644 index 000000000..d71457147 --- /dev/null +++ b/mobile/lib/providers/infrastructure/asset.provider.dart @@ -0,0 +1,8 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; +import 'package:immich_mobile/infrastructure/repositories/local_asset.repository.dart'; +import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; + +final localAssetRepository = Provider( + (ref) => DriftLocalAssetRepository(ref.watch(driftProvider)), +); diff --git a/mobile/lib/providers/infrastructure/storage.provider.dart b/mobile/lib/providers/infrastructure/storage.provider.dart new file mode 100644 index 000000000..d8ac79f1c --- /dev/null +++ b/mobile/lib/providers/infrastructure/storage.provider.dart @@ -0,0 +1,7 @@ +import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; +import 'package:immich_mobile/infrastructure/repositories/storage.repository.dart'; + +final storageRepositoryProvider = Provider( + (ref) => StorageRepository(), +); diff --git a/mobile/lib/providers/infrastructure/sync.provider.dart b/mobile/lib/providers/infrastructure/sync.provider.dart index 96e470eba..359af6323 100644 --- a/mobile/lib/providers/infrastructure/sync.provider.dart +++ b/mobile/lib/providers/infrastructure/sync.provider.dart @@ -1,13 +1,16 @@ import 'package:hooks_riverpod/hooks_riverpod.dart'; +import 'package:immich_mobile/domain/services/hash.service.dart'; import 'package:immich_mobile/domain/services/local_sync.service.dart'; import 'package:immich_mobile/domain/services/sync_stream.service.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_api.repository.dart'; import 'package:immich_mobile/infrastructure/repositories/sync_stream.repository.dart'; import 'package:immich_mobile/providers/api.provider.dart'; import 'package:immich_mobile/providers/infrastructure/album.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/asset.provider.dart'; import 'package:immich_mobile/providers/infrastructure/cancel.provider.dart'; import 'package:immich_mobile/providers/infrastructure/db.provider.dart'; import 'package:immich_mobile/providers/infrastructure/platform.provider.dart'; +import 'package:immich_mobile/providers/infrastructure/storage.provider.dart'; import 'package:immich_mobile/providers/infrastructure/store.provider.dart'; final syncStreamServiceProvider = Provider( @@ -33,3 +36,12 @@ final localSyncServiceProvider = Provider( storeService: ref.watch(storeServiceProvider), ), ); + +final hashServiceProvider = Provider( + (ref) => HashService( + localAlbumRepository: ref.watch(localAlbumRepository), + localAssetRepository: ref.watch(localAssetRepository), + storageRepository: ref.watch(storageRepositoryProvider), + nativeSyncApi: ref.watch(nativeSyncApiProvider), + ), +); diff --git a/mobile/lib/utils/migration.dart b/mobile/lib/utils/migration.dart index 4519c6d80..a31e441b1 100644 --- a/mobile/lib/utils/migration.dart +++ b/mobile/lib/utils/migration.dart @@ -1,10 +1,13 @@ // ignore_for_file: avoid-unsafe-collection-methods import 'dart:async'; +import 'dart:convert'; import 'dart:io'; +import 'package:drift/drift.dart'; import 'package:flutter/foundation.dart'; import 'package:immich_mobile/domain/models/store.model.dart'; +import 'package:immich_mobile/domain/utils/background_sync.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/entities/android_device_asset.entity.dart'; import 'package:immich_mobile/entities/asset.entity.dart'; @@ -13,14 +16,16 @@ import 'package:immich_mobile/entities/ios_device_asset.entity.dart'; import 'package:immich_mobile/entities/store.entity.dart'; import 'package:immich_mobile/infrastructure/entities/device_asset.entity.dart'; import 'package:immich_mobile/infrastructure/entities/exif.entity.dart'; +import 'package:immich_mobile/infrastructure/entities/local_asset.entity.drift.dart'; import 'package:immich_mobile/infrastructure/entities/store.entity.dart'; import 'package:immich_mobile/infrastructure/entities/user.entity.dart'; +import 'package:immich_mobile/infrastructure/repositories/db.repository.dart'; import 'package:immich_mobile/utils/diff.dart'; import 'package:isar/isar.dart'; // ignore: import_rule_photo_manager import 'package:photo_manager/photo_manager.dart'; -const int targetVersion = 11; +const int targetVersion = 12; Future migrateDatabaseIfNeeded(Isar db) async { final int version = Store.get(StoreKey.version, targetVersion); @@ -45,7 +50,15 @@ Future migrateDatabaseIfNeeded(Isar db) async { await _migrateDeviceAsset(db); } - final shouldTruncate = version < 8 && version < targetVersion; + if (version < 12 && (!kReleaseMode)) { + final backgroundSync = BackgroundSyncManager(); + await backgroundSync.syncLocal(); + final drift = Drift(); + await _migrateDeviceAssetToSqlite(db, drift); + await drift.close(); + } + + final shouldTruncate = version < 8 || version < targetVersion; if (shouldTruncate) { await _migrateTo(db, targetVersion); } @@ -154,6 +167,28 @@ Future _migrateDeviceAsset(Isar db) async { }); } +Future _migrateDeviceAssetToSqlite(Isar db, Drift drift) async { + final isarDeviceAssets = + await db.deviceAssetEntitys.where().sortByAssetId().findAll(); + await drift.batch((batch) { + for (final deviceAsset in isarDeviceAssets) { + final companion = LocalAssetEntityCompanion( + updatedAt: Value(deviceAsset.modifiedTime), + id: Value(deviceAsset.assetId), + checksum: Value(base64.encode(deviceAsset.hash)), + ); + batch.insert<$LocalAssetEntityTable, LocalAssetEntityData>( + drift.localAssetEntity, + companion, + onConflict: DoUpdate( + (_) => companion, + where: (old) => old.updatedAt.equals(deviceAsset.modifiedTime), + ), + ); + } + }); +} + class _DeviceAsset { final String assetId; final List? hash; diff --git a/mobile/pigeon/native_sync_api.dart b/mobile/pigeon/native_sync_api.dart index b8a7500d6..9bcb816a6 100644 --- a/mobile/pigeon/native_sync_api.dart +++ b/mobile/pigeon/native_sync_api.dart @@ -86,4 +86,7 @@ abstract class NativeSyncApi { @TaskQueue(type: TaskQueueType.serialBackgroundThread) List getAssetsForAlbum(String albumId, {int? updatedTimeCond}); + + @TaskQueue(type: TaskQueueType.serialBackgroundThread) + List hashPaths(List paths); } diff --git a/mobile/test/domain/service.mock.dart b/mobile/test/domain/service.mock.dart index 97a3f3029..f4c5a32a4 100644 --- a/mobile/test/domain/service.mock.dart +++ b/mobile/test/domain/service.mock.dart @@ -1,6 +1,7 @@ import 'package:immich_mobile/domain/services/store.service.dart'; import 'package:immich_mobile/domain/services/user.service.dart'; import 'package:immich_mobile/domain/utils/background_sync.dart'; +import 'package:immich_mobile/platform/native_sync_api.g.dart'; import 'package:mocktail/mocktail.dart'; class MockStoreService extends Mock implements StoreService {} @@ -8,3 +9,5 @@ class MockStoreService extends Mock implements StoreService {} class MockUserService extends Mock implements UserService {} class MockBackgroundSyncManager extends Mock implements BackgroundSyncManager {} + +class MockNativeSyncApi extends Mock implements NativeSyncApi {} diff --git a/mobile/test/domain/services/hash_service_test.dart b/mobile/test/domain/services/hash_service_test.dart index 2da41cd70..1401f5d2a 100644 --- a/mobile/test/domain/services/hash_service_test.dart +++ b/mobile/test/domain/services/hash_service_test.dart @@ -1,425 +1,292 @@ import 'dart:convert'; import 'dart:io'; -import 'dart:math'; +import 'dart:typed_data'; -import 'package:collection/collection.dart'; -import 'package:file/memory.dart'; -import 'package:flutter/foundation.dart'; import 'package:flutter_test/flutter_test.dart'; -import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; -import 'package:immich_mobile/domain/models/device_asset.model.dart'; -import 'package:immich_mobile/entities/asset.entity.dart'; -import 'package:immich_mobile/services/background.service.dart'; -import 'package:immich_mobile/services/hash.service.dart'; +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; +import 'package:immich_mobile/domain/models/local_album.model.dart'; +import 'package:immich_mobile/domain/services/hash.service.dart'; import 'package:mocktail/mocktail.dart'; -import 'package:photo_manager/photo_manager.dart'; +import '../../fixtures/album.stub.dart'; import '../../fixtures/asset.stub.dart'; import '../../infrastructure/repository.mock.dart'; -import '../../service.mocks.dart'; +import '../service.mock.dart'; -class MockAsset extends Mock implements Asset {} - -class MockAssetEntity extends Mock implements AssetEntity {} +class MockFile extends Mock implements File {} void main() { late HashService sut; - late BackgroundService mockBackgroundService; - late IDeviceAssetRepository mockDeviceAssetRepository; + late MockLocalAlbumRepository mockAlbumRepo; + late MockLocalAssetRepository mockAssetRepo; + late MockStorageRepository mockStorageRepo; + late MockNativeSyncApi mockNativeApi; setUp(() { - mockBackgroundService = MockBackgroundService(); - mockDeviceAssetRepository = MockDeviceAssetRepository(); + mockAlbumRepo = MockLocalAlbumRepository(); + mockAssetRepo = MockLocalAssetRepository(); + mockStorageRepo = MockStorageRepository(); + mockNativeApi = MockNativeSyncApi(); sut = HashService( - deviceAssetRepository: mockDeviceAssetRepository, - backgroundService: mockBackgroundService, + localAlbumRepository: mockAlbumRepo, + localAssetRepository: mockAssetRepo, + storageRepository: mockStorageRepo, + nativeSyncApi: mockNativeApi, ); - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?)?.call(); - }); - when(() => mockDeviceAssetRepository.updateAll(any())) - .thenAnswer((_) async => true); - when(() => mockDeviceAssetRepository.deleteIds(any())) - .thenAnswer((_) async => true); + registerFallbackValue(LocalAlbumStub.recent); + registerFallbackValue(LocalAssetStub.image1); + + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); }); - group("HashService: No DeviceAsset entry", () { - test("hash successfully", () async { - final (mockAsset, file, deviceAsset, hash) = - await _createAssetMock(AssetStub.image1); - - when(() => mockBackgroundService.digestFiles([file.path])) - .thenAnswer((_) async => [hash]); - // No DB entries for this asset - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer((_) async => []); - - final result = await sut.hashAssets([mockAsset]); - - // Verify we stored the new hash in DB - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?) - ?.call(); - verify( - () => mockDeviceAssetRepository.updateAll([ - deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), - ]), - ).called(1); - verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); - }); - expect( - result, - [AssetStub.image1.copyWith(checksum: base64.encode(hash))], + group('HashService hashAssets', () { + test('processes albums in correct order', () async { + final album1 = LocalAlbumStub.recent + .copyWith(id: "1", backupSelection: BackupSelection.none); + final album2 = LocalAlbumStub.recent + .copyWith(id: "2", backupSelection: BackupSelection.excluded); + final album3 = LocalAlbumStub.recent + .copyWith(id: "3", backupSelection: BackupSelection.selected); + final album4 = LocalAlbumStub.recent.copyWith( + id: "4", + backupSelection: BackupSelection.selected, + isIosSharedAlbum: true, ); - }); - }); - group("HashService: Has DeviceAsset entry", () { - test("when the asset is not modified", () async { - final hash = utf8.encode("image1-hash"); + when(() => mockAlbumRepo.getAll()) + .thenAnswer((_) async => [album1, album2, album4, album3]); + when(() => mockAlbumRepo.getAssetsToHash(any())) + .thenAnswer((_) async => []); - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer( - (_) async => [ - DeviceAsset( - assetId: AssetStub.image1.localId!, - hash: hash, - modifiedTime: AssetStub.image1.fileModifiedAt, - ), - ], - ); - final result = await sut.hashAssets([AssetStub.image1]); + await sut.hashAssets(); - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockBackgroundService.digestFile(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); - - expect(result, [ - AssetStub.image1.copyWith(checksum: base64.encode(hash)), + verifyInOrder([ + () => mockAlbumRepo.getAll(), + () => mockAlbumRepo.getAssetsToHash(album3.id), + () => mockAlbumRepo.getAssetsToHash(album4.id), + () => mockAlbumRepo.getAssetsToHash(album1.id), + () => mockAlbumRepo.getAssetsToHash(album2.id), ]); }); - test("hashed successful when asset is modified", () async { - final (mockAsset, file, deviceAsset, hash) = - await _createAssetMock(AssetStub.image1); + test('skips albums with no assets to hash', () async { + when(() => mockAlbumRepo.getAll()).thenAnswer( + (_) async => [LocalAlbumStub.recent.copyWith(assetCount: 0)], + ); + when(() => mockAlbumRepo.getAssetsToHash(LocalAlbumStub.recent.id)) + .thenAnswer((_) async => []); - when(() => mockBackgroundService.digestFiles([file.path])) - .thenAnswer((_) async => [hash]); - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer((_) async => [deviceAsset]); + await sut.hashAssets(); - final result = await sut.hashAssets([mockAsset]); - - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?) - ?.call(); - verify( - () => mockDeviceAssetRepository.updateAll([ - deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), - ]), - ).called(1); - verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); - }); - - verify(() => mockBackgroundService.digestFiles([file.path])).called(1); - - expect(result, [ - AssetStub.image1.copyWith(checksum: base64.encode(hash)), - ]); + verifyNever(() => mockStorageRepo.getFileForAsset(any())); + verifyNever(() => mockNativeApi.hashPaths(any())); }); }); - group("HashService: Cleanup", () { - late Asset mockAsset; - late Uint8List hash; - late DeviceAsset deviceAsset; - late File file; + group('HashService _hashAssets', () { + test('skips assets without files', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => null); - setUp(() async { - (mockAsset, file, deviceAsset, hash) = - await _createAssetMock(AssetStub.image1); + await sut.hashAssets(); - when(() => mockBackgroundService.digestFiles([file.path])) - .thenAnswer((_) async => [hash]); - when( - () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), - ).thenAnswer((_) async => [deviceAsset]); + verifyNever(() => mockNativeApi.hashPaths(any())); }); - test("cleanups DeviceAsset when local file cannot be obtained", () async { - when(() => mockAsset.local).thenThrow(Exception("File not found")); - final result = await sut.hashAssets([mockAsset]); + test('processes assets when available', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + final mockFile = MockFile(); + final hash = Uint8List.fromList(List.generate(20, (i) => i)); - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockBackgroundService.digestFile(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - verify( - () => mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), - ).called(1); + when(() => mockFile.length()).thenAnswer((_) async => 1000); + when(() => mockFile.path).thenReturn('image-path'); - expect(result, isEmpty); - }); - - test("cleanups DeviceAsset when hashing failed", () async { - when(() => mockDeviceAssetRepository.transaction(any())) - .thenAnswer((_) async { - final capturedCallback = verify( - () => mockDeviceAssetRepository.transaction(captureAny()), - ).captured; - // Invoke the transaction callback - await (capturedCallback.firstOrNull as Future Function()?) - ?.call(); - - // Verify the callback inside the transaction because, doing it outside results - // in a small delay before the callback is invoked, resulting in other LOCs getting executed - // resulting in an incorrect state - // - // i.e, consider the following piece of code - // await _deviceAssetRepository.transaction(() async { - // await _deviceAssetRepository.updateAll(toBeAdded); - // await _deviceAssetRepository.deleteIds(toBeDeleted); - // }); - // toBeDeleted.clear(); - // since the transaction method is mocked, the callback is not invoked until it is captured - // and executed manually in the next event loop. However, the toBeDeleted.clear() is executed - // immediately once the transaction stub is executed, resulting in the deleteIds method being - // called with an empty list. - // - // To avoid this, we capture the callback and execute it within the transaction stub itself - // and verify the results inside the transaction stub - verify(() => mockDeviceAssetRepository.updateAll([])).called(1); - verify( - () => - mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), - ).called(1); - }); - - when(() => mockBackgroundService.digestFiles([file.path])).thenAnswer( - // Invalid hash, length != 20 - (_) async => [Uint8List.fromList(hash.slice(2).toList())], + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => mockFile); + when(() => mockNativeApi.hashPaths(['image-path'])).thenAnswer( + (_) async => [hash], ); - final result = await sut.hashAssets([mockAsset]); + await sut.hashAssets(); - verify(() => mockBackgroundService.digestFiles([file.path])).called(1); - expect(result, isEmpty); - }); - }); - - group("HashService: Batch processing", () { - test("processes assets in batches when size limit is reached", () async { - // Setup multiple assets with large file sizes - final (mock1, mock2, mock3) = await ( - _createAssetMock(AssetStub.image1), - _createAssetMock(AssetStub.image2), - _createAssetMock(AssetStub.image3), - ).wait; - - final (asset1, file1, deviceAsset1, hash1) = mock1; - final (asset2, file2, deviceAsset2, hash2) = mock2; - final (asset3, file3, deviceAsset3, hash3) = mock3; - - when(() => mockDeviceAssetRepository.getByIds(any())) - .thenAnswer((_) async => []); - - // Setup for multiple batch processing calls - when(() => mockBackgroundService.digestFiles([file1.path, file2.path])) - .thenAnswer((_) async => [hash1, hash2]); - when(() => mockBackgroundService.digestFiles([file3.path])) - .thenAnswer((_) async => [hash3]); - - final size = await file1.length() + await file2.length(); - - sut = HashService( - deviceAssetRepository: mockDeviceAssetRepository, - backgroundService: mockBackgroundService, - batchSizeLimit: size, - ); - final result = await sut.hashAssets([asset1, asset2, asset3]); - - // Verify multiple batch process calls - verify(() => mockBackgroundService.digestFiles([file1.path, file2.path])) - .called(1); - verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); - - expect( - result, - [ - AssetStub.image1.copyWith(checksum: base64.encode(hash1)), - AssetStub.image2.copyWith(checksum: base64.encode(hash2)), - AssetStub.image3.copyWith(checksum: base64.encode(hash3)), - ], - ); + verify(() => mockNativeApi.hashPaths(['image-path'])).called(1); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 1); + expect(captured[0].checksum, base64.encode(hash)); }); - test("processes assets in batches when file limit is reached", () async { - // Setup multiple assets with large file sizes - final (mock1, mock2, mock3) = await ( - _createAssetMock(AssetStub.image1), - _createAssetMock(AssetStub.image2), - _createAssetMock(AssetStub.image3), - ).wait; + test('handles failed hashes', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + final mockFile = MockFile(); + when(() => mockFile.length()).thenAnswer((_) async => 1000); + when(() => mockFile.path).thenReturn('image-path'); - final (asset1, file1, deviceAsset1, hash1) = mock1; - final (asset2, file2, deviceAsset2, hash2) = mock2; - final (asset3, file3, deviceAsset3, hash3) = mock3; + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => mockFile); + when(() => mockNativeApi.hashPaths(['image-path'])) + .thenAnswer((_) async => [null]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); - when(() => mockDeviceAssetRepository.getByIds(any())) - .thenAnswer((_) async => []); + await sut.hashAssets(); - when(() => mockBackgroundService.digestFiles([file1.path])) - .thenAnswer((_) async => [hash1]); - when(() => mockBackgroundService.digestFiles([file2.path])) - .thenAnswer((_) async => [hash2]); - when(() => mockBackgroundService.digestFiles([file3.path])) - .thenAnswer((_) async => [hash3]); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 0); + }); - sut = HashService( - deviceAssetRepository: mockDeviceAssetRepository, - backgroundService: mockBackgroundService, + test('handles invalid hash length', () async { + final album = LocalAlbumStub.recent; + final asset = LocalAssetStub.image1; + final mockFile = MockFile(); + when(() => mockFile.length()).thenAnswer((_) async => 1000); + when(() => mockFile.path).thenReturn('image-path'); + + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset]); + when(() => mockStorageRepo.getFileForAsset(asset)) + .thenAnswer((_) async => mockFile); + + final invalidHash = Uint8List.fromList([1, 2, 3]); + when(() => mockNativeApi.hashPaths(['image-path'])) + .thenAnswer((_) async => [invalidHash]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + + await sut.hashAssets(); + + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 0); + }); + + test('batches by file count limit', () async { + final sut = HashService( + localAlbumRepository: mockAlbumRepo, + localAssetRepository: mockAssetRepo, + storageRepository: mockStorageRepo, + nativeSyncApi: mockNativeApi, batchFileLimit: 1, ); - final result = await sut.hashAssets([asset1, asset2, asset3]); - // Verify multiple batch process calls - verify(() => mockBackgroundService.digestFiles([file1.path])).called(1); - verify(() => mockBackgroundService.digestFiles([file2.path])).called(1); - verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + final album = LocalAlbumStub.recent; + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + final mockFile1 = MockFile(); + final mockFile2 = MockFile(); + when(() => mockFile1.length()).thenAnswer((_) async => 100); + when(() => mockFile1.path).thenReturn('path-1'); + when(() => mockFile2.length()).thenAnswer((_) async => 100); + when(() => mockFile2.path).thenReturn('path-2'); - expect( - result, - [ - AssetStub.image1.copyWith(checksum: base64.encode(hash1)), - AssetStub.image2.copyWith(checksum: base64.encode(hash2)), - AssetStub.image3.copyWith(checksum: base64.encode(hash3)), - ], - ); + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset1, asset2]); + when(() => mockStorageRepo.getFileForAsset(asset1)) + .thenAnswer((_) async => mockFile1); + when(() => mockStorageRepo.getFileForAsset(asset2)) + .thenAnswer((_) async => mockFile2); + + final hash = Uint8List.fromList(List.generate(20, (i) => i)); + when(() => mockNativeApi.hashPaths(any())) + .thenAnswer((_) async => [hash]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + + await sut.hashAssets(); + + verify(() => mockNativeApi.hashPaths(['path-1'])).called(1); + verify(() => mockNativeApi.hashPaths(['path-2'])).called(1); + verify(() => mockAssetRepo.updateHashes(any())).called(2); }); - test("HashService: Sort & Process different states", () async { - final (asset1, file1, deviceAsset1, hash1) = - await _createAssetMock(AssetStub.image1); // Will need rehashing - final (asset2, file2, deviceAsset2, hash2) = - await _createAssetMock(AssetStub.image2); // Will have matching hash - final (asset3, file3, deviceAsset3, hash3) = - await _createAssetMock(AssetStub.image3); // No DB entry - final asset4 = - AssetStub.image3.copyWith(localId: "image4"); // Cannot be hashed - - when(() => mockBackgroundService.digestFiles([file1.path, file3.path])) - .thenAnswer((_) async => [hash1, hash3]); - // DB entries are not sorted and a dummy entry added - when( - () => mockDeviceAssetRepository.getByIds([ - AssetStub.image1.localId!, - AssetStub.image2.localId!, - AssetStub.image3.localId!, - asset4.localId!, - ]), - ).thenAnswer( - (_) async => [ - // Same timestamp to reuse deviceAsset - deviceAsset2.copyWith(modifiedTime: asset2.fileModifiedAt), - deviceAsset1, - deviceAsset3.copyWith(assetId: asset4.localId!), - ], + test('batches by size limit', () async { + final sut = HashService( + localAlbumRepository: mockAlbumRepo, + localAssetRepository: mockAssetRepo, + storageRepository: mockStorageRepo, + nativeSyncApi: mockNativeApi, + batchSizeLimit: 80, ); - final result = await sut.hashAssets([asset1, asset2, asset3, asset4]); + final album = LocalAlbumStub.recent; + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + final mockFile1 = MockFile(); + final mockFile2 = MockFile(); + when(() => mockFile1.length()).thenAnswer((_) async => 100); + when(() => mockFile1.path).thenReturn('path-1'); + when(() => mockFile2.length()).thenAnswer((_) async => 100); + when(() => mockFile2.path).thenReturn('path-2'); - // Verify correct processing of all assets - verify(() => mockBackgroundService.digestFiles([file1.path, file3.path])) - .called(1); - expect(result.length, 3); - expect(result, [ - AssetStub.image2.copyWith(checksum: base64.encode(hash2)), - AssetStub.image1.copyWith(checksum: base64.encode(hash1)), - AssetStub.image3.copyWith(checksum: base64.encode(hash3)), - ]); + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset1, asset2]); + when(() => mockStorageRepo.getFileForAsset(asset1)) + .thenAnswer((_) async => mockFile1); + when(() => mockStorageRepo.getFileForAsset(asset2)) + .thenAnswer((_) async => mockFile2); + + final hash = Uint8List.fromList(List.generate(20, (i) => i)); + when(() => mockNativeApi.hashPaths(any())) + .thenAnswer((_) async => [hash]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); + + await sut.hashAssets(); + + verify(() => mockNativeApi.hashPaths(['path-1'])).called(1); + verify(() => mockNativeApi.hashPaths(['path-2'])).called(1); + verify(() => mockAssetRepo.updateHashes(any())).called(2); }); - group("HashService: Edge cases", () { - test("handles empty list of assets", () async { - when(() => mockDeviceAssetRepository.getByIds(any())) - .thenAnswer((_) async => []); + test('handles mixed success and failure in batch', () async { + final album = LocalAlbumStub.recent; + final asset1 = LocalAssetStub.image1; + final asset2 = LocalAssetStub.image2; + final mockFile1 = MockFile(); + final mockFile2 = MockFile(); + when(() => mockFile1.length()).thenAnswer((_) async => 100); + when(() => mockFile1.path).thenReturn('path-1'); + when(() => mockFile2.length()).thenAnswer((_) async => 100); + when(() => mockFile2.path).thenReturn('path-2'); - final result = await sut.hashAssets([]); + when(() => mockAlbumRepo.getAll()).thenAnswer((_) async => [album]); + when(() => mockAlbumRepo.getAssetsToHash(album.id)) + .thenAnswer((_) async => [asset1, asset2]); + when(() => mockStorageRepo.getFileForAsset(asset1)) + .thenAnswer((_) async => mockFile1); + when(() => mockStorageRepo.getFileForAsset(asset2)) + .thenAnswer((_) async => mockFile2); - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + final validHash = Uint8List.fromList(List.generate(20, (i) => i)); + when(() => mockNativeApi.hashPaths(['path-1', 'path-2'])) + .thenAnswer((_) async => [validHash, null]); + when(() => mockAssetRepo.updateHashes(any())).thenAnswer((_) async => {}); - expect(result, isEmpty); - }); + await sut.hashAssets(); - test("handles all file access failures", () async { - // No DB entries - when( - () => mockDeviceAssetRepository.getByIds( - [AssetStub.image1.localId!, AssetStub.image2.localId!], - ), - ).thenAnswer((_) async => []); - - final result = await sut.hashAssets([ - AssetStub.image1, - AssetStub.image2, - ]); - - verifyNever(() => mockBackgroundService.digestFiles(any())); - verifyNever(() => mockDeviceAssetRepository.updateAll(any())); - expect(result, isEmpty); - }); + final captured = verify(() => mockAssetRepo.updateHashes(captureAny())) + .captured + .first as List; + expect(captured.length, 1); + expect(captured.first.id, asset1.id); }); }); } - -Future<(Asset, File, DeviceAsset, Uint8List)> _createAssetMock( - Asset asset, -) async { - final random = Random(); - final hash = - Uint8List.fromList(List.generate(20, (i) => random.nextInt(255))); - final mockAsset = MockAsset(); - final mockAssetEntity = MockAssetEntity(); - final fs = MemoryFileSystem(); - final deviceAsset = DeviceAsset( - assetId: asset.localId!, - hash: Uint8List.fromList(hash), - modifiedTime: DateTime.now(), - ); - final tmp = await fs.systemTempDirectory.createTemp(); - final file = tmp.childFile("${asset.fileName}-path"); - await file.writeAsString("${asset.fileName}-content"); - - when(() => mockAsset.localId).thenReturn(asset.localId); - when(() => mockAsset.fileName).thenReturn(asset.fileName); - when(() => mockAsset.fileCreatedAt).thenReturn(asset.fileCreatedAt); - when(() => mockAsset.fileModifiedAt).thenReturn(asset.fileModifiedAt); - when(() => mockAsset.copyWith(checksum: any(named: "checksum"))) - .thenReturn(asset.copyWith(checksum: base64.encode(hash))); - when(() => mockAsset.local).thenAnswer((_) => mockAssetEntity); - when(() => mockAssetEntity.originFile).thenAnswer((_) async => file); - - return (mockAsset, file, deviceAsset, hash); -} diff --git a/mobile/test/fixtures/album.stub.dart b/mobile/test/fixtures/album.stub.dart index c6ea199c0..1432d3590 100644 --- a/mobile/test/fixtures/album.stub.dart +++ b/mobile/test/fixtures/album.stub.dart @@ -1,3 +1,4 @@ +import 'package:immich_mobile/domain/models/local_album.model.dart'; import 'package:immich_mobile/entities/album.entity.dart'; import 'package:immich_mobile/infrastructure/entities/user.entity.dart'; @@ -101,3 +102,16 @@ final class AlbumStub { endDate: DateTime(2026), ); } + +abstract final class LocalAlbumStub { + const LocalAlbumStub._(); + + static final recent = LocalAlbum( + id: "recent-local-id", + name: "Recent", + updatedAt: DateTime(2023), + assetCount: 1000, + backupSelection: BackupSelection.none, + isIosSharedAlbum: false, + ); +} diff --git a/mobile/test/fixtures/asset.stub.dart b/mobile/test/fixtures/asset.stub.dart index 771b2dda9..8d9201199 100644 --- a/mobile/test/fixtures/asset.stub.dart +++ b/mobile/test/fixtures/asset.stub.dart @@ -1,10 +1,11 @@ +import 'package:immich_mobile/domain/models/asset/base_asset.model.dart'; import 'package:immich_mobile/domain/models/exif.model.dart'; -import 'package:immich_mobile/entities/asset.entity.dart'; +import 'package:immich_mobile/entities/asset.entity.dart' as old; final class AssetStub { const AssetStub._(); - static final image1 = Asset( + static final image1 = old.Asset( checksum: "image1-checksum", localId: "image1", remoteId: 'image1-remote', @@ -13,7 +14,7 @@ final class AssetStub { fileModifiedAt: DateTime(2020), updatedAt: DateTime.now(), durationInSeconds: 0, - type: AssetType.image, + type: old.AssetType.image, fileName: "image1.jpg", isFavorite: true, isArchived: false, @@ -21,7 +22,7 @@ final class AssetStub { exifInfo: const ExifInfo(isFlipped: false), ); - static final image2 = Asset( + static final image2 = old.Asset( checksum: "image2-checksum", localId: "image2", remoteId: 'image2-remote', @@ -30,7 +31,7 @@ final class AssetStub { fileModifiedAt: DateTime(2010), updatedAt: DateTime.now(), durationInSeconds: 60, - type: AssetType.video, + type: old.AssetType.video, fileName: "image2.jpg", isFavorite: false, isArchived: false, @@ -38,7 +39,7 @@ final class AssetStub { exifInfo: const ExifInfo(isFlipped: true), ); - static final image3 = Asset( + static final image3 = old.Asset( checksum: "image3-checksum", localId: "image3", ownerId: 1, @@ -46,10 +47,30 @@ final class AssetStub { fileModifiedAt: DateTime(2025), updatedAt: DateTime.now(), durationInSeconds: 60, - type: AssetType.image, + type: old.AssetType.image, fileName: "image3.jpg", isFavorite: true, isArchived: false, isTrashed: false, ); } + +abstract final class LocalAssetStub { + const LocalAssetStub._(); + + static final image1 = LocalAsset( + id: "image1", + name: "image1.jpg", + type: AssetType.image, + createdAt: DateTime(2025), + updatedAt: DateTime(2025, 2), + ); + + static final image2 = LocalAsset( + id: "image2", + name: "image2.jpg", + type: AssetType.image, + createdAt: DateTime(2000), + updatedAt: DateTime(20021), + ); +} diff --git a/mobile/test/infrastructure/repository.mock.dart b/mobile/test/infrastructure/repository.mock.dart index c4a5680f7..0dc241ca9 100644 --- a/mobile/test/infrastructure/repository.mock.dart +++ b/mobile/test/infrastructure/repository.mock.dart @@ -1,5 +1,8 @@ import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/domain/interfaces/local_album.interface.dart'; +import 'package:immich_mobile/domain/interfaces/local_asset.interface.dart'; import 'package:immich_mobile/domain/interfaces/log.interface.dart'; +import 'package:immich_mobile/domain/interfaces/storage.interface.dart'; import 'package:immich_mobile/domain/interfaces/store.interface.dart'; import 'package:immich_mobile/domain/interfaces/sync_api.interface.dart'; import 'package:immich_mobile/domain/interfaces/sync_stream.interface.dart'; @@ -18,6 +21,12 @@ class MockDeviceAssetRepository extends Mock class MockSyncStreamRepository extends Mock implements ISyncStreamRepository {} +class MockLocalAlbumRepository extends Mock implements ILocalAlbumRepository {} + +class MockLocalAssetRepository extends Mock implements ILocalAssetRepository {} + +class MockStorageRepository extends Mock implements IStorageRepository {} + // API Repos class MockUserApiRepository extends Mock implements IUserApiRepository {} diff --git a/mobile/test/services/hash_service_test.dart b/mobile/test/services/hash_service_test.dart new file mode 100644 index 000000000..e278199e4 --- /dev/null +++ b/mobile/test/services/hash_service_test.dart @@ -0,0 +1,425 @@ +import 'dart:convert'; +import 'dart:io'; +import 'dart:math'; + +import 'package:collection/collection.dart'; +import 'package:file/memory.dart'; +import 'package:flutter/foundation.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:immich_mobile/domain/interfaces/device_asset.interface.dart'; +import 'package:immich_mobile/domain/models/device_asset.model.dart'; +import 'package:immich_mobile/entities/asset.entity.dart'; +import 'package:immich_mobile/services/background.service.dart'; +import 'package:immich_mobile/services/hash.service.dart'; +import 'package:mocktail/mocktail.dart'; +import 'package:photo_manager/photo_manager.dart'; + +import '../fixtures/asset.stub.dart'; +import '../infrastructure/repository.mock.dart'; +import '../service.mocks.dart'; + +class MockAsset extends Mock implements Asset {} + +class MockAssetEntity extends Mock implements AssetEntity {} + +void main() { + late HashService sut; + late BackgroundService mockBackgroundService; + late IDeviceAssetRepository mockDeviceAssetRepository; + + setUp(() { + mockBackgroundService = MockBackgroundService(); + mockDeviceAssetRepository = MockDeviceAssetRepository(); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + ); + + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?)?.call(); + }); + when(() => mockDeviceAssetRepository.updateAll(any())) + .thenAnswer((_) async => true); + when(() => mockDeviceAssetRepository.deleteIds(any())) + .thenAnswer((_) async => true); + }); + + group("HashService: No DeviceAsset entry", () { + test("hash successfully", () async { + final (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + // No DB entries for this asset + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => []); + + final result = await sut.hashAssets([mockAsset]); + + // Verify we stored the new hash in DB + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + verify( + () => mockDeviceAssetRepository.updateAll([ + deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), + ]), + ).called(1); + verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); + }); + expect( + result, + [AssetStub.image1.copyWith(checksum: base64.encode(hash))], + ); + }); + }); + + group("HashService: Has DeviceAsset entry", () { + test("when the asset is not modified", () async { + final hash = utf8.encode("image1-hash"); + + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer( + (_) async => [ + DeviceAsset( + assetId: AssetStub.image1.localId!, + hash: hash, + modifiedTime: AssetStub.image1.fileModifiedAt, + ), + ], + ); + final result = await sut.hashAssets([AssetStub.image1]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockBackgroundService.digestFile(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + + expect(result, [ + AssetStub.image1.copyWith(checksum: base64.encode(hash)), + ]); + }); + + test("hashed successful when asset is modified", () async { + final (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => [deviceAsset]); + + final result = await sut.hashAssets([mockAsset]); + + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + verify( + () => mockDeviceAssetRepository.updateAll([ + deviceAsset.copyWith(modifiedTime: AssetStub.image1.fileModifiedAt), + ]), + ).called(1); + verify(() => mockDeviceAssetRepository.deleteIds([])).called(1); + }); + + verify(() => mockBackgroundService.digestFiles([file.path])).called(1); + + expect(result, [ + AssetStub.image1.copyWith(checksum: base64.encode(hash)), + ]); + }); + }); + + group("HashService: Cleanup", () { + late Asset mockAsset; + late Uint8List hash; + late DeviceAsset deviceAsset; + late File file; + + setUp(() async { + (mockAsset, file, deviceAsset, hash) = + await _createAssetMock(AssetStub.image1); + + when(() => mockBackgroundService.digestFiles([file.path])) + .thenAnswer((_) async => [hash]); + when( + () => mockDeviceAssetRepository.getByIds([AssetStub.image1.localId!]), + ).thenAnswer((_) async => [deviceAsset]); + }); + + test("cleanups DeviceAsset when local file cannot be obtained", () async { + when(() => mockAsset.local).thenThrow(Exception("File not found")); + final result = await sut.hashAssets([mockAsset]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockBackgroundService.digestFile(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verify( + () => mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), + ).called(1); + + expect(result, isEmpty); + }); + + test("cleanups DeviceAsset when hashing failed", () async { + when(() => mockDeviceAssetRepository.transaction(any())) + .thenAnswer((_) async { + final capturedCallback = verify( + () => mockDeviceAssetRepository.transaction(captureAny()), + ).captured; + // Invoke the transaction callback + await (capturedCallback.firstOrNull as Future Function()?) + ?.call(); + + // Verify the callback inside the transaction because, doing it outside results + // in a small delay before the callback is invoked, resulting in other LOCs getting executed + // resulting in an incorrect state + // + // i.e, consider the following piece of code + // await _deviceAssetRepository.transaction(() async { + // await _deviceAssetRepository.updateAll(toBeAdded); + // await _deviceAssetRepository.deleteIds(toBeDeleted); + // }); + // toBeDeleted.clear(); + // since the transaction method is mocked, the callback is not invoked until it is captured + // and executed manually in the next event loop. However, the toBeDeleted.clear() is executed + // immediately once the transaction stub is executed, resulting in the deleteIds method being + // called with an empty list. + // + // To avoid this, we capture the callback and execute it within the transaction stub itself + // and verify the results inside the transaction stub + verify(() => mockDeviceAssetRepository.updateAll([])).called(1); + verify( + () => + mockDeviceAssetRepository.deleteIds([AssetStub.image1.localId!]), + ).called(1); + }); + + when(() => mockBackgroundService.digestFiles([file.path])).thenAnswer( + // Invalid hash, length != 20 + (_) async => [Uint8List.fromList(hash.slice(2).toList())], + ); + + final result = await sut.hashAssets([mockAsset]); + + verify(() => mockBackgroundService.digestFiles([file.path])).called(1); + expect(result, isEmpty); + }); + }); + + group("HashService: Batch processing", () { + test("processes assets in batches when size limit is reached", () async { + // Setup multiple assets with large file sizes + final (mock1, mock2, mock3) = await ( + _createAssetMock(AssetStub.image1), + _createAssetMock(AssetStub.image2), + _createAssetMock(AssetStub.image3), + ).wait; + + final (asset1, file1, deviceAsset1, hash1) = mock1; + final (asset2, file2, deviceAsset2, hash2) = mock2; + final (asset3, file3, deviceAsset3, hash3) = mock3; + + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + // Setup for multiple batch processing calls + when(() => mockBackgroundService.digestFiles([file1.path, file2.path])) + .thenAnswer((_) async => [hash1, hash2]); + when(() => mockBackgroundService.digestFiles([file3.path])) + .thenAnswer((_) async => [hash3]); + + final size = await file1.length() + await file2.length(); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + batchSizeLimit: size, + ); + final result = await sut.hashAssets([asset1, asset2, asset3]); + + // Verify multiple batch process calls + verify(() => mockBackgroundService.digestFiles([file1.path, file2.path])) + .called(1); + verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + + expect( + result, + [ + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ], + ); + }); + + test("processes assets in batches when file limit is reached", () async { + // Setup multiple assets with large file sizes + final (mock1, mock2, mock3) = await ( + _createAssetMock(AssetStub.image1), + _createAssetMock(AssetStub.image2), + _createAssetMock(AssetStub.image3), + ).wait; + + final (asset1, file1, deviceAsset1, hash1) = mock1; + final (asset2, file2, deviceAsset2, hash2) = mock2; + final (asset3, file3, deviceAsset3, hash3) = mock3; + + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + when(() => mockBackgroundService.digestFiles([file1.path])) + .thenAnswer((_) async => [hash1]); + when(() => mockBackgroundService.digestFiles([file2.path])) + .thenAnswer((_) async => [hash2]); + when(() => mockBackgroundService.digestFiles([file3.path])) + .thenAnswer((_) async => [hash3]); + + sut = HashService( + deviceAssetRepository: mockDeviceAssetRepository, + backgroundService: mockBackgroundService, + batchFileLimit: 1, + ); + final result = await sut.hashAssets([asset1, asset2, asset3]); + + // Verify multiple batch process calls + verify(() => mockBackgroundService.digestFiles([file1.path])).called(1); + verify(() => mockBackgroundService.digestFiles([file2.path])).called(1); + verify(() => mockBackgroundService.digestFiles([file3.path])).called(1); + + expect( + result, + [ + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ], + ); + }); + + test("HashService: Sort & Process different states", () async { + final (asset1, file1, deviceAsset1, hash1) = + await _createAssetMock(AssetStub.image1); // Will need rehashing + final (asset2, file2, deviceAsset2, hash2) = + await _createAssetMock(AssetStub.image2); // Will have matching hash + final (asset3, file3, deviceAsset3, hash3) = + await _createAssetMock(AssetStub.image3); // No DB entry + final asset4 = + AssetStub.image3.copyWith(localId: "image4"); // Cannot be hashed + + when(() => mockBackgroundService.digestFiles([file1.path, file3.path])) + .thenAnswer((_) async => [hash1, hash3]); + // DB entries are not sorted and a dummy entry added + when( + () => mockDeviceAssetRepository.getByIds([ + AssetStub.image1.localId!, + AssetStub.image2.localId!, + AssetStub.image3.localId!, + asset4.localId!, + ]), + ).thenAnswer( + (_) async => [ + // Same timestamp to reuse deviceAsset + deviceAsset2.copyWith(modifiedTime: asset2.fileModifiedAt), + deviceAsset1, + deviceAsset3.copyWith(assetId: asset4.localId!), + ], + ); + + final result = await sut.hashAssets([asset1, asset2, asset3, asset4]); + + // Verify correct processing of all assets + verify(() => mockBackgroundService.digestFiles([file1.path, file3.path])) + .called(1); + expect(result.length, 3); + expect(result, [ + AssetStub.image2.copyWith(checksum: base64.encode(hash2)), + AssetStub.image1.copyWith(checksum: base64.encode(hash1)), + AssetStub.image3.copyWith(checksum: base64.encode(hash3)), + ]); + }); + + group("HashService: Edge cases", () { + test("handles empty list of assets", () async { + when(() => mockDeviceAssetRepository.getByIds(any())) + .thenAnswer((_) async => []); + + final result = await sut.hashAssets([]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + verifyNever(() => mockDeviceAssetRepository.deleteIds(any())); + + expect(result, isEmpty); + }); + + test("handles all file access failures", () async { + // No DB entries + when( + () => mockDeviceAssetRepository.getByIds( + [AssetStub.image1.localId!, AssetStub.image2.localId!], + ), + ).thenAnswer((_) async => []); + + final result = await sut.hashAssets([ + AssetStub.image1, + AssetStub.image2, + ]); + + verifyNever(() => mockBackgroundService.digestFiles(any())); + verifyNever(() => mockDeviceAssetRepository.updateAll(any())); + expect(result, isEmpty); + }); + }); + }); +} + +Future<(Asset, File, DeviceAsset, Uint8List)> _createAssetMock( + Asset asset, +) async { + final random = Random(); + final hash = + Uint8List.fromList(List.generate(20, (i) => random.nextInt(255))); + final mockAsset = MockAsset(); + final mockAssetEntity = MockAssetEntity(); + final fs = MemoryFileSystem(); + final deviceAsset = DeviceAsset( + assetId: asset.localId!, + hash: Uint8List.fromList(hash), + modifiedTime: DateTime.now(), + ); + final tmp = await fs.systemTempDirectory.createTemp(); + final file = tmp.childFile("${asset.fileName}-path"); + await file.writeAsString("${asset.fileName}-content"); + + when(() => mockAsset.localId).thenReturn(asset.localId); + when(() => mockAsset.fileName).thenReturn(asset.fileName); + when(() => mockAsset.fileCreatedAt).thenReturn(asset.fileCreatedAt); + when(() => mockAsset.fileModifiedAt).thenReturn(asset.fileModifiedAt); + when(() => mockAsset.copyWith(checksum: any(named: "checksum"))) + .thenReturn(asset.copyWith(checksum: base64.encode(hash))); + when(() => mockAsset.local).thenAnswer((_) => mockAssetEntity); + when(() => mockAssetEntity.originFile).thenAnswer((_) async => file); + + return (mockAsset, file, deviceAsset, hash); +}