feat(mobile)!: batched full/initial sync (#4840)

* feat(mobile): batched full/initial sync

* use OptionalBetween

* skip/take as integer

---------

Co-authored-by: Fynn Petersen-Frey <zoodyy@users.noreply.github.com>
Co-authored-by: Alex Tran <alex.tran1502@gmail.com>

cli/src/api/open-api/api.ts

@@ -6695,16 +6695,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
         },
         /**
          * Get all AssetEntity belong to the user
+         * @param {number} [skip] 
+         * @param {number} [take] 
          * @param {string} [userId] 
          * @param {boolean} [isFavorite] 
          * @param {boolean} [isArchived] 
-         * @param {number} [skip] 
          * @param {string} [updatedAfter] 
+         * @param {string} [updatedBefore] 
          * @param {string} [ifNoneMatch] ETag of data already cached on the client
          * @param {*} [options] Override http request option.
          * @throws {RequiredError}
          */
-        getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
+        getAllAssets: async (skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
             const localVarPath = `/asset`;
             // use dummy base URL string because the URL constructor only accepts absolute URLs.
             const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
@@ -6726,6 +6728,14 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
             // http bearer authentication required
             await setBearerAuthToObject(localVarHeaderParameter, configuration)
 
+            if (skip !== undefined) {
+                localVarQueryParameter['skip'] = skip;
+            }
+
+            if (take !== undefined) {
+                localVarQueryParameter['take'] = take;
+            }
+
             if (userId !== undefined) {
                 localVarQueryParameter['userId'] = userId;
             }
@@ -6738,16 +6748,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
                 localVarQueryParameter['isArchived'] = isArchived;
             }
 
-            if (skip !== undefined) {
-                localVarQueryParameter['skip'] = skip;
-            }
-
             if (updatedAfter !== undefined) {
                 localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ?
                     (updatedAfter as any).toISOString() :
                     updatedAfter;
             }
 
+            if (updatedBefore !== undefined) {
+                localVarQueryParameter['updatedBefore'] = (updatedBefore as any instanceof Date) ?
+                    (updatedBefore as any).toISOString() :
+                    updatedBefore;
+            }
+
             if (ifNoneMatch != null) {
                 localVarHeaderParameter['if-none-match'] = String(ifNoneMatch);
             }
@@ -8066,17 +8078,19 @@ export const AssetApiFp = function(configuration?: Configuration) {
         },
         /**
          * Get all AssetEntity belong to the user
+         * @param {number} [skip] 
+         * @param {number} [take] 
          * @param {string} [userId] 
          * @param {boolean} [isFavorite] 
          * @param {boolean} [isArchived] 
-         * @param {number} [skip] 
          * @param {string} [updatedAfter] 
+         * @param {string} [updatedBefore] 
          * @param {string} [ifNoneMatch] ETag of data already cached on the client
          * @param {*} [options] Override http request option.
          * @throws {RequiredError}
          */
-        async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
+        async getAllAssets(skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
-            const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch, options);
+            const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch, options);
             return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
         },
         /**
@@ -8421,7 +8435,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
          * @throws {RequiredError}
          */
         getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise<Array<AssetResponseDto>> {
-            return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
+            return localVarFp.getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
         },
         /**
          * Get a single asset\'s information
@@ -8719,6 +8733,20 @@ export interface AssetApiDownloadFileRequest {
  * @interface AssetApiGetAllAssetsRequest
  */
 export interface AssetApiGetAllAssetsRequest {
+    /**
+     * 
+     * @type {number}
+     * @memberof AssetApiGetAllAssets
+     */
+    readonly skip?: number
+
+    /**
+     * 
+     * @type {number}
+     * @memberof AssetApiGetAllAssets
+     */
+    readonly take?: number
+
     /**
      * 
      * @type {string}
@@ -8742,17 +8770,17 @@ export interface AssetApiGetAllAssetsRequest {
 
     /**
      * 
-     * @type {number}
+     * @type {string}
      * @memberof AssetApiGetAllAssets
      */
-    readonly skip?: number
+    readonly updatedAfter?: string
 
     /**
      * 
      * @type {string}
      * @memberof AssetApiGetAllAssets
      */
-    readonly updatedAfter?: string
+    readonly updatedBefore?: string
 
     /**
      * ETag of data already cached on the client
@@ -9430,7 +9458,7 @@ export class AssetApi extends BaseAPI {
      * @memberof AssetApi
      */
     public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) {
-        return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
+        return AssetApiFp(this.configuration).getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
     }
 
     /**

mobile/lib/shared/services/asset.service.dart

@@ -62,20 +62,31 @@ class AssetService {
 
   /// Returns `null` if the server state did not change, else list of assets
   Future<List<Asset>?> _getRemoteAssets(User user) async {
+    const int chunkSize = 5000;
     try {
-      final List<AssetResponseDto>? assets =
+      final DateTime now = DateTime.now().toUtc();
-          await _apiService.assetApi.getAllAssets(
+      final List<Asset> allAssets = [];
-        userId: user.id,
+      for (int i = 0;; i += chunkSize) {
-      );
+        final List<AssetResponseDto>? assets =
-      if (assets == null) {
+            await _apiService.assetApi.getAllAssets(
-        return null;
+          userId: user.id,
-      } else if (assets.isNotEmpty && assets.first.ownerId != user.id) {
+          // updatedBefore is important! without it we could
-        log.warning("Make sure that server and app versions match!"
+          // a) get the same Asset multiple times in different versions (when
-            " The server returned assets for user ${assets.first.ownerId}"
+          // the asset is modified while the chunks are loaded from the server)
-            " while requesting assets of user ${user.id}");
+          // b) miss assets when new assets are inserted in between the calls
-        return null;
+          updatedBefore: now,
+          skip: i,
+          take: chunkSize,
+        );
+        if (assets == null) {
+          return null;
+        }
+        allAssets.addAll(assets.map(Asset.remote));
+        if (assets.length < chunkSize) {
+          break;
+        }
       }
-      return assets.map(Asset.remote).toList();
+      return allAssets;
     } catch (error, stack) {
       log.severe(
         'Error while getting remote assets: ${error.toString()}',

mobile/lib/shared/services/sync.service.dart

@@ -197,7 +197,7 @@ class SyncService {
     User user,
     FutureOr<List<Asset>?> Function(User user) loadAssets,
   ) async {
-    final DateTime now = DateTime.now();
+    final DateTime now = DateTime.now().toUtc();
     final List<Asset>? remote = await loadAssets(user);
     if (remote == null) {
       return false;
@@ -210,6 +210,10 @@ class SyncService {
     assert(inDb.isSorted(Asset.compareByChecksum), "inDb not sorted!");
 
     remote.sort(Asset.compareByChecksum);
+
+    // filter our duplicates that might be introduced by the chunked retrieval
+    remote.uniqueConsecutive(compare: Asset.compareByChecksum);
+
     final (toAdd, toUpdate, toRemove) = _diffAssets(remote, inDb, remote: true);
     if (toAdd.isEmpty && toUpdate.isEmpty && toRemove.isEmpty) {
       await _updateUserAssetsETag(user, now);
@@ -759,6 +763,12 @@ class SyncService {
   final List<Asset> toAdd = [];
   final List<Asset> toUpdate = [];
   final List<Asset> toRemove = [];
+  if (assets.isEmpty || inDb.isEmpty) {
+    // fast path for trivial cases: halfes memory usage during initial sync
+    return assets.isEmpty
+        ? (toAdd, toUpdate, inDb) // remove all from DB
+        : (assets, toUpdate, toRemove); // add all assets
+  }
   diffSortedListsSync(
     inDb,
     assets,

server/immich-openapi-specs.json

@@ -914,6 +914,22 @@
         "description": "Get all AssetEntity belong to the user",
         "operationId": "getAllAssets",
         "parameters": [
+          {
+            "name": "skip",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "integer"
+            }
+          },
+          {
+            "name": "take",
+            "required": false,
+            "in": "query",
+            "schema": {
+              "type": "integer"
+            }
+          },
           {
             "name": "userId",
             "required": false,
@@ -940,15 +956,16 @@
             }
           },
           {
-            "name": "skip",
+            "name": "updatedAfter",
             "required": false,
             "in": "query",
             "schema": {
-              "type": "number"
+              "format": "date-time",
+              "type": "string"
             }
           },
           {
-            "name": "updatedAfter",
+            "name": "updatedBefore",
             "required": false,
             "in": "query",
             "schema": {

server/src/immich/api-v1/asset/asset-repository.ts

@@ -1,8 +1,8 @@
 import { AssetCreate } from '@app/domain';
 import { AssetEntity } from '@app/infra/entities';
+import OptionalBetween from '@app/infra/utils/optional-between.util';
 import { Injectable } from '@nestjs/common';
 import { InjectRepository } from '@nestjs/typeorm';
-import { MoreThan } from 'typeorm';
 import { In } from 'typeorm/find-options/operator/In';
 import { Repository } from 'typeorm/repository/Repository';
 import { AssetSearchDto } from './dto/asset-search.dto';
@@ -129,7 +129,7 @@ export class AssetRepository implements IAssetRepository {
         isVisible: true,
         isFavorite: dto.isFavorite,
         isArchived: dto.isArchived,
-        updatedAt: dto.updatedAfter ? MoreThan(dto.updatedAfter) : undefined,
+        updatedAt: OptionalBetween(dto.updatedAfter, dto.updatedBefore),
       },
       relations: {
         exifInfo: true,
@@ -137,6 +137,7 @@ export class AssetRepository implements IAssetRepository {
         stack: true,
       },
       skip: dto.skip || 0,
+      take: dto.take,
       order: {
         fileCreatedAt: 'DESC',
       },

server/src/immich/api-v1/asset/dto/asset-search.dto.ts

@@ -1,7 +1,7 @@
 import { Optional, toBoolean } from '@app/domain';
 import { ApiProperty } from '@nestjs/swagger';
 import { Transform, Type } from 'class-transformer';
-import { IsBoolean, IsDate, IsNotEmpty, IsNumber, IsUUID } from 'class-validator';
+import { IsBoolean, IsDate, IsInt, IsNotEmpty, IsUUID } from 'class-validator';
 
 export class AssetSearchDto {
   @Optional()
@@ -17,9 +17,17 @@ export class AssetSearchDto {
   isArchived?: boolean;
 
   @Optional()
-  @IsNumber()
+  @IsInt()
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
   skip?: number;
 
+  @Optional()
+  @IsInt()
+  @Type(() => Number)
+  @ApiProperty({ type: 'integer' })
+  take?: number;
+
   @Optional()
   @IsUUID('4')
   @ApiProperty({ format: 'uuid' })
@@ -29,4 +37,9 @@ export class AssetSearchDto {
   @IsDate()
   @Type(() => Date)
   updatedAfter?: Date;
+
+  @Optional()
+  @IsDate()
+  @Type(() => Date)
+  updatedBefore?: Date;
 }

web/src/api/open-api/api.ts

@@ -6695,16 +6695,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
         },
         /**
          * Get all AssetEntity belong to the user
+         * @param {number} [skip] 
+         * @param {number} [take] 
          * @param {string} [userId] 
          * @param {boolean} [isFavorite] 
          * @param {boolean} [isArchived] 
-         * @param {number} [skip] 
          * @param {string} [updatedAfter] 
+         * @param {string} [updatedBefore] 
          * @param {string} [ifNoneMatch] ETag of data already cached on the client
          * @param {*} [options] Override http request option.
          * @throws {RequiredError}
          */
-        getAllAssets: async (userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
+        getAllAssets: async (skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options: AxiosRequestConfig = {}): Promise<RequestArgs> => {
             const localVarPath = `/asset`;
             // use dummy base URL string because the URL constructor only accepts absolute URLs.
             const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL);
@@ -6726,6 +6728,14 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
             // http bearer authentication required
             await setBearerAuthToObject(localVarHeaderParameter, configuration)
 
+            if (skip !== undefined) {
+                localVarQueryParameter['skip'] = skip;
+            }
+
+            if (take !== undefined) {
+                localVarQueryParameter['take'] = take;
+            }
+
             if (userId !== undefined) {
                 localVarQueryParameter['userId'] = userId;
             }
@@ -6738,16 +6748,18 @@ export const AssetApiAxiosParamCreator = function (configuration?: Configuration
                 localVarQueryParameter['isArchived'] = isArchived;
             }
 
-            if (skip !== undefined) {
-                localVarQueryParameter['skip'] = skip;
-            }
-
             if (updatedAfter !== undefined) {
                 localVarQueryParameter['updatedAfter'] = (updatedAfter as any instanceof Date) ?
                     (updatedAfter as any).toISOString() :
                     updatedAfter;
             }
 
+            if (updatedBefore !== undefined) {
+                localVarQueryParameter['updatedBefore'] = (updatedBefore as any instanceof Date) ?
+                    (updatedBefore as any).toISOString() :
+                    updatedBefore;
+            }
+
             if (ifNoneMatch != null) {
                 localVarHeaderParameter['if-none-match'] = String(ifNoneMatch);
             }
@@ -8066,17 +8078,19 @@ export const AssetApiFp = function(configuration?: Configuration) {
         },
         /**
          * Get all AssetEntity belong to the user
+         * @param {number} [skip] 
+         * @param {number} [take] 
          * @param {string} [userId] 
          * @param {boolean} [isFavorite] 
          * @param {boolean} [isArchived] 
-         * @param {number} [skip] 
          * @param {string} [updatedAfter] 
+         * @param {string} [updatedBefore] 
          * @param {string} [ifNoneMatch] ETag of data already cached on the client
          * @param {*} [options] Override http request option.
          * @throws {RequiredError}
          */
-        async getAllAssets(userId?: string, isFavorite?: boolean, isArchived?: boolean, skip?: number, updatedAfter?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
+        async getAllAssets(skip?: number, take?: number, userId?: string, isFavorite?: boolean, isArchived?: boolean, updatedAfter?: string, updatedBefore?: string, ifNoneMatch?: string, options?: AxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise<Array<AssetResponseDto>>> {
-            const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(userId, isFavorite, isArchived, skip, updatedAfter, ifNoneMatch, options);
+            const localVarAxiosArgs = await localVarAxiosParamCreator.getAllAssets(skip, take, userId, isFavorite, isArchived, updatedAfter, updatedBefore, ifNoneMatch, options);
             return createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration);
         },
         /**
@@ -8421,7 +8435,7 @@ export const AssetApiFactory = function (configuration?: Configuration, basePath
          * @throws {RequiredError}
          */
         getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig): AxiosPromise<Array<AssetResponseDto>> {
-            return localVarFp.getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
+            return localVarFp.getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(axios, basePath));
         },
         /**
          * Get a single asset\'s information
@@ -8719,6 +8733,20 @@ export interface AssetApiDownloadFileRequest {
  * @interface AssetApiGetAllAssetsRequest
  */
 export interface AssetApiGetAllAssetsRequest {
+    /**
+     * 
+     * @type {number}
+     * @memberof AssetApiGetAllAssets
+     */
+    readonly skip?: number
+
+    /**
+     * 
+     * @type {number}
+     * @memberof AssetApiGetAllAssets
+     */
+    readonly take?: number
+
     /**
      * 
      * @type {string}
@@ -8742,17 +8770,17 @@ export interface AssetApiGetAllAssetsRequest {
 
     /**
      * 
-     * @type {number}
+     * @type {string}
      * @memberof AssetApiGetAllAssets
      */
-    readonly skip?: number
+    readonly updatedAfter?: string
 
     /**
      * 
      * @type {string}
      * @memberof AssetApiGetAllAssets
      */
-    readonly updatedAfter?: string
+    readonly updatedBefore?: string
 
     /**
      * ETag of data already cached on the client
@@ -9430,7 +9458,7 @@ export class AssetApi extends BaseAPI {
      * @memberof AssetApi
      */
     public getAllAssets(requestParameters: AssetApiGetAllAssetsRequest = {}, options?: AxiosRequestConfig) {
-        return AssetApiFp(this.configuration).getAllAssets(requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.skip, requestParameters.updatedAfter, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
+        return AssetApiFp(this.configuration).getAllAssets(requestParameters.skip, requestParameters.take, requestParameters.userId, requestParameters.isFavorite, requestParameters.isArchived, requestParameters.updatedAfter, requestParameters.updatedBefore, requestParameters.ifNoneMatch, options).then((request) => request(this.axios, this.basePath));
     }
 
     /**