From 37a79292c00a26fb47ece6b693655106a7fc3c8a Mon Sep 17 00:00:00 2001 From: Arthur Normand Date: Thu, 4 Sep 2025 10:22:09 -0400 Subject: [PATCH] feat: view similar photos (#21108) * Enable filteing by example * Drop `@GenerateSql` for `getEmbedding`? * Improve error message * PR Feedback * Sort en.json * Add SQL * Fix lint * Drop test that is no longer valid * Fix i18n file sorting * Fix TS error * Add a `requireAccess` before pulling the embedding * Fix decorators * Run `make open-api` --------- Co-authored-by: Alex --- i18n/en.json | 2 ++ .../openapi/lib/model/smart_search_dto.dart | Bin 22104 -> 23269 bytes open-api/immich-openapi-specs.json | 7 ++-- open-api/typescript-sdk/src/fetch-client.ts | 3 +- .../src/controllers/search.controller.spec.ts | 6 ---- server/src/dtos/search.dto.ts | 7 +++- server/src/queries/search.repository.sql | 8 +++++ server/src/repositories/search.repository.ts | 7 ++++ server/src/services/search.service.ts | 31 +++++++++++++----- .../asset-viewer/asset-viewer-nav-bar.svelte | 10 ++++++ web/src/lib/modals/SearchFilterModal.svelte | 10 +++++- .../[[assetId=id]]/+page.svelte | 5 +-- 12 files changed, 73 insertions(+), 23 deletions(-) diff --git a/i18n/en.json b/i18n/en.json index 82748c775..9e7b6fae6 100644 --- a/i18n/en.json +++ b/i18n/en.json @@ -1557,6 +1557,7 @@ "purchase_server_description_2": "Supporter status", "purchase_server_title": "Server", "purchase_settings_server_activated": "The server product key is managed by the admin", + "query_asset_id": "Query Asset ID", "queue_status": "Queuing {count}/{total}", "rating": "Star rating", "rating_clear": "Clear rating", @@ -2077,6 +2078,7 @@ "view_next_asset": "View next asset", "view_previous_asset": "View previous asset", "view_qr_code": "View QR code", + "view_similar_photos": "View similar photos", "view_stack": "View Stack", "view_user": "View User", "viewer_remove_from_stack": "Remove from Stack", diff --git a/mobile/openapi/lib/model/smart_search_dto.dart b/mobile/openapi/lib/model/smart_search_dto.dart index 0d16b56d74109143caff842882143cbbae6243b0..90902b979177749c30d462356376a0e1f3b8622f 100644 GIT binary patch delta 388 zcmcbyhVkiE#trY76iYHPi}eaiQ;RBfxDM!G8erzV(%j7|3KGpL&B;-)S1?ep zf+$d&+@~+D2vvqn0YqwYqpry2?fOkDQpncWsvz?=o11WQ@nohcXu$k4Il)YNvZtNY z+C)&z!Av8@6w2@Xqa6MDhV-;*MgJW``r5rbsvdMy$@|$z)a{0s{;Rgys dB { await request(ctx.getHttpServer()).post('/search/smart'); expect(ctx.authenticate).toHaveBeenCalled(); }); - - it('should require a query', async () => { - const { status, body } = await request(ctx.getHttpServer()).post('/search/smart').send({}); - expect(status).toBe(400); - expect(body).toEqual(errorDto.badRequest(['query should not be empty', 'query must be a string'])); - }); }); describe('GET /search/explore', () => { diff --git a/server/src/dtos/search.dto.ts b/server/src/dtos/search.dto.ts index f709ad94a..ac7bf4feb 100644 --- a/server/src/dtos/search.dto.ts +++ b/server/src/dtos/search.dto.ts @@ -199,7 +199,12 @@ export class StatisticsSearchDto extends BaseSearchDto { export class SmartSearchDto extends BaseSearchWithResultsDto { @IsString() @IsNotEmpty() - query!: string; + @Optional() + query?: string; + + @ValidateUUID({ optional: true }) + @Optional() + queryAssetId?: string; @IsString() @IsNotEmpty() diff --git a/server/src/queries/search.repository.sql b/server/src/queries/search.repository.sql index be2245a74..e0aaedfdf 100644 --- a/server/src/queries/search.repository.sql +++ b/server/src/queries/search.repository.sql @@ -123,6 +123,14 @@ offset $8 commit +-- SearchRepository.getEmbedding +select + * +from + "smart_search" +where + "assetId" = $1 + -- SearchRepository.searchFaces begin set diff --git a/server/src/repositories/search.repository.ts b/server/src/repositories/search.repository.ts index 36ef7a27f..88de2fb06 100644 --- a/server/src/repositories/search.repository.ts +++ b/server/src/repositories/search.repository.ts @@ -293,6 +293,13 @@ export class SearchRepository { }); } + @GenerateSql({ + params: [DummyValue.UUID], + }) + async getEmbedding(assetId: string) { + return this.db.selectFrom('smart_search').selectAll().where('assetId', '=', assetId).executeTakeFirst(); + } + @GenerateSql({ params: [ { diff --git a/server/src/services/search.service.ts b/server/src/services/search.service.ts index b9391fed9..51a2c9433 100644 --- a/server/src/services/search.service.ts +++ b/server/src/services/search.service.ts @@ -18,7 +18,7 @@ import { SmartSearchDto, StatisticsSearchDto, } from 'src/dtos/search.dto'; -import { AssetOrder, AssetVisibility } from 'src/enum'; +import { AssetOrder, AssetVisibility, Permission } from 'src/enum'; import { BaseService } from 'src/services/base.service'; import { requireElevatedPermission } from 'src/utils/access'; import { getMyPartnerIds } from 'src/utils/asset.util'; @@ -113,14 +113,27 @@ export class SearchService extends BaseService { } const userIds = this.getUserIdsToSearch(auth); - const key = machineLearning.clip.modelName + dto.query + dto.language; - let embedding = this.embeddingCache.get(key); - if (!embedding) { - embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, { - modelName: machineLearning.clip.modelName, - language: dto.language, - }); - this.embeddingCache.set(key, embedding); + let embedding; + if (dto.query) { + const key = machineLearning.clip.modelName + dto.query + dto.language; + embedding = this.embeddingCache.get(key); + if (!embedding) { + embedding = await this.machineLearningRepository.encodeText(machineLearning.urls, dto.query, { + modelName: machineLearning.clip.modelName, + language: dto.language, + }); + this.embeddingCache.set(key, embedding); + } + } else if (dto.queryAssetId) { + await this.requireAccess({ auth, permission: Permission.AssetRead, ids: [dto.queryAssetId] }); + const getEmbeddingResponse = await this.searchRepository.getEmbedding(dto.queryAssetId); + const assetEmbedding = getEmbeddingResponse?.embedding; + if (!assetEmbedding) { + throw new BadRequestException(`Asset ${dto.queryAssetId} has no embedding`); + } + embedding = assetEmbedding; + } else { + throw new BadRequestException('Either `query` or `queryAssetId` must be set'); } const page = dto.page ?? 1; const size = dto.size || 100; diff --git a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte index 66061ebb0..dd197f3a9 100644 --- a/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte +++ b/web/src/lib/components/asset-viewer/asset-viewer-nav-bar.svelte @@ -22,6 +22,7 @@ import ButtonContextMenu from '$lib/components/shared-components/context-menu/button-context-menu.svelte'; import MenuOption from '$lib/components/shared-components/context-menu/menu-option.svelte'; import { AppRoute } from '$lib/constants'; + import { featureFlags } from '$lib/stores/server-config.store'; import { user } from '$lib/stores/user.store'; import { photoZoomState } from '$lib/stores/zoom-image.store'; import { getAssetJobName, getSharedLink } from '$lib/utils'; @@ -41,6 +42,7 @@ import { mdiAlertOutline, mdiCogRefreshOutline, + mdiCompare, mdiContentCopy, mdiDatabaseRefreshOutline, mdiDotsVertical, @@ -98,6 +100,7 @@ let isOwner = $derived($user && asset.ownerId === $user?.id); let showDownloadButton = $derived(sharedLink ? sharedLink.allowDownload : !asset.isOffline); let isLocked = $derived(asset.visibility === AssetVisibility.Locked); + let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch); // $: showEditorButton = // isOwner && @@ -225,6 +228,13 @@ text={$t('view_in_timeline')} /> {/if} + {#if !asset.isArchived && !asset.isTrashed && smartSearchEnabled} + goto(`${AppRoute.SEARCH}?query={"queryAssetId":"${stack?.primaryAssetId ?? asset.id}"}`)} + text={$t('view_similar_photos')} + /> + {/if} {/if} {#if !asset.isTrashed} diff --git a/web/src/lib/modals/SearchFilterModal.svelte b/web/src/lib/modals/SearchFilterModal.svelte index 0647d0bb7..b9841c311 100644 --- a/web/src/lib/modals/SearchFilterModal.svelte +++ b/web/src/lib/modals/SearchFilterModal.svelte @@ -64,8 +64,16 @@ return validQueryTypes.has(storedQueryType) ? storedQueryType : QueryType.SMART; } + let query = ''; + if ('query' in searchQuery && searchQuery.query) { + query = searchQuery.query; + } + if ('originalFileName' in searchQuery && searchQuery.originalFileName) { + query = searchQuery.originalFileName; + } + let filter: SearchFilter = $state({ - query: 'query' in searchQuery ? searchQuery.query : searchQuery.originalFileName || '', + query, queryType: defaultQueryType(), personIds: new SvelteSet('personIds' in searchQuery ? searchQuery.personIds : []), tagIds: diff --git a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte index 56e58c363..512d8c548 100644 --- a/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte +++ b/web/src/routes/(user)/search/[[photos=photos]]/[[assetId=id]]/+page.svelte @@ -68,7 +68,7 @@ const assetInteraction = new AssetInteraction(); - type SearchTerms = MetadataSearchDto & Pick; + type SearchTerms = MetadataSearchDto & Pick; let searchQuery = $derived(page.url.searchParams.get(QueryParameter.QUERY)); let smartSearchEnabled = $derived($featureFlags.loaded && $featureFlags.smartSearch); let terms = $derived(searchQuery ? JSON.parse(searchQuery) : {}); @@ -164,7 +164,7 @@ try { const { albums, assets } = - 'query' in searchDto && smartSearchEnabled + ('query' in searchDto || 'queryAssetId' in searchDto) && smartSearchEnabled ? await searchSmart({ smartSearchDto: searchDto }) : await searchAssets({ metadataSearchDto: searchDto }); @@ -210,6 +210,7 @@ tagIds: $t('tags'), originalFileName: $t('file_name'), description: $t('description'), + queryAssetId: $t('query_asset_id'), }; return keyMap[key] || key; }