From a2934b88301d350489d390e52be85dc2c489e006 Mon Sep 17 00:00:00 2001 From: martin <74269598+martabal@users.noreply.github.com> Date: Sat, 24 Feb 2024 01:42:37 +0100 Subject: [PATCH] feat(server, web): search location (#7139) * feat: search location * fix: tests * feat: outclick * location search index * update query * fixed query * updated sql * update query * Update search.dto.ts Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> * coalesce * fix: tests * feat: add alternate names * fix: generate sql files * single table, add alternate names to query, cleanup * merge main * update sql * pr feedback * pr feedback * chore: fix merge --------- Co-authored-by: mertalev <101130780+mertalev@users.noreply.github.com> Co-authored-by: Daniel Dietzler <36593685+danieldietzler@users.noreply.github.com> --- mobile/openapi/.openapi-generator/FILES | 3 + mobile/openapi/README.md | Bin 24816 -> 24959 bytes mobile/openapi/doc/PlacesResponseDto.md | Bin 0 -> 560 bytes mobile/openapi/doc/SearchApi.md | Bin 13525 -> 15521 bytes mobile/openapi/lib/api.dart | Bin 8531 -> 8570 bytes mobile/openapi/lib/api/search_api.dart | Bin 12967 -> 14647 bytes mobile/openapi/lib/api_client.dart | Bin 24008 -> 24094 bytes .../lib/model/places_response_dto.dart | Bin 0 -> 4683 bytes .../test/places_response_dto_test.dart | Bin 0 -> 975 bytes mobile/openapi/test/search_api_test.dart | Bin 1490 -> 1624 bytes open-api/immich-openapi-specs.json | 69 ++++++++ open-api/typescript-sdk/axios-client/api.ts | 128 +++++++++++++++ open-api/typescript-sdk/fetch-client.ts | Bin 74500 -> 74942 bytes server/src/domain/domain.constant.ts | 2 +- .../domain/repositories/search.repository.ts | 3 +- server/src/domain/search/dto/search.dto.ts | 26 ++- server/src/domain/search/search.service.ts | 15 +- .../immich/controllers/search.controller.ts | 7 + .../infra/entities/geodata-admin1.entity.ts | 10 -- .../infra/entities/geodata-admin2.entity.ts | 10 -- .../infra/entities/geodata-places.entity.ts | 31 +--- server/src/infra/entities/index.ts | 6 - .../1708059341865-GeodataLocationSearch.ts | 152 ++++++++++++++++++ .../1708116312820-GeonamesEnhancement.ts | 18 +++ .../infra/repositories/metadata.repository.ts | 109 ++++++------- .../infra/repositories/search.repository.ts | 30 +++- server/src/infra/sql/search.repository.sql | 34 ++++ .../repositories/search.repository.mock.ts | 1 + .../lib/components/elements/search-bar.svelte | 7 +- .../shared-components/change-location.svelte | 134 ++++++++++++++- .../shared-components/map/map.svelte | 11 ++ 31 files changed, 689 insertions(+), 117 deletions(-) create mode 100644 mobile/openapi/doc/PlacesResponseDto.md create mode 100644 mobile/openapi/lib/model/places_response_dto.dart create mode 100644 mobile/openapi/test/places_response_dto_test.dart delete mode 100644 server/src/infra/entities/geodata-admin1.entity.ts delete mode 100644 server/src/infra/entities/geodata-admin2.entity.ts create mode 100644 server/src/infra/migrations/1708059341865-GeodataLocationSearch.ts create mode 100644 server/src/infra/migrations/1708116312820-GeonamesEnhancement.ts diff --git a/mobile/openapi/.openapi-generator/FILES b/mobile/openapi/.openapi-generator/FILES index 0679a1749..ea413b487 100644 --- a/mobile/openapi/.openapi-generator/FILES +++ b/mobile/openapi/.openapi-generator/FILES @@ -108,6 +108,7 @@ doc/PersonResponseDto.md doc/PersonStatisticsResponseDto.md doc/PersonUpdateDto.md doc/PersonWithFacesResponseDto.md +doc/PlacesResponseDto.md doc/QueueStatusDto.md doc/ReactionLevel.md doc/ReactionType.md @@ -308,6 +309,7 @@ lib/model/person_response_dto.dart lib/model/person_statistics_response_dto.dart lib/model/person_update_dto.dart lib/model/person_with_faces_response_dto.dart +lib/model/places_response_dto.dart lib/model/queue_status_dto.dart lib/model/reaction_level.dart lib/model/reaction_type.dart @@ -485,6 +487,7 @@ test/person_response_dto_test.dart test/person_statistics_response_dto_test.dart test/person_update_dto_test.dart test/person_with_faces_response_dto_test.dart +test/places_response_dto_test.dart test/queue_status_dto_test.dart test/reaction_level_test.dart test/reaction_type_test.dart diff --git a/mobile/openapi/README.md b/mobile/openapi/README.md index 5dd6d196d2ab2c3911d1ce49697245d2f7eb2438..41e65ee8b358fb3ac205c3e928641da2e94f90d2 100644 GIT binary patch delta 100 zcmexxkn#T^#tj_0Yymlm$*IMYH|ogBfmxagH40iTh9l` diff --git a/mobile/openapi/doc/PlacesResponseDto.md b/mobile/openapi/doc/PlacesResponseDto.md new file mode 100644 index 0000000000000000000000000000000000000000..a4bf36493c9faf97c1af5d98719b3b656acd07cf GIT binary patch literal 560 zcma)(F>Avx5QTUBiUSX61afG{c1m!DLK95d%@`~!)fJU=Lb?ov{`ksnOw&SWw1{{2 z?t4O400!-4Fr`DKZCP8{>vYg5pijCYWUMIQ38eyh(=eGJ$o^dpnsr^LWF%%>z(}w- z)$dQp)rcJpi-*xVk?NS6k&I4kFP(-@c=-t9Rq5166Ud;?YLAp1;_T$&nG)hZ%{Z6J z8Gq|@4CmR-`7=f1he>4J;FKn93$yv|O0$(@=%{9{Ke`#3Nf!NZxqC5zv8LJNj|D>5 zjk^GA61DA3S*{M-b+y};P8ACKqtl-91JH|CjAB|zTEfrtx_-QJHZMqMuL2rRzYyPn Jf5MlHu`m2+t9bwb literal 0 HcmV?d00001 diff --git a/mobile/openapi/doc/SearchApi.md b/mobile/openapi/doc/SearchApi.md index f975e94484c4b5e65a7f8b36a32a4f9a19767557..f63488222b156a7190c45a9548e5bad140fd3948 100644 GIT binary patch delta 222 zcmcbbxv+9WER$_OPGWLuv6fb>MsR9kQF4Z3L8e}AigGcSQvjCKRH#wV(sFkV(b7`T zhe+r{WHtvd9oI5}8Np?z;FDQgVgum^r4|?D=M|^Al;qnfKy*PQH1ZO2Q#Uuts<2F+ zq^UHShhKg21w%G&B+1FTO5tn}{^SD5*2(|0BqtY3iz?zU9preZYHQ<7o3(USa{vGq C{!ViM delta 17 ZcmZ2jc{OuGEYs%YOuMu;KQ`IN4ggLT2sZ!# diff --git a/mobile/openapi/lib/api.dart b/mobile/openapi/lib/api.dart index 72a6567648de8e33f322882d8370930c6042fac4..56bd907e0ab8f58dd1c7d9ca88c086968e451a97 100644 GIT binary patch delta 23 ecmccY^vh{Or7&ATPGWLu@nlC1#m$w%w|D_~0|_qx delta 12 Tcmez6blGV`rSRqj!nb$Q2Wn>Pn2-DjNar(F$o^EqvG L2y62%?X_$GZP!oz delta 12 Tcmdm9v^;e~g3jg-M(%6?DJlh& diff --git a/mobile/openapi/lib/api_client.dart b/mobile/openapi/lib/api_client.dart index 2df5e67119073f18eec7e21d7da21905bab6a2d1..24cffb7cff2290e7241d841674680215f48a1467 100644 GIT binary patch delta 26 icmX@Hn{nPA#syZa0Xd1usgoU56qo{XHea;-t_%Q;BnnXg delta 13 VcmbQYhw;R2#syZJ!>mpy0{|>d1?T_( diff --git a/mobile/openapi/lib/model/places_response_dto.dart b/mobile/openapi/lib/model/places_response_dto.dart new file mode 100644 index 0000000000000000000000000000000000000000..a2d8378883dd36e213ee0ae18886df198bf6bfc8 GIT binary patch literal 4683 zcmeHL-EZ4A5P$byaVdh@!BnU1)8Ni-izXe~weiqq0|p}y7>SP6%A!kBHH_5%`+i4? zl59C`wue3Kp)N(<9q;3JAI#}#@AQ;j{&6{d^y~S>`G=R6=V$c#&Aao6&ZcxRy`=Zk zvp28*IszF}zDR|(Cj&vH4UvM7yAW0}jPENm3Mv*k)@dyqrq zYq2r$s5_y%0ymJ9cyMvOFuKiE~xl zfXpsL5&!%+P8LF&UJutCTwLe! zh(2aTBB7QfvyurSkw7_B5CfwUo+@crG<=feP#a!L(^uTn>jj0NjJ?OFnhLvo1k|2}uQ^I!SgrH}J@ zQ7$RuOtTRZ#qIcL5hsZIexZOjjnE>J*GlDF9{@U~X*?cGuRqBIRRgHkCp4aruR@8) z2MmNLoP7J{R|If$Y5+Za2-)N*T2%lSv@mmk%Gp&@`L7*W!BnQH*Si$Kc1Kcw?9i|) z`1qhpvcl4hw8$>Ac*Th~o;YLb=DunQ>%;CMnz%NKJZ9k7?Ah z{Ad`2pU_V&uG)M~=r)%-IKUNL(3Qs>!PUy{5nJ8tb|6u~#VWrTI280dF`hMcj*pvl zxQR=;>{B1$(n$dU=_(_5nosOKBb zJS#*_pRh$^>QLWqpE*Bgb8R?l-!X@3~XL0kZA{IpgJZ~(fW0FSknfPkYBJii>GSQ>Ib9&#Qzd!TydANEm$-A<|EV{ZrwYw8i+;`H7PUf%I_#bp)SH3-fcQCK-C z?Lf49LDyr9@`K?B_It;Gs@)~Ja@zeOcpf)@*YS>S>;rRG>Y_L`)h4_KV?4<4?&SBS zpCNXFh+Vx2VI1;TCQUGh9q&!LM>w6Np(AVs=Lcbht7e|@v)phWbo?HdeEWA|3)WVJ z=g@E=c5YIB%v@)7fh|rVeyDch@W|7Ftt#^TD!~(rAGQ$J6yo=-34xJxAeko2!Dk2Y z5z+n96*Ah$x$CZfgP(gs<32E}8bGTQ{18PjY&XuGU<^4-wewnFit21_@;0gw<^i&k z^th`9j8nYIWsQOQ$q%?03|#_|=9llakYidw1}GXH*OT_A^-pY literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/places_response_dto_test.dart b/mobile/openapi/test/places_response_dto_test.dart new file mode 100644 index 0000000000000000000000000000000000000000..5a320fce643b3974ba2a28d7039ecf525d5a1e95 GIT binary patch literal 975 zcmb7?K~KU!5QXpk72^p^kP0V*F_ENTNR%4HgQs=U4rODzyY6l^#Q5KxT_heL5f0sU zlJC7Yv(q$9(iFze%lzVQGM_BQ%Si^)+5MykSq}4j2@iQTo4%Ye7RcKYt-l%e2g4+k zvQpZTSZOO(vZfKVs&!Nfs;MF6d%IdTMq7VSf%PYDoG`5ymfl-wS+20*)1c+IC+pbU zy2J89E9X&pAxjRFD@iuW-NCS0DsrOMs$$HqQHiTdQLNFrB-!=|9fQRiRy5Lv6dIZG z(+^X3uJ;g6&Zptn0Uu@tjZ}kho2-Y}85`*M0e($jEKh0u244Tr z*R2=2jVfZ3LCTFT^DK`x-`jrnf`-QjMf9W~)izM0m%c6kA7%%hp1^;`kr6xcq>SIa R9Cc9vI?kN85_j^Q`~WOvFS7su literal 0 HcmV?d00001 diff --git a/mobile/openapi/test/search_api_test.dart b/mobile/openapi/test/search_api_test.dart index 14169e461d283d630fe94bd66cc15a1bc40069be..aa4a94847b4eeba806332fd499bf9af0cb16b631 100644 GIT binary patch delta 80 zcmcb_eS>F%JFB5jW^sv4Ku%(EYH?6%aY24wajHv6zMY*yacW{wat1_3Be => { + // verify required parameter 'name' is not null or undefined + assertParamExists('searchPlaces', 'name', name) + const localVarPath = `/search/places`; + // use dummy base URL string because the URL constructor only accepts absolute URLs. + const localVarUrlObj = new URL(localVarPath, DUMMY_BASE_URL); + let baseOptions; + if (configuration) { + baseOptions = configuration.baseOptions; + } + + const localVarRequestOptions = { method: 'GET', ...baseOptions, ...options}; + const localVarHeaderParameter = {} as any; + const localVarQueryParameter = {} as any; + + // authentication cookie required + + // authentication api_key required + await setApiKeyToObject(localVarHeaderParameter, "x-api-key", configuration) + + // authentication bearer required + // http bearer authentication required + await setBearerAuthToObject(localVarHeaderParameter, configuration) + + if (name !== undefined) { + localVarQueryParameter['name'] = name; + } + + + setSearchParams(localVarUrlObj, localVarQueryParameter); let headersFromBaseOptions = baseOptions && baseOptions.headers ? baseOptions.headers : {}; localVarRequestOptions.headers = {...localVarHeaderParameter, ...headersFromBaseOptions, ...options.headers}; @@ -15584,6 +15666,18 @@ export const SearchApiFp = function(configuration?: Configuration) { const operationBasePath = operationServerMap['SearchApi.searchPerson']?.[index]?.url; return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); }, + /** + * + * @param {string} name + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + async searchPlaces(name: string, options?: RawAxiosRequestConfig): Promise<(axios?: AxiosInstance, basePath?: string) => AxiosPromise>> { + const localVarAxiosArgs = await localVarAxiosParamCreator.searchPlaces(name, options); + const index = configuration?.serverIndex ?? 0; + const operationBasePath = operationServerMap['SearchApi.searchPlaces']?.[index]?.url; + return (axios, basePath) => createRequestFunction(localVarAxiosArgs, globalAxios, BASE_PATH, configuration)(axios, operationBasePath || basePath); + }, /** * * @param {SmartSearchDto} smartSearchDto @@ -15651,6 +15745,15 @@ export const SearchApiFactory = function (configuration?: Configuration, basePat searchPerson(requestParameters: SearchApiSearchPersonRequest, options?: RawAxiosRequestConfig): AxiosPromise> { return localVarFp.searchPerson(requestParameters.name, requestParameters.withHidden, options).then((request) => request(axios, basePath)); }, + /** + * + * @param {SearchApiSearchPlacesRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + */ + searchPlaces(requestParameters: SearchApiSearchPlacesRequest, options?: RawAxiosRequestConfig): AxiosPromise> { + return localVarFp.searchPlaces(requestParameters.name, options).then((request) => request(axios, basePath)); + }, /** * * @param {SearchApiSearchSmartRequest} requestParameters Request parameters. @@ -15817,6 +15920,20 @@ export interface SearchApiSearchPersonRequest { readonly withHidden?: boolean } +/** + * Request parameters for searchPlaces operation in SearchApi. + * @export + * @interface SearchApiSearchPlacesRequest + */ +export interface SearchApiSearchPlacesRequest { + /** + * + * @type {string} + * @memberof SearchApiSearchPlaces + */ + readonly name: string +} + /** * Request parameters for searchSmart operation in SearchApi. * @export @@ -15893,6 +16010,17 @@ export class SearchApi extends BaseAPI { return SearchApiFp(this.configuration).searchPerson(requestParameters.name, requestParameters.withHidden, options).then((request) => request(this.axios, this.basePath)); } + /** + * + * @param {SearchApiSearchPlacesRequest} requestParameters Request parameters. + * @param {*} [options] Override http request option. + * @throws {RequiredError} + * @memberof SearchApi + */ + public searchPlaces(requestParameters: SearchApiSearchPlacesRequest, options?: RawAxiosRequestConfig) { + return SearchApiFp(this.configuration).searchPlaces(requestParameters.name, options).then((request) => request(this.axios, this.basePath)); + } + /** * * @param {SearchApiSearchSmartRequest} requestParameters Request parameters. diff --git a/open-api/typescript-sdk/fetch-client.ts b/open-api/typescript-sdk/fetch-client.ts index 0ee871ca60c8a76dd7269cb4beff284ad9561818..d023f8ef0a615f14485b1418349711360f8efe51 100644 GIT binary patch delta 151 zcmZoU#hLaUzgdt2L5R)S(u_UvkG$nQNI}vdq zD}}t$+@#bZYc2&K$V<#kot&sBxmnm&X(=z*pkj?`i13F864SYb851T8qzN(w&8Sy7O)`;7h7)De{|IW01BfuApigX delta 19 bcmdmYlBMMs%LZZF%`0t{mTrFk_`CxETzv`7 diff --git a/server/src/domain/domain.constant.ts b/server/src/domain/domain.constant.ts index 4e7c4d552..0dc9c5414 100644 --- a/server/src/domain/domain.constant.ts +++ b/server/src/domain/domain.constant.ts @@ -91,7 +91,7 @@ export const citiesFile = 'cities500.txt'; export const geodataDatePath = join(GEODATA_ROOT_PATH, 'geodata-date.txt'); export const geodataAdmin1Path = join(GEODATA_ROOT_PATH, 'admin1CodesASCII.txt'); export const geodataAdmin2Path = join(GEODATA_ROOT_PATH, 'admin2Codes.txt'); -export const geodataCitites500Path = join(GEODATA_ROOT_PATH, citiesFile); +export const geodataCities500Path = join(GEODATA_ROOT_PATH, citiesFile); const image: Record = { '.3fr': ['image/3fr', 'image/x-hasselblad-3fr'], diff --git a/server/src/domain/repositories/search.repository.ts b/server/src/domain/repositories/search.repository.ts index 7183e9e3f..8566fcd8e 100644 --- a/server/src/domain/repositories/search.repository.ts +++ b/server/src/domain/repositories/search.repository.ts @@ -1,4 +1,4 @@ -import { AssetEntity, AssetFaceEntity, AssetType, SmartInfoEntity } from '@app/infra/entities'; +import { AssetEntity, AssetFaceEntity, AssetType, GeodataPlacesEntity, SmartInfoEntity } from '@app/infra/entities'; import { Paginated } from '../domain.util'; export const ISearchRepository = 'ISearchRepository'; @@ -186,4 +186,5 @@ export interface ISearchRepository { searchSmart(pagination: SearchPaginationOptions, options: SmartSearchOptions): Paginated; searchFaces(search: FaceEmbeddingSearch): Promise; upsert(smartInfo: Partial, embedding?: Embedding): Promise; + searchPlaces(placeName: string): Promise; } diff --git a/server/src/domain/search/dto/search.dto.ts b/server/src/domain/search/dto/search.dto.ts index 4f2aa1819..877a494e4 100644 --- a/server/src/domain/search/dto/search.dto.ts +++ b/server/src/domain/search/dto/search.dto.ts @@ -1,5 +1,5 @@ import { AssetOrder } from '@app/domain/asset/dto/asset.dto'; -import { AssetType } from '@app/infra/entities'; +import { AssetType, GeodataPlacesEntity } from '@app/infra/entities'; import { ApiProperty } from '@nestjs/swagger'; import { Transform, Type } from 'class-transformer'; import { IsBoolean, IsEnum, IsInt, IsNotEmpty, IsString, Max, Min } from 'class-validator'; @@ -241,6 +241,12 @@ export class SearchDto { size?: number; } +export class SearchPlacesDto { + @IsString() + @IsNotEmpty() + name!: string; +} + export class SearchPeopleDto { @IsString() @IsNotEmpty() @@ -251,3 +257,21 @@ export class SearchPeopleDto { @Optional() withHidden?: boolean; } + +export class PlacesResponseDto { + name!: string; + latitude!: number; + longitude!: number; + admin1name?: string; + admin2name?: string; +} + +export function mapPlaces(place: GeodataPlacesEntity): PlacesResponseDto { + return { + name: place.name, + latitude: place.latitude, + longitude: place.longitude, + admin1name: place.admin1Name, + admin2name: place.admin2Name, + }; +} diff --git a/server/src/domain/search/search.service.ts b/server/src/domain/search/search.service.ts index 452c556f4..95ad848e2 100644 --- a/server/src/domain/search/search.service.ts +++ b/server/src/domain/search/search.service.ts @@ -16,7 +16,15 @@ import { SearchStrategy, } from '../repositories'; import { FeatureFlag, SystemConfigCore } from '../system-config'; -import { MetadataSearchDto, SearchDto, SearchPeopleDto, SmartSearchDto } from './dto'; +import { + MetadataSearchDto, + PlacesResponseDto, + SearchDto, + SearchPeopleDto, + SearchPlacesDto, + SmartSearchDto, + mapPlaces, +} from './dto'; import { SearchSuggestionRequestDto, SearchSuggestionType } from './dto/search-suggestion.dto'; import { SearchResponseDto } from './response-dto'; @@ -41,6 +49,11 @@ export class SearchService { return this.personRepository.getByName(auth.user.id, dto.name, { withHidden: dto.withHidden }); } + async searchPlaces(dto: SearchPlacesDto): Promise { + const places = await this.searchRepository.searchPlaces(dto.name); + return places.map((place) => mapPlaces(place)); + } + async getExploreData(auth: AuthDto): Promise[]> { await this.configCore.requireFeature(FeatureFlag.SEARCH); const options = { maxFields: 12, minAssetsPerField: 5 }; diff --git a/server/src/immich/controllers/search.controller.ts b/server/src/immich/controllers/search.controller.ts index 4e57cfaa6..b807da966 100644 --- a/server/src/immich/controllers/search.controller.ts +++ b/server/src/immich/controllers/search.controller.ts @@ -2,9 +2,11 @@ import { AuthDto, MetadataSearchDto, PersonResponseDto, + PlacesResponseDto, SearchDto, SearchExploreResponseDto, SearchPeopleDto, + SearchPlacesDto, SearchResponseDto, SearchService, SmartSearchDto, @@ -48,6 +50,11 @@ export class SearchController { return this.service.searchPerson(auth, dto); } + @Get('places') + searchPlaces(@Query() dto: SearchPlacesDto): Promise { + return this.service.searchPlaces(dto); + } + @Get('suggestions') getSearchSuggestions(@Auth() auth: AuthDto, @Query() dto: SearchSuggestionRequestDto): Promise { return this.service.getSearchSuggestions(auth, dto); diff --git a/server/src/infra/entities/geodata-admin1.entity.ts b/server/src/infra/entities/geodata-admin1.entity.ts deleted file mode 100644 index 36cf0a805..000000000 --- a/server/src/infra/entities/geodata-admin1.entity.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Column, Entity, PrimaryColumn } from 'typeorm'; - -@Entity('geodata_admin1') -export class GeodataAdmin1Entity { - @PrimaryColumn({ type: 'varchar' }) - key!: string; - - @Column({ type: 'varchar' }) - name!: string; -} diff --git a/server/src/infra/entities/geodata-admin2.entity.ts b/server/src/infra/entities/geodata-admin2.entity.ts deleted file mode 100644 index bd03e8377..000000000 --- a/server/src/infra/entities/geodata-admin2.entity.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { Column, Entity, PrimaryColumn } from 'typeorm'; - -@Entity('geodata_admin2') -export class GeodataAdmin2Entity { - @PrimaryColumn({ type: 'varchar' }) - key!: string; - - @Column({ type: 'varchar' }) - name!: string; -} diff --git a/server/src/infra/entities/geodata-places.entity.ts b/server/src/infra/entities/geodata-places.entity.ts index 244e4261b..966a50d5c 100644 --- a/server/src/infra/entities/geodata-places.entity.ts +++ b/server/src/infra/entities/geodata-places.entity.ts @@ -1,6 +1,4 @@ -import { GeodataAdmin1Entity } from '@app/infra/entities/geodata-admin1.entity'; -import { GeodataAdmin2Entity } from '@app/infra/entities/geodata-admin2.entity'; -import { Column, Entity, ManyToOne, PrimaryColumn } from 'typeorm'; +import { Column, Entity, PrimaryColumn } from 'typeorm'; @Entity('geodata_places', { synchronize: false }) export class GeodataPlacesEntity { @@ -21,7 +19,7 @@ export class GeodataPlacesEntity { // asExpression: 'll_to_earth((latitude)::double precision, (longitude)::double precision)', // type: 'earth', // }) - earthCoord!: unknown; + // earthCoord!: unknown; @Column({ type: 'char', length: 2 }) countryCode!: string; @@ -32,27 +30,14 @@ export class GeodataPlacesEntity { @Column({ type: 'varchar', length: 80, nullable: true }) admin2Code!: string; - @Column({ - type: 'varchar', - generatedType: 'STORED', - asExpression: `"countryCode" || '.' || "admin1Code"`, - nullable: true, - }) - admin1Key!: string; + @Column({ type: 'varchar', nullable: true }) + admin1Name!: string; - @ManyToOne(() => GeodataAdmin1Entity, { eager: true, nullable: true, createForeignKeyConstraints: false }) - admin1!: GeodataAdmin1Entity; + @Column({ type: 'varchar', nullable: true }) + admin2Name!: string; - @Column({ - type: 'varchar', - generatedType: 'STORED', - asExpression: `"countryCode" || '.' || "admin1Code" || '.' || "admin2Code"`, - nullable: true, - }) - admin2Key!: string; - - @ManyToOne(() => GeodataAdmin2Entity, { eager: true, nullable: true, createForeignKeyConstraints: false }) - admin2!: GeodataAdmin2Entity; + @Column({ type: 'varchar', nullable: true }) + alternateNames!: string; @Column({ type: 'date' }) modificationDate!: Date; diff --git a/server/src/infra/entities/index.ts b/server/src/infra/entities/index.ts index 957e15a88..af620790e 100644 --- a/server/src/infra/entities/index.ts +++ b/server/src/infra/entities/index.ts @@ -7,8 +7,6 @@ import { AssetStackEntity } from './asset-stack.entity'; import { AssetEntity } from './asset.entity'; import { AuditEntity } from './audit.entity'; import { ExifEntity } from './exif.entity'; -import { GeodataAdmin1Entity } from './geodata-admin1.entity'; -import { GeodataAdmin2Entity } from './geodata-admin2.entity'; import { GeodataPlacesEntity } from './geodata-places.entity'; import { LibraryEntity } from './library.entity'; import { MoveEntity } from './move.entity'; @@ -32,8 +30,6 @@ export * from './asset-stack.entity'; export * from './asset.entity'; export * from './audit.entity'; export * from './exif.entity'; -export * from './geodata-admin1.entity'; -export * from './geodata-admin2.entity'; export * from './geodata-places.entity'; export * from './library.entity'; export * from './move.entity'; @@ -59,8 +55,6 @@ export const databaseEntities = [ AuditEntity, ExifEntity, GeodataPlacesEntity, - GeodataAdmin1Entity, - GeodataAdmin2Entity, MoveEntity, PartnerEntity, PersonEntity, diff --git a/server/src/infra/migrations/1708059341865-GeodataLocationSearch.ts b/server/src/infra/migrations/1708059341865-GeodataLocationSearch.ts new file mode 100644 index 000000000..136ca2598 --- /dev/null +++ b/server/src/infra/migrations/1708059341865-GeodataLocationSearch.ts @@ -0,0 +1,152 @@ +import { MigrationInterface, QueryRunner } from 'typeorm'; + +export class GeodataLocationSearch1708059341865 implements MigrationInterface { + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS pg_trgm`); + await queryRunner.query(`CREATE EXTENSION IF NOT EXISTS unaccent`); + + // https://stackoverflow.com/a/11007216 + await queryRunner.query(` + CREATE OR REPLACE FUNCTION f_unaccent(text) + RETURNS text + LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT + RETURN unaccent('unaccent', $1)`); + + await queryRunner.query(`ALTER TABLE geodata_places ADD COLUMN "admin1Name" varchar`); + await queryRunner.query(`ALTER TABLE geodata_places ADD COLUMN "admin2Name" varchar`); + + await queryRunner.query(` + UPDATE geodata_places + SET "admin1Name" = admin1.name + FROM geodata_admin1 admin1 + WHERE admin1.key = "admin1Key"`); + + await queryRunner.query(` + UPDATE geodata_places + SET "admin2Name" = admin2.name + FROM geodata_admin2 admin2 + WHERE admin2.key = "admin2Key"`); + + await queryRunner.query(`DROP TABLE geodata_admin1 CASCADE`); + await queryRunner.query(`DROP TABLE geodata_admin2 CASCADE`); + + await queryRunner.query(` + ALTER TABLE geodata_places + DROP COLUMN "admin1Key", + DROP COLUMN "admin2Key"`); + + await queryRunner.query(` + CREATE INDEX idx_geodata_places_name + ON geodata_places + USING gin (f_unaccent(name) gin_trgm_ops)`); + + await queryRunner.query(` + CREATE INDEX idx_geodata_places_admin1_name + ON geodata_places + USING gin (f_unaccent("admin1Name") gin_trgm_ops)`); + + await queryRunner.query(` + CREATE INDEX idx_geodata_places_admin2_name + ON geodata_places + USING gin (f_unaccent("admin2Name") gin_trgm_ops)`); + + await queryRunner.query( + ` + DELETE FROM "typeorm_metadata" + WHERE + "type" = $1 AND + "name" = $2 AND + "database" = $3 AND + "schema" = $4 AND + "table" = $5`, + ['GENERATED_COLUMN', 'admin1Key', 'immich', 'public', 'geodata_places'], + ); + + await queryRunner.query( + ` + DELETE FROM "typeorm_metadata" + WHERE + "type" = $1 AND + "name" = $2 AND + "database" = $3 AND + "schema" = $4 AND + "table" = $5`, + ['GENERATED_COLUMN', 'admin2Key', 'immich', 'public', 'geodata_places'], + ); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(` + CREATE TABLE "geodata_admin1" ( + "key" character varying NOT NULL, + "name" character varying NOT NULL, + CONSTRAINT "PK_3fe3a89c5aac789d365871cb172" PRIMARY KEY ("key") + )`); + + await queryRunner.query(` + CREATE TABLE "geodata_admin2" ( + "key" character varying NOT NULL, + "name" character varying NOT NULL, + CONSTRAINT "PK_1e3886455dbb684d6f6b4756726" PRIMARY KEY ("key") + )`); + + await queryRunner.query(` + ALTER TABLE geodata_places + ADD COLUMN "admin1Key" character varying + GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code") STORED, + ADD COLUMN "admin2Key" character varying + GENERATED ALWAYS AS ("countryCode" || '.' || "admin1Code" || '.' || "admin2Code") STORED`); + + await queryRunner.query( + ` + INSERT INTO "geodata_admin1" + SELECT DISTINCT + "admin1Key" AS "key", + "admin1Name" AS "name" + FROM geodata_places + WHERE "admin1Name" IS NOT NULL`, + ); + + await queryRunner.query( + ` + INSERT INTO "geodata_admin2" + SELECT DISTINCT + "admin2Key" AS "key", + "admin2Name" AS "name" + FROM geodata_places + WHERE "admin2Name" IS NOT NULL`, + ); + + await queryRunner.query(` + UPDATE geodata_places + SET "admin1Name" = admin1.name + FROM geodata_admin1 admin1 + WHERE admin1.key = "admin1Key"`); + + await queryRunner.query(` + UPDATE geodata_places + SET "admin2Name" = admin2.name + FROM geodata_admin2 admin2 + WHERE admin2.key = "admin2Key";`); + + await queryRunner.query( + ` + INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") + VALUES ($1, $2, $3, $4, $5, $6)`, + ['immich', 'public', 'geodata_places', 'GENERATED_COLUMN', 'admin1Key', '"countryCode" || \'.\' || "admin1Code"'], + ); + + await queryRunner.query( + `INSERT INTO "typeorm_metadata"("database", "schema", "table", "type", "name", "value") + VALUES ($1, $2, $3, $4, $5, $6)`, + [ + 'immich', + 'public', + 'geodata_places', + 'GENERATED_COLUMN', + 'admin2Key', + '"countryCode" || \'.\' || "admin1Code" || \'.\' || "admin2Code"', + ], + ); + } +} diff --git a/server/src/infra/migrations/1708116312820-GeonamesEnhancement.ts b/server/src/infra/migrations/1708116312820-GeonamesEnhancement.ts new file mode 100644 index 000000000..0cea9a041 --- /dev/null +++ b/server/src/infra/migrations/1708116312820-GeonamesEnhancement.ts @@ -0,0 +1,18 @@ +import { MigrationInterface, QueryRunner } from "typeorm"; + +export class GeonamesEnhancement1708116312820 implements MigrationInterface { + name = 'GeonamesEnhancement1708116312820' + + public async up(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE geodata_places ADD COLUMN "alternateNames" varchar`); + await queryRunner.query(` + CREATE INDEX idx_geodata_places_admin2_alternate_names + ON geodata_places + USING gin (f_unaccent("alternateNames") gin_trgm_ops)`); + } + + public async down(queryRunner: QueryRunner): Promise { + await queryRunner.query(`ALTER TABLE geodata_places DROP COLUMN "alternateNames"`); + } + +} diff --git a/server/src/infra/repositories/metadata.repository.ts b/server/src/infra/repositories/metadata.repository.ts index 6a90ad108..4abfe0eac 100644 --- a/server/src/infra/repositories/metadata.repository.ts +++ b/server/src/infra/repositories/metadata.repository.ts @@ -2,7 +2,7 @@ import { citiesFile, geodataAdmin1Path, geodataAdmin2Path, - geodataCitites500Path, + geodataCities500Path, geodataDatePath, GeoPoint, IMetadataRepository, @@ -10,13 +10,7 @@ import { ISystemMetadataRepository, ReverseGeocodeResult, } from '@app/domain'; -import { - ExifEntity, - GeodataAdmin1Entity, - GeodataAdmin2Entity, - GeodataPlacesEntity, - SystemMetadataKey, -} from '@app/infra/entities'; +import { ExifEntity, GeodataPlacesEntity, SystemMetadataKey } from '@app/infra/entities'; import { ImmichLogger } from '@app/infra/logger'; import { Inject } from '@nestjs/common'; import { InjectDataSource, InjectRepository } from '@nestjs/typeorm'; @@ -26,19 +20,16 @@ import { getName } from 'i18n-iso-countries'; import { createReadStream, existsSync } from 'node:fs'; import { readFile } from 'node:fs/promises'; import * as readLine from 'node:readline'; -import { DataSource, DeepPartial, QueryRunner, Repository } from 'typeorm'; +import { DataSource, QueryRunner, Repository } from 'typeorm'; +import { QueryDeepPartialEntity } from 'typeorm/query-builder/QueryPartialEntity.js'; import { DummyValue, GenerateSql } from '../infra.util'; -type GeoEntity = GeodataPlacesEntity | GeodataAdmin1Entity | GeodataAdmin2Entity; -type GeoEntityClass = typeof GeodataPlacesEntity | typeof GeodataAdmin1Entity | typeof GeodataAdmin2Entity; - export class MetadataRepository implements IMetadataRepository { constructor( @InjectRepository(ExifEntity) private exifRepository: Repository, @InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository, - @InjectRepository(GeodataAdmin1Entity) private readonly geodataAdmin1Repository: Repository, - @InjectRepository(GeodataAdmin2Entity) private readonly geodataAdmin2Repository: Repository, - @Inject(ISystemMetadataRepository) private readonly systemMetadataRepository: ISystemMetadataRepository, + @Inject(ISystemMetadataRepository) + private readonly systemMetadataRepository: ISystemMetadataRepository, @InjectDataSource() private dataSource: DataSource, ) {} @@ -54,7 +45,6 @@ export class MetadataRepository implements IMetadataRepository { return; } - this.logger.log('Importing geodata to database from file'); await this.importGeodata(); await this.systemMetadataRepository.set(SystemMetadataKey.REVERSE_GEOCODING_STATE, { @@ -69,12 +59,14 @@ export class MetadataRepository implements IMetadataRepository { const queryRunner = this.dataSource.createQueryRunner(); await queryRunner.connect(); + const admin1 = await this.loadAdmin(geodataAdmin1Path); + const admin2 = await this.loadAdmin(geodataAdmin2Path); + try { await queryRunner.startTransaction(); - await this.loadCities500(queryRunner); - await this.loadAdmin1(queryRunner); - await this.loadAdmin2(queryRunner); + await queryRunner.manager.clear(GeodataPlacesEntity); + await this.loadCities500(queryRunner, admin1, admin2); await queryRunner.commitTransaction(); } catch (error) { @@ -86,76 +78,73 @@ export class MetadataRepository implements IMetadataRepository { } } - private async loadGeodataToTableFromFile( + private async loadGeodataToTableFromFile( queryRunner: QueryRunner, - lineToEntityMapper: (lineSplit: string[]) => T, + lineToEntityMapper: (lineSplit: string[]) => GeodataPlacesEntity, filePath: string, - entity: GeoEntityClass, ) { if (!existsSync(filePath)) { this.logger.error(`Geodata file ${filePath} not found`); throw new Error(`Geodata file ${filePath} not found`); } - await queryRunner.manager.clear(entity); const input = createReadStream(filePath); - let buffer: DeepPartial[] = []; - const lineReader = readLine.createInterface({ input: input }); + let bufferGeodata: QueryDeepPartialEntity[] = []; + const lineReader = readLine.createInterface({ input }); for await (const line of lineReader) { const lineSplit = line.split('\t'); - buffer.push(lineToEntityMapper(lineSplit)); - if (buffer.length > 1000) { - await queryRunner.manager.save(buffer); - buffer = []; + const geoData = lineToEntityMapper(lineSplit); + bufferGeodata.push(geoData); + if (bufferGeodata.length > 1000) { + await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']); + bufferGeodata = []; } } - await queryRunner.manager.save(buffer); + await queryRunner.manager.upsert(GeodataPlacesEntity, bufferGeodata, ['id']); } - private async loadCities500(queryRunner: QueryRunner) { - await this.loadGeodataToTableFromFile( + private async loadCities500( + queryRunner: QueryRunner, + admin1Map: Map, + admin2Map: Map, + ) { + await this.loadGeodataToTableFromFile( queryRunner, (lineSplit: string[]) => this.geodataPlacesRepository.create({ id: Number.parseInt(lineSplit[0]), name: lineSplit[1], + alternateNames: lineSplit[3], latitude: Number.parseFloat(lineSplit[4]), longitude: Number.parseFloat(lineSplit[5]), countryCode: lineSplit[8], admin1Code: lineSplit[10], admin2Code: lineSplit[11], modificationDate: lineSplit[18], + admin1Name: admin1Map.get(`${lineSplit[8]}.${lineSplit[10]}`), + admin2Name: admin2Map.get(`${lineSplit[8]}.${lineSplit[10]}.${lineSplit[11]}`), }), - geodataCitites500Path, - GeodataPlacesEntity, + geodataCities500Path, ); } - private async loadAdmin1(queryRunner: QueryRunner) { - await this.loadGeodataToTableFromFile( - queryRunner, - (lineSplit: string[]) => - this.geodataAdmin1Repository.create({ - key: lineSplit[0], - name: lineSplit[1], - }), - geodataAdmin1Path, - GeodataAdmin1Entity, - ); - } + private async loadAdmin(filePath: string) { + if (!existsSync(filePath)) { + this.logger.error(`Geodata file ${filePath} not found`); + throw new Error(`Geodata file ${filePath} not found`); + } - private async loadAdmin2(queryRunner: QueryRunner) { - await this.loadGeodataToTableFromFile( - queryRunner, - (lineSplit: string[]) => - this.geodataAdmin2Repository.create({ - key: lineSplit[0], - name: lineSplit[1], - }), - geodataAdmin2Path, - GeodataAdmin2Entity, - ); + const input = createReadStream(filePath); + const lineReader = readLine.createInterface({ input: input }); + + const adminMap = new Map(); + for await (const line of lineReader) { + const lineSplit = line.split('\t'); + adminMap.set(lineSplit[0], lineSplit[1]); + } + + return adminMap; } async teardown() { @@ -167,8 +156,6 @@ export class MetadataRepository implements IMetadataRepository { const response = await this.geodataPlacesRepository .createQueryBuilder('geoplaces') - .leftJoinAndSelect('geoplaces.admin1', 'admin1') - .leftJoinAndSelect('geoplaces.admin2', 'admin2') .where('earth_box(ll_to_earth(:latitude, :longitude), 25000) @> "earthCoord"', point) .orderBy('earth_distance(ll_to_earth(:latitude, :longitude), "earthCoord")') .limit(1) @@ -183,9 +170,9 @@ export class MetadataRepository implements IMetadataRepository { this.logger.verbose(`Raw: ${JSON.stringify(response, null, 2)}`); - const { countryCode, name: city, admin1, admin2 } = response; + const { countryCode, name: city, admin1Name, admin2Name } = response; const country = getName(countryCode, 'en') ?? null; - const stateParts = [admin2?.name, admin1?.name].filter((name) => !!name); + const stateParts = [admin2Name, admin1Name].filter((name) => !!name); const state = stateParts.length > 0 ? stateParts.join(', ') : null; return { country, state, city }; diff --git a/server/src/infra/repositories/search.repository.ts b/server/src/infra/repositories/search.repository.ts index a30c96b10..089640128 100644 --- a/server/src/infra/repositories/search.repository.ts +++ b/server/src/infra/repositories/search.repository.ts @@ -12,7 +12,13 @@ import { SmartSearchOptions, } from '@app/domain'; import { getCLIPModelInfo } from '@app/domain/smart-info/smart-info.constant'; -import { AssetEntity, AssetFaceEntity, SmartInfoEntity, SmartSearchEntity } from '@app/infra/entities'; +import { + AssetEntity, + AssetFaceEntity, + GeodataPlacesEntity, + SmartInfoEntity, + SmartSearchEntity, +} from '@app/infra/entities'; import { ImmichLogger } from '@app/infra/logger'; import { Injectable } from '@nestjs/common'; import { InjectRepository } from '@nestjs/typeorm'; @@ -31,6 +37,7 @@ export class SearchRepository implements ISearchRepository { @InjectRepository(AssetEntity) private assetRepository: Repository, @InjectRepository(AssetFaceEntity) private assetFaceRepository: Repository, @InjectRepository(SmartSearchEntity) private smartSearchRepository: Repository, + @InjectRepository(GeodataPlacesEntity) private readonly geodataPlacesRepository: Repository, ) { this.faceColumns = this.assetFaceRepository.manager.connection .getMetadata(AssetFaceEntity) @@ -172,6 +179,27 @@ export class SearchRepository implements ISearchRepository { })); } + @GenerateSql({ params: [DummyValue.STRING] }) + async searchPlaces(placeName: string): Promise { + return await this.geodataPlacesRepository + .createQueryBuilder('geoplaces') + .where(`f_unaccent(name) %>> f_unaccent(:placeName)`) + .orWhere(`f_unaccent("admin2Name") %>> f_unaccent(:placeName)`) + .orWhere(`f_unaccent("admin1Name") %>> f_unaccent(:placeName)`) + .orWhere(`f_unaccent("alternateNames") %>> f_unaccent(:placeName)`) + .orderBy( + ` + COALESCE(f_unaccent(name) <->>> f_unaccent(:placeName), 0) + + COALESCE(f_unaccent("admin2Name") <->>> f_unaccent(:placeName), 0) + + COALESCE(f_unaccent("admin1Name") <->>> f_unaccent(:placeName), 0) + + COALESCE(f_unaccent("alternateNames") <->>> f_unaccent(:placeName), 0) + `, + ) + .setParameters({ placeName }) + .limit(20) + .getMany(); + } + async upsert(smartInfo: Partial, embedding?: Embedding): Promise { await this.repository.upsert(smartInfo, { conflictPaths: ['assetId'] }); if (!smartInfo.assetId || !embedding) { diff --git a/server/src/infra/sql/search.repository.sql b/server/src/infra/sql/search.repository.sql index a21697c26..c45d90a7a 100644 --- a/server/src/infra/sql/search.repository.sql +++ b/server/src/infra/sql/search.repository.sql @@ -238,3 +238,37 @@ FROM WHERE res.distance <= $3 COMMIT + +-- SearchRepository.searchPlaces +SELECT + "geoplaces"."id" AS "geoplaces_id", + "geoplaces"."name" AS "geoplaces_name", + "geoplaces"."longitude" AS "geoplaces_longitude", + "geoplaces"."latitude" AS "geoplaces_latitude", + "geoplaces"."countryCode" AS "geoplaces_countryCode", + "geoplaces"."admin1Code" AS "geoplaces_admin1Code", + "geoplaces"."admin2Code" AS "geoplaces_admin2Code", + "geoplaces"."admin1Name" AS "geoplaces_admin1Name", + "geoplaces"."admin2Name" AS "geoplaces_admin2Name", + "geoplaces"."alternateNames" AS "geoplaces_alternateNames", + "geoplaces"."modificationDate" AS "geoplaces_modificationDate" +FROM + "geodata_places" "geoplaces" +WHERE + f_unaccent (name) %>> f_unaccent ($1) + OR f_unaccent ("admin2Name") %>> f_unaccent ($1) + OR f_unaccent ("admin1Name") %>> f_unaccent ($1) + OR f_unaccent ("alternateNames") %>> f_unaccent ($1) +ORDER BY + COALESCE(f_unaccent (name) <->>> f_unaccent ($1), 0) + COALESCE( + f_unaccent ("admin2Name") <->>> f_unaccent ($1), + 0 + ) + COALESCE( + f_unaccent ("admin1Name") <->>> f_unaccent ($1), + 0 + ) + COALESCE( + f_unaccent ("alternateNames") <->>> f_unaccent ($1), + 0 + ) ASC +LIMIT + 20 diff --git a/server/test/repositories/search.repository.mock.ts b/server/test/repositories/search.repository.mock.ts index e0bdab269..06a2cb76d 100644 --- a/server/test/repositories/search.repository.mock.ts +++ b/server/test/repositories/search.repository.mock.ts @@ -7,5 +7,6 @@ export const newSearchRepositoryMock = (): jest.Mocked => { searchSmart: jest.fn(), searchFaces: jest.fn(), upsert: jest.fn(), + searchPlaces: jest.fn(), }; }; diff --git a/web/src/lib/components/elements/search-bar.svelte b/web/src/lib/components/elements/search-bar.svelte index 9c6eded22..898601d0a 100644 --- a/web/src/lib/components/elements/search-bar.svelte +++ b/web/src/lib/components/elements/search-bar.svelte @@ -6,6 +6,7 @@ import LoadingSpinner from '../shared-components/loading-spinner.svelte'; export let name: string; + export let roundedBottom = true; export let isSearching: boolean; export let placeholder: string; @@ -17,7 +18,11 @@ }; -
+
+
+ {#if !hideSuggestion} + {#each suggestedPlaces as place, index} + + {/each} + {/if} +
+
{#await import('../shared-components/map/map.svelte')} @@ -63,6 +194,7 @@ this={component.default} mapMarkers={lat && lng && asset ? [{ id: asset.id, lat, lon: lng }] : []} {zoom} + bind:addClipMapMarker center={lat && lng ? { lat, lng } : undefined} simplified={true} clickable={true} diff --git a/web/src/lib/components/shared-components/map/map.svelte b/web/src/lib/components/shared-components/map/map.svelte index 1752a753a..682042604 100644 --- a/web/src/lib/components/shared-components/map/map.svelte +++ b/web/src/lib/components/shared-components/map/map.svelte @@ -32,6 +32,17 @@ export let simplified = false; export let clickable = false; export let useLocationPin = false; + export function addClipMapMarker(lng: number, lat: number) { + if (map) { + if (marker) { + marker.remove(); + } + + center = { lng, lat }; + marker = new maplibregl.Marker().setLngLat([lng, lat]).addTo(map); + map.setZoom(15); + } + } let map: maplibregl.Map; let marker: maplibregl.Marker | null = null;