feat: sync partner stacks (#19635)

This commit is contained in:
Jason Rasmussen 2025-06-30 16:41:06 -04:00 committed by GitHub
parent 32a7087883
commit 58ca1402ed
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
11 changed files with 433 additions and 26 deletions

Binary file not shown.

Binary file not shown.

View file

@ -13804,6 +13804,9 @@
"PartnerAssetDeleteV1",
"PartnerAssetExifV1",
"PartnerAssetExifBackfillV1",
"PartnerStackBackfillV1",
"PartnerStackDeleteV1",
"PartnerStackV1",
"AlbumV1",
"AlbumDeleteV1",
"AlbumUserV1",
@ -13973,20 +13976,21 @@
},
"SyncRequestType": {
"enum": [
"UsersV1",
"PartnersV1",
"AssetsV1",
"AssetExifsV1",
"PartnerAssetsV1",
"PartnerAssetExifsV1",
"AlbumsV1",
"AlbumUsersV1",
"AlbumToAssetsV1",
"AlbumAssetsV1",
"AlbumAssetExifsV1",
"AssetsV1",
"AssetExifsV1",
"MemoriesV1",
"MemoryToAssetsV1",
"StacksV1"
"PartnersV1",
"PartnerAssetsV1",
"PartnerAssetExifsV1",
"PartnerStacksV1",
"StacksV1",
"UsersV1"
],
"type": "string"
},

View file

@ -4073,6 +4073,9 @@ export enum SyncEntityType {
PartnerAssetDeleteV1 = "PartnerAssetDeleteV1",
PartnerAssetExifV1 = "PartnerAssetExifV1",
PartnerAssetExifBackfillV1 = "PartnerAssetExifBackfillV1",
PartnerStackBackfillV1 = "PartnerStackBackfillV1",
PartnerStackDeleteV1 = "PartnerStackDeleteV1",
PartnerStackV1 = "PartnerStackV1",
AlbumV1 = "AlbumV1",
AlbumDeleteV1 = "AlbumDeleteV1",
AlbumUserV1 = "AlbumUserV1",
@ -4094,20 +4097,21 @@ export enum SyncEntityType {
SyncAckV1 = "SyncAckV1"
}
export enum SyncRequestType {
UsersV1 = "UsersV1",
PartnersV1 = "PartnersV1",
AssetsV1 = "AssetsV1",
AssetExifsV1 = "AssetExifsV1",
PartnerAssetsV1 = "PartnerAssetsV1",
PartnerAssetExifsV1 = "PartnerAssetExifsV1",
AlbumsV1 = "AlbumsV1",
AlbumUsersV1 = "AlbumUsersV1",
AlbumToAssetsV1 = "AlbumToAssetsV1",
AlbumAssetsV1 = "AlbumAssetsV1",
AlbumAssetExifsV1 = "AlbumAssetExifsV1",
AssetsV1 = "AssetsV1",
AssetExifsV1 = "AssetExifsV1",
MemoriesV1 = "MemoriesV1",
MemoryToAssetsV1 = "MemoryToAssetsV1",
StacksV1 = "StacksV1"
PartnersV1 = "PartnersV1",
PartnerAssetsV1 = "PartnerAssetsV1",
PartnerAssetExifsV1 = "PartnerAssetExifsV1",
PartnerStacksV1 = "PartnerStacksV1",
StacksV1 = "StacksV1",
UsersV1 = "UsersV1"
}
export enum TranscodeHWAccel {
Nvenc = "nvenc",

View file

@ -356,6 +356,13 @@ export const columns = {
'assets.duration',
],
syncAlbumUser: ['album_users.albumsId as albumId', 'album_users.usersId as userId', 'album_users.role'],
syncStack: [
'asset_stack.id',
'asset_stack.createdAt',
'asset_stack.updatedAt',
'asset_stack.primaryAssetId',
'asset_stack.ownerId',
],
stack: ['stack.id', 'stack.primaryAssetId', 'ownerId'],
syncAssetExif: [
'exif.assetId',

View file

@ -267,6 +267,9 @@ export type SyncItem = {
[SyncEntityType.MemoryToAssetDeleteV1]: SyncMemoryAssetDeleteV1;
[SyncEntityType.StackV1]: SyncStackV1;
[SyncEntityType.StackDeleteV1]: SyncStackDeleteV1;
[SyncEntityType.PartnerStackBackfillV1]: SyncStackV1;
[SyncEntityType.PartnerStackDeleteV1]: SyncStackDeleteV1;
[SyncEntityType.PartnerStackV1]: SyncStackV1;
[SyncEntityType.SyncAckV1]: SyncAckV1;
};

View file

@ -573,20 +573,21 @@ export enum DatabaseLock {
}
export enum SyncRequestType {
UsersV1 = 'UsersV1',
PartnersV1 = 'PartnersV1',
AssetsV1 = 'AssetsV1',
AssetExifsV1 = 'AssetExifsV1',
PartnerAssetsV1 = 'PartnerAssetsV1',
PartnerAssetExifsV1 = 'PartnerAssetExifsV1',
AlbumsV1 = 'AlbumsV1',
AlbumUsersV1 = 'AlbumUsersV1',
AlbumToAssetsV1 = 'AlbumToAssetsV1',
AlbumAssetsV1 = 'AlbumAssetsV1',
AlbumAssetExifsV1 = 'AlbumAssetExifsV1',
AssetsV1 = 'AssetsV1',
AssetExifsV1 = 'AssetExifsV1',
MemoriesV1 = 'MemoriesV1',
MemoryToAssetsV1 = 'MemoryToAssetsV1',
PartnersV1 = 'PartnersV1',
PartnerAssetsV1 = 'PartnerAssetsV1',
PartnerAssetExifsV1 = 'PartnerAssetExifsV1',
PartnerStacksV1 = 'PartnerStacksV1',
StacksV1 = 'StacksV1',
UsersV1 = 'UsersV1',
}
export enum SyncEntityType {
@ -605,6 +606,9 @@ export enum SyncEntityType {
PartnerAssetDeleteV1 = 'PartnerAssetDeleteV1',
PartnerAssetExifV1 = 'PartnerAssetExifV1',
PartnerAssetExifBackfillV1 = 'PartnerAssetExifBackfillV1',
PartnerStackBackfillV1 = 'PartnerStackBackfillV1',
PartnerStackDeleteV1 = 'PartnerStackDeleteV1',
PartnerStackV1 = 'PartnerStackV1',
AlbumV1 = 'AlbumV1',
AlbumDeleteV1 = 'AlbumDeleteV1',

View file

@ -689,6 +689,66 @@ where
order by
"updateId" asc
-- SyncRepository.partnerStack.getDeletes
select
"id",
"stackId"
from
"stacks_audit"
where
"userId" in (
select
"sharedById"
from
"partners"
where
"sharedWithId" = $1
)
and "deletedAt" < now() - interval '1 millisecond'
order by
"id" asc
-- SyncRepository.partnerStack.getBackfill
select
"asset_stack"."id",
"asset_stack"."createdAt",
"asset_stack"."updatedAt",
"asset_stack"."primaryAssetId",
"asset_stack"."ownerId",
"updateId"
from
"asset_stack"
where
"ownerId" = $1
and "updatedAt" < now() - interval '1 millisecond'
and "updateId" <= $2
and "updateId" >= $3
order by
"updateId" asc
-- SyncRepository.partnerStack.getUpserts
select
"asset_stack"."id",
"asset_stack"."createdAt",
"asset_stack"."updatedAt",
"asset_stack"."primaryAssetId",
"asset_stack"."ownerId",
"updateId"
from
"asset_stack"
where
"ownerId" in (
select
"sharedById"
from
"partners"
where
"sharedWithId" = $1
)
and "updatedAt" < now() - interval '1 millisecond'
order by
"updateId" asc
-- SyncRepository.stack.getDeletes
select
"id",
@ -703,11 +763,11 @@ order by
-- SyncRepository.stack.getUpserts
select
"id",
"createdAt",
"updatedAt",
"primaryAssetId",
"ownerId",
"asset_stack"."id",
"asset_stack"."createdAt",
"asset_stack"."updatedAt",
"asset_stack"."primaryAssetId",
"asset_stack"."ownerId",
"updateId"
from
"asset_stack"

View file

@ -41,6 +41,7 @@ export class SyncRepository {
partner: PartnerSync;
partnerAsset: PartnerAssetsSync;
partnerAssetExif: PartnerAssetExifsSync;
partnerStack: PartnerStackSync;
stack: StackSync;
user: UserSync;
@ -57,6 +58,7 @@ export class SyncRepository {
this.partner = new PartnerSync(this.db);
this.partnerAsset = new PartnerAssetsSync(this.db);
this.partnerAssetExif = new PartnerAssetExifsSync(this.db);
this.partnerStack = new PartnerStackSync(this.db);
this.stack = new StackSync(this.db);
this.user = new UserSync(this.db);
}
@ -552,13 +554,54 @@ class StackSync extends BaseSync {
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('asset_stack')
.select(['id', 'createdAt', 'updatedAt', 'primaryAssetId', 'ownerId', 'updateId'])
.select(columns.syncStack)
.select('updateId')
.where('ownerId', '=', userId)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class PartnerStackSync extends BaseSync {
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getDeletes(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('stacks_audit')
.select(['id', 'stackId'])
.where('userId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.auditTableFilters(qb, ack))
.stream();
}
@GenerateSql({ params: [DummyValue.UUID, DummyValue.UUID, DummyValue.UUID], stream: true })
getBackfill(partnerId: string, afterUpdateId: string | undefined, beforeUpdateId: string) {
return this.db
.selectFrom('asset_stack')
.select(columns.syncStack)
.select('updateId')
.where('ownerId', '=', partnerId)
.where('updatedAt', '<', sql.raw<Date>("now() - interval '1 millisecond'"))
.where('updateId', '<=', beforeUpdateId)
.$if(!!afterUpdateId, (eb) => eb.where('updateId', '>=', afterUpdateId!))
.orderBy('updateId', 'asc')
.stream();
}
@GenerateSql({ params: [DummyValue.UUID], stream: true })
getUpserts(userId: string, ack?: SyncAck) {
return this.db
.selectFrom('asset_stack')
.select(columns.syncStack)
.select('updateId')
.where('ownerId', 'in', (eb) =>
eb.selectFrom('partners').select(['sharedById']).where('sharedWithId', '=', userId),
)
.$call((qb) => this.upsertTableFilters(qb, ack))
.stream();
}
}
class UserSync extends BaseSync {
@GenerateSql({ params: [], stream: true })
getDeletes(ack?: SyncAck) {

View file

@ -59,6 +59,7 @@ export const SYNC_TYPES_ORDER = [
SyncRequestType.AssetsV1,
SyncRequestType.StacksV1,
SyncRequestType.PartnerAssetsV1,
SyncRequestType.PartnerStacksV1,
SyncRequestType.AlbumAssetsV1,
SyncRequestType.AlbumsV1,
SyncRequestType.AlbumUsersV1,
@ -139,6 +140,7 @@ export class SyncService extends BaseService {
[SyncRequestType.MemoriesV1]: () => this.syncMemoriesV1(response, checkpointMap, auth),
[SyncRequestType.MemoryToAssetsV1]: () => this.syncMemoryAssetsV1(response, checkpointMap, auth),
[SyncRequestType.StacksV1]: () => this.syncStackV1(response, checkpointMap, auth),
[SyncRequestType.PartnerStacksV1]: () => this.syncPartnerStackV1(response, checkpointMap, auth, sessionId),
};
for (const type of SYNC_TYPES_ORDER.filter((type) => dto.types.includes(type))) {
@ -526,6 +528,54 @@ export class SyncService extends BaseService {
}
}
private async syncPartnerStackV1(response: Writable, checkpointMap: CheckpointMap, auth: AuthDto, sessionId: string) {
const deleteType = SyncEntityType.PartnerStackDeleteV1;
const deletes = this.syncRepository.partnerStack.getDeletes(auth.user.id, checkpointMap[deleteType]);
for await (const { id, ...data } of deletes) {
send(response, { type: deleteType, ids: [id], data });
}
const backfillType = SyncEntityType.PartnerStackBackfillV1;
const backfillCheckpoint = checkpointMap[backfillType];
const partners = await this.syncRepository.partner.getCreatedAfter(auth.user.id, backfillCheckpoint?.updateId);
const upsertType = SyncEntityType.PartnerStackV1;
const upsertCheckpoint = checkpointMap[upsertType];
if (upsertCheckpoint) {
const endId = upsertCheckpoint.updateId;
for (const partner of partners) {
const createId = partner.createId;
if (isEntityBackfillComplete(createId, backfillCheckpoint)) {
continue;
}
const startId = getStartId(createId, backfillCheckpoint);
const backfill = this.syncRepository.partnerStack.getBackfill(partner.sharedById, startId, endId);
for await (const { updateId, ...data } of backfill) {
send(response, {
type: backfillType,
ids: [createId, updateId],
data,
});
}
sendEntityBackfillCompleteAck(response, backfillType, createId);
}
} else if (partners.length > 0) {
await this.upsertBackfillCheckpoint({
type: backfillType,
sessionId,
createId: partners.at(-1)!.createId,
});
}
const upserts = this.syncRepository.partnerStack.getUpserts(auth.user.id, checkpointMap[upsertType]);
for await (const { updateId, ...data } of upserts) {
send(response, { type: upsertType, ids: [updateId], data });
}
}
private async upsertBackfillCheckpoint(item: { type: SyncEntityType; sessionId: string; createId: string }) {
const { type, sessionId, createId } = item;
await this.syncCheckpointRepository.upsertAll([

View file

@ -0,0 +1,232 @@
import { Kysely } from 'kysely';
import { SyncEntityType, SyncRequestType } from 'src/enum';
import { PartnerRepository } from 'src/repositories/partner.repository';
import { StackRepository } from 'src/repositories/stack.repository';
import { UserRepository } from 'src/repositories/user.repository';
import { DB } from 'src/schema';
import { SyncTestContext } from 'test/medium.factory';
import { factory } from 'test/small.factory';
import { getKyselyDB, wait } from 'test/utils';
let defaultDatabase: Kysely<DB>;
const setup = async (db?: Kysely<DB>) => {
const ctx = new SyncTestContext(db || defaultDatabase);
const { auth, user, session } = await ctx.newSyncAuthUser();
return { auth, user, session, ctx };
};
beforeAll(async () => {
defaultDatabase = await getKyselyDB();
});
describe(SyncRequestType.PartnerStacksV1, () => {
it('should detect and sync the first partner stack', async () => {
const { auth, user, ctx } = await setup();
const { user: user2 } = await ctx.newUser();
await ctx.newPartner({ sharedById: user2.id, sharedWithId: user.id });
const { asset } = await ctx.newAsset({ ownerId: user2.id });
const { stack } = await ctx.newStack({ ownerId: user2.id }, [asset.id]);
const response = await ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1]);
expect(response).toHaveLength(1);
expect(response).toEqual([
{
ack: expect.any(String),
data: {
id: stack.id,
ownerId: stack.ownerId,
createdAt: (stack.createdAt as Date).toISOString(),
updatedAt: (stack.updatedAt as Date).toISOString(),
primaryAssetId: stack.primaryAssetId,
},
type: SyncEntityType.PartnerStackV1,
},
]);
await ctx.syncAckAll(auth, response);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toEqual([]);
});
it('should detect and sync a deleted partner stack', async () => {
const { auth, user, ctx } = await setup();
const stackRepo = ctx.get(StackRepository);
const { user: user2 } = await ctx.newUser();
await ctx.newPartner({ sharedById: user2.id, sharedWithId: user.id });
const { asset } = await ctx.newAsset({ ownerId: user2.id });
const { stack } = await ctx.newStack({ ownerId: user2.id }, [asset.id]);
await stackRepo.delete(stack.id);
const response = await ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1]);
expect(response).toHaveLength(1);
expect(response).toEqual([
{
ack: expect.stringContaining('PartnerStackDeleteV1'),
data: {
stackId: stack.id,
},
type: SyncEntityType.PartnerStackDeleteV1,
},
]);
await ctx.syncAckAll(auth, response);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toEqual([]);
});
it('should not sync a deleted partner stack due to a user delete', async () => {
const { auth, user, ctx } = await setup();
const userRepo = ctx.get(UserRepository);
const { user: user2 } = await ctx.newUser();
await ctx.newPartner({ sharedById: user2.id, sharedWithId: user.id });
const { asset } = await ctx.newAsset({ ownerId: user2.id });
await ctx.newStack({ ownerId: user2.id }, [asset.id]);
await userRepo.delete({ id: user2.id }, true);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toEqual([]);
});
it('should not sync a deleted partner stack due to a partner delete (unshare)', async () => {
const { auth, user, ctx } = await setup();
const partnerRepo = ctx.get(PartnerRepository);
const { user: user2 } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user2.id });
await ctx.newStack({ ownerId: user2.id }, [asset.id]);
const { partner } = await ctx.newPartner({ sharedById: user2.id, sharedWithId: user.id });
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toHaveLength(1);
await partnerRepo.remove(partner);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toEqual([]);
});
it('should not sync a stack or stack delete for own user', async () => {
const { auth, user, ctx } = await setup();
const stackRepo = ctx.get(StackRepository);
const { user: user2 } = await ctx.newUser();
const { asset } = await ctx.newAsset({ ownerId: user.id });
const { stack } = await ctx.newStack({ ownerId: user.id }, [asset.id]);
await ctx.newPartner({ sharedById: user2.id, sharedWithId: user.id });
await expect(ctx.syncStream(auth, [SyncRequestType.StacksV1])).resolves.toHaveLength(1);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toHaveLength(0);
await stackRepo.delete(stack.id);
await expect(ctx.syncStream(auth, [SyncRequestType.StacksV1])).resolves.toHaveLength(1);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toHaveLength(0);
});
it('should not sync a stack or stack delete for unrelated user', async () => {
const { auth, ctx } = await setup();
const stackRepo = ctx.get(StackRepository);
const { user: user2 } = await ctx.newUser();
const { session } = await ctx.newSession({ userId: user2.id });
const { asset } = await ctx.newAsset({ ownerId: user2.id });
const { stack } = await ctx.newStack({ ownerId: user2.id }, [asset.id]);
const auth2 = factory.auth({ session, user: user2 });
await expect(ctx.syncStream(auth2, [SyncRequestType.StacksV1])).resolves.toHaveLength(1);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toHaveLength(0);
await stackRepo.delete(stack.id);
await expect(ctx.syncStream(auth2, [SyncRequestType.StacksV1])).resolves.toHaveLength(1);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toHaveLength(0);
});
it('should backfill partner stacks when a partner shared their library with you', async () => {
const { auth, user, ctx } = await setup();
const { user: user2 } = await ctx.newUser();
const { user: user3 } = await ctx.newUser();
const { asset: asset3 } = await ctx.newAsset({ ownerId: user3.id });
const { stack: stack3 } = await ctx.newStack({ ownerId: user3.id }, [asset3.id]);
await wait(2);
const { asset: asset2 } = await ctx.newAsset({ ownerId: user2.id });
const { stack: stack2 } = await ctx.newStack({ ownerId: user2.id }, [asset2.id]);
await ctx.newPartner({ sharedById: user2.id, sharedWithId: auth.user.id });
const response = await ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1]);
expect(response).toHaveLength(1);
expect(response).toEqual([
{
ack: expect.stringContaining('PartnerStackV1'),
data: expect.objectContaining({
id: stack2.id,
}),
type: SyncEntityType.PartnerStackV1,
},
]);
await ctx.syncAckAll(auth, response);
await ctx.newPartner({ sharedById: user3.id, sharedWithId: user.id });
const newResponse = await ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1]);
expect(newResponse).toHaveLength(2);
expect(newResponse).toEqual([
{
ack: expect.stringContaining(SyncEntityType.PartnerStackBackfillV1),
data: expect.objectContaining({
id: stack3.id,
}),
type: SyncEntityType.PartnerStackBackfillV1,
},
{
ack: expect.stringContaining(SyncEntityType.PartnerStackBackfillV1),
data: {},
type: SyncEntityType.SyncAckV1,
},
]);
await ctx.syncAckAll(auth, newResponse);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toEqual([]);
});
it('should only backfill partner stacks created prior to the current partner stack checkpoint', async () => {
const { auth, ctx } = await setup();
const { user: user2 } = await ctx.newUser();
const { user: user3 } = await ctx.newUser();
const { asset: asset3 } = await ctx.newAsset({ ownerId: user3.id });
const { stack: stack3 } = await ctx.newStack({ ownerId: user3.id }, [asset3.id]);
await wait(2);
const { asset: asset2 } = await ctx.newAsset({ ownerId: user2.id });
const { stack: stack2 } = await ctx.newStack({ ownerId: user2.id }, [asset2.id]);
await wait(2);
const { asset: asset4 } = await ctx.newAsset({ ownerId: user3.id });
const { stack: stack4 } = await ctx.newStack({ ownerId: user3.id }, [asset4.id]);
await ctx.newPartner({ sharedById: user2.id, sharedWithId: auth.user.id });
const response = await ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1]);
expect(response).toHaveLength(1);
expect(response).toEqual([
{
ack: expect.stringContaining(SyncEntityType.PartnerStackV1),
data: expect.objectContaining({
id: stack2.id,
}),
type: SyncEntityType.PartnerStackV1,
},
]);
await ctx.syncAckAll(auth, response);
await ctx.newPartner({ sharedById: user3.id, sharedWithId: auth.user.id });
const newResponse = await ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1]);
expect(newResponse).toHaveLength(3);
expect(newResponse).toEqual([
{
ack: expect.any(String),
data: expect.objectContaining({
id: stack3.id,
}),
type: SyncEntityType.PartnerStackBackfillV1,
},
{
ack: expect.stringContaining(SyncEntityType.PartnerStackBackfillV1),
data: {},
type: SyncEntityType.SyncAckV1,
},
{
ack: expect.any(String),
data: expect.objectContaining({
id: stack4.id,
}),
type: SyncEntityType.PartnerStackV1,
},
]);
await ctx.syncAckAll(auth, newResponse);
await expect(ctx.syncStream(auth, [SyncRequestType.PartnerStacksV1])).resolves.toEqual([]);
});
});