From cc362e7e8fd89d210ff0ba98eb983806ad2033db Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Sat, 15 Feb 2014 16:17:33 +0100 Subject: [PATCH] First working sync --- vdirsyncer/storage/base.py | 8 +++- vdirsyncer/storage/filesystem.py | 5 ++- vdirsyncer/storage/memory.py | 9 ++-- vdirsyncer/sync.py | 74 +++++++++++++++++++++----------- vdirsyncer/tests/test_sync.py | 43 +++++++++++++++++++ 5 files changed, 110 insertions(+), 29 deletions(-) create mode 100644 vdirsyncer/tests/test_sync.py diff --git a/vdirsyncer/storage/base.py b/vdirsyncer/storage/base.py index 58d30f0..0f12ce4 100644 --- a/vdirsyncer/storage/base.py +++ b/vdirsyncer/storage/base.py @@ -42,7 +42,7 @@ class Storage(object): ''' Upload a new object, raise :exc:`vdirsyncer.exceptions.AlreadyExistingError` if it already exists. - :returns: (uid, etag) + :returns: etag on the server ''' raise NotImplementedError() @@ -55,3 +55,9 @@ class Storage(object): :returns: etag on the server ''' raise NotImplementedError() + + def delete(self, uid): + ''' + Delete the object, raise exceptions on error, no return value + ''' + raise NotImplementedError() diff --git a/vdirsyncer/storage/filesystem.py b/vdirsyncer/storage/filesystem.py index a272fcb..2e9da1c 100644 --- a/vdirsyncer/storage/filesystem.py +++ b/vdirsyncer/storage/filesystem.py @@ -32,7 +32,7 @@ class FilesystemStorage(Storage): raise exceptions.AlreadyExistingError(obj.uid) with open(fpath, 'wb+') as f: f.write(obj.raw) - return obj.uid, os.path.getmtime(fpath) + return os.path.getmtime(fpath) def update(self, obj, etag): fpath = self._get_filepath(obj) @@ -44,3 +44,6 @@ class FilesystemStorage(Storage): with open(fpath, 'wb') as f: f.write(obj.raw) return os.path.getmtime(fpath) + + def delete(self, uid): + os.remove(self._get_filepath(uid)) diff --git a/vdirsyncer/storage/memory.py b/vdirsyncer/storage/memory.py index b9c7e51..f423c67 100644 --- a/vdirsyncer/storage/memory.py +++ b/vdirsyncer/storage/memory.py @@ -14,7 +14,7 @@ class MemoryStorage(Storage): def get_items(self, uids): for uid in uids: etag, obj = self.items[uid] - return obj, uid, etag + yield obj, uid, etag def item_exists(self, uid): return uid in self.items @@ -24,11 +24,14 @@ class MemoryStorage(Storage): raise exceptions.AlreadyExistingError(obj) etag = datetime.datetime.now() self.items[obj.uid] = (etag, obj) - return obj.uid, etag + return etag def update(self, obj, etag): if obj.uid not in self.items: raise exceptions.NotFoundError(obj) etag = datetime.datetime.now() self.items[obj.uid] = (etag, obj) - return obj.uid, etag + return etag + + def delete(self, uid): + del self.items[uid] diff --git a/vdirsyncer/sync.py b/vdirsyncer/sync.py index 50be43c..0e390d1 100644 --- a/vdirsyncer/sync.py +++ b/vdirsyncer/sync.py @@ -5,32 +5,58 @@ def sync(storage_a, storage_b, status): :param storage_b: The second storage :param status: {uid: (etag_a, etag_b)} ''' - items_a = dict(storage_a.list_items()) - items_b = dict(storage_b.list_items()) - downloads_a = set() # uids which to copy from a to b - downloads_b = set() # uids which to copy from b to a - deletes_a = set() - deletes_b = set() + list_a = dict(storage_a.list_items()) + list_b = dict(storage_b.list_items()) - for uid in set(items_a) + set(items_b): + prefetch_items_from_a = [] + prefetch_items_from_b = [] + actions = [] # list(tuple(action, uid, source, dest)) + + for uid in set(list_a).union(set(list_b)): if uid not in status: - if uid in items_a and uid in items_b: # missing status - status[uid] = (items_a[uid], items_b[uid]) # TODO: might need etag diffing too? - elif uid in items_a and uid not in items_b: # new item in a - downloads_a.add(uid) - elif uid not in items_a and uid in items_b: # new item in b - downloads_b.add(uid) + if uid in list_a and uid in list_b: # missing status + status[uid] = (list_a[uid], list_b[uid]) # TODO: might need etag diffing too? + elif uid in list_a and uid not in list_b: # new item in a + prefetch_items_from_a.append(uid) + actions.append(('upload', uid, 'a', 'b')) + elif uid not in list_a and uid in list_b: # new item in b + prefetch_items_from_b.append(uid) + actions.append(('upload', uid, 'b', 'a')) else: - if uid in items_a and uid in items_b: - if items_a[uid] != status[uid][0] and items_a[uid] != status[uid][1]: - 1/0 # conflict resolution - elif items_a[uid] != status[uid][0]: # item update in a - downloads_a.add(uid) - elif items_b[uid] != status[uid][1]: # item update in b - downloads_b.add(uid) + if uid in list_a and uid in list_b: + if list_a[uid] != status[uid][0] and list_b[uid] != status[uid][1]: + 1/0 # conflict resolution TODO + elif list_a[uid] != status[uid][0]: # item update in a + prefetch_items_from_a.append(uid) + actions.append(('update', uid, 'a', 'b')) + elif list_b[uid] != status[uid][1]: # item update in b + prefetch_items_from_b.append(uid) + actions.append(('update', uid, 'b', 'a')) else: # completely in sync! pass - elif uid in items_a and uid not in items_b: # deleted from b - deletes_a.add(uid) - elif uid not in items_a and uid in items_b: # deleted from a - deletes_b.add(uid) + elif uid in list_a and uid not in list_b: # deleted from b + actions.append(('delete', uid, 'b', 'a')) + elif uid not in list_a and uid in list_b: # deleted from a + actions.append(('delete', uid, 'a', 'b')) + + items_a = {} + items_b = {} + for item, uid, etag in storage_a.get_items(prefetch_items_from_a): + items_a[uid] = (item, etag) + for item, uid, etag in storage_b.get_items(prefetch_items_from_b): + items_b[uid] = (item, etag) + + for action, uid, source, dest in actions: + source_storage = storage_a if source == 'a' else storage_b + dest_storage = storage_a if dest == 'a' else storage_b + source_items = items_a if source == 'a' else items_b + if action in ('upload', 'update'): + item, source_etag = source_items[uid] + if action == 'upload': + dest_etag = dest_storage.upload(item) + else: + dest_etag = dest_storage.update(item, etag) + status[uid] = (source_etag, dest_etag) if source == 'a' else (dest_etag, source_etag) + elif action == 'delete': + dest_storage.delete(uid) + del status[uid] diff --git a/vdirsyncer/tests/test_sync.py b/vdirsyncer/tests/test_sync.py new file mode 100644 index 0000000..53aa339 --- /dev/null +++ b/vdirsyncer/tests/test_sync.py @@ -0,0 +1,43 @@ +from unittest import TestCase +from vdirsyncer.storage.base import Item +from vdirsyncer.storage.memory import MemoryStorage +from vdirsyncer.sync import sync +import vdirsyncer.exceptions as exceptions + +class SyncTests(TestCase): + def test_basic(self): + a = MemoryStorage() + b = MemoryStorage() + status = {} + sync(a, b, status) + assert len(status) == 0 + assert list(a.list_items()) == [] + assert list(b.list_items()) == [] + + item = Item('UID:1') + a.upload(item) + sync(a, b, status) + assert list(status) == ['1'] + obj, uid, etag = next(b.get_items(['1'])) + assert obj.raw == item.raw + + item2 = Item('UID:2') + b.upload(item2) + b.delete('1') + sync(a, b, status) + assert list(status) == ['2'] + assert next(a.list_items())[0] == '2' + assert next(b.list_items())[0] == '2' + obj, uid, etag = next(a.get_items(['2'])) + assert obj.raw == item2.raw + + new_item2 = Item('UID:2\nHUEHUEHUE:PRECISELY') + old_status = status.copy() + a.update(new_item2, next(a.list_items())[1]) + sync(a, b, status) + assert status != old_status + assert list(status) == list(old_status) + assert next(a.list_items())[0] == '2' + assert next(b.list_items())[0] == '2' + obj, uid, etag = next(b.get_items(['2'])) + assert obj.raw == new_item2.raw