First working sync

vdirsyncer/storage/base.py

@@ -42,7 +42,7 @@ class Storage(object):
         '''
         Upload a new object, raise
         :exc:`vdirsyncer.exceptions.AlreadyExistingError` if it already exists.
-        :returns: (uid, etag)
+        :returns: etag on the server
         '''
         raise NotImplementedError()
 
@@ -55,3 +55,9 @@ class Storage(object):
         :returns: etag on the server
         '''
         raise NotImplementedError()
+
+    def delete(self, uid):
+        '''
+        Delete the object, raise exceptions on error, no return value
+        '''
+        raise NotImplementedError()

vdirsyncer/storage/filesystem.py

@@ -32,7 +32,7 @@ class FilesystemStorage(Storage):
             raise exceptions.AlreadyExistingError(obj.uid)
         with open(fpath, 'wb+') as f:
             f.write(obj.raw)
-        return obj.uid, os.path.getmtime(fpath)
+        return os.path.getmtime(fpath)
 
     def update(self, obj, etag):
         fpath = self._get_filepath(obj)
@@ -44,3 +44,6 @@ class FilesystemStorage(Storage):
         with open(fpath, 'wb') as f:
             f.write(obj.raw)
         return os.path.getmtime(fpath)
+
+    def delete(self, uid):
+        os.remove(self._get_filepath(uid))

vdirsyncer/storage/memory.py

@@ -14,7 +14,7 @@ class MemoryStorage(Storage):
     def get_items(self, uids):
         for uid in uids:
             etag, obj = self.items[uid]
-            return obj, uid, etag
+            yield obj, uid, etag
 
     def item_exists(self, uid):
         return uid in self.items
@@ -24,11 +24,14 @@ class MemoryStorage(Storage):
             raise exceptions.AlreadyExistingError(obj)
         etag = datetime.datetime.now()
         self.items[obj.uid] = (etag, obj)
-        return obj.uid, etag
+        return etag
 
     def update(self, obj, etag):
         if obj.uid not in self.items:
             raise exceptions.NotFoundError(obj)
         etag = datetime.datetime.now()
         self.items[obj.uid] = (etag, obj)
-        return obj.uid, etag
+        return etag
+
+    def delete(self, uid):
+        del self.items[uid]

vdirsyncer/sync.py

@@ -5,32 +5,58 @@ def sync(storage_a, storage_b, status):
     :param storage_b: The second storage
     :param status: {uid: (etag_a, etag_b)}
     '''
-    items_a = dict(storage_a.list_items())
+    list_a = dict(storage_a.list_items())
-    items_b = dict(storage_b.list_items())
+    list_b = dict(storage_b.list_items())
-    downloads_a = set()  # uids which to copy from a to b
-    downloads_b = set()  # uids which to copy from b to a
-    deletes_a = set()
-    deletes_b = set()
 
-    for uid in set(items_a) + set(items_b):
+    prefetch_items_from_a = []
+    prefetch_items_from_b = []
+    actions = []  # list(tuple(action, uid, source, dest))
+
+    for uid in set(list_a).union(set(list_b)):
         if uid not in status:
-            if uid in items_a and uid in items_b:  # missing status
+            if uid in list_a and uid in list_b:  # missing status
-                status[uid] = (items_a[uid], items_b[uid])  # TODO: might need etag diffing too?
+                status[uid] = (list_a[uid], list_b[uid])  # TODO: might need etag diffing too?
-            elif uid in items_a and uid not in items_b:  # new item in a
+            elif uid in list_a and uid not in list_b:  # new item in a
-                downloads_a.add(uid)
+                prefetch_items_from_a.append(uid)
-            elif uid not in items_a and uid in items_b:  # new item in b
+                actions.append(('upload', uid, 'a', 'b'))
-                downloads_b.add(uid)
+            elif uid not in list_a and uid in list_b:  # new item in b
+                prefetch_items_from_b.append(uid)
+                actions.append(('upload', uid, 'b', 'a'))
         else:
-            if uid in items_a and uid in items_b:
+            if uid in list_a and uid in list_b:
-                if items_a[uid] != status[uid][0] and items_a[uid] != status[uid][1]:
+                if list_a[uid] != status[uid][0] and list_b[uid] != status[uid][1]:
-                    1/0  # conflict resolution
+                    1/0  # conflict resolution TODO
-                elif items_a[uid] != status[uid][0]:  # item update in a
+                elif list_a[uid] != status[uid][0]:  # item update in a
-                    downloads_a.add(uid)
+                    prefetch_items_from_a.append(uid)
-                elif items_b[uid] != status[uid][1]:  # item update in b
+                    actions.append(('update', uid, 'a', 'b'))
-                    downloads_b.add(uid)
+                elif list_b[uid] != status[uid][1]:  # item update in b
+                    prefetch_items_from_b.append(uid)
+                    actions.append(('update', uid, 'b', 'a'))
                 else:  # completely in sync!
                     pass
-            elif uid in items_a and uid not in items_b:  # deleted from b
+            elif uid in list_a and uid not in list_b:  # deleted from b
-                deletes_a.add(uid)
+                actions.append(('delete', uid, 'b', 'a'))
-            elif uid not in items_a and uid in items_b:  # deleted from a
+            elif uid not in list_a and uid in list_b:  # deleted from a
-                deletes_b.add(uid)
+                actions.append(('delete', uid, 'a', 'b'))
+
+    items_a = {}
+    items_b = {}
+    for item, uid, etag in storage_a.get_items(prefetch_items_from_a):
+        items_a[uid] = (item, etag)
+    for item, uid, etag in storage_b.get_items(prefetch_items_from_b):
+        items_b[uid] = (item, etag)
+
+    for action, uid, source, dest in actions:
+        source_storage = storage_a if source == 'a' else storage_b
+        dest_storage = storage_a if dest == 'a' else storage_b
+        source_items = items_a if source == 'a' else items_b
+        if action in ('upload', 'update'):
+            item, source_etag = source_items[uid]
+            if action == 'upload':
+                dest_etag = dest_storage.upload(item)
+            else:
+                dest_etag = dest_storage.update(item, etag)
+            status[uid] = (source_etag, dest_etag) if source == 'a' else (dest_etag, source_etag)
+        elif action == 'delete':
+            dest_storage.delete(uid)
+            del status[uid]

vdirsyncer/tests/test_sync.py

@@ -0,0 +1,43 @@
+from unittest import TestCase
+from vdirsyncer.storage.base import Item
+from vdirsyncer.storage.memory import MemoryStorage
+from vdirsyncer.sync import sync
+import vdirsyncer.exceptions as exceptions
+
+class SyncTests(TestCase):
+    def test_basic(self):
+        a = MemoryStorage()
+        b = MemoryStorage()
+        status = {}
+        sync(a, b, status)
+        assert len(status) == 0
+        assert list(a.list_items()) == []
+        assert list(b.list_items()) == []
+
+        item = Item('UID:1')
+        a.upload(item)
+        sync(a, b, status)
+        assert list(status) == ['1']
+        obj, uid, etag = next(b.get_items(['1']))
+        assert obj.raw == item.raw
+
+        item2 = Item('UID:2')
+        b.upload(item2)
+        b.delete('1')
+        sync(a, b, status)
+        assert list(status) == ['2']
+        assert next(a.list_items())[0] == '2'
+        assert next(b.list_items())[0] == '2'
+        obj, uid, etag = next(a.get_items(['2']))
+        assert obj.raw == item2.raw
+
+        new_item2 = Item('UID:2\nHUEHUEHUE:PRECISELY')
+        old_status = status.copy()
+        a.update(new_item2, next(a.list_items())[1])
+        sync(a, b, status)
+        assert status != old_status
+        assert list(status) == list(old_status)
+        assert next(a.list_items())[0] == '2'
+        assert next(b.list_items())[0] == '2'
+        obj, uid, etag = next(b.get_items(['2']))
+        assert obj.raw == new_item2.raw