This commit is contained in:
Markus Unterwaditzer 2014-02-26 20:46:17 +01:00
parent 784c0edb5d
commit 98e49de418
12 changed files with 71 additions and 21 deletions

View file

@ -11,26 +11,31 @@ import os
import ConfigParser
from vdirsyncer.sync import sync_classes
def _path(p):
p = os.path.expanduser(p)
p = os.path.abspath(p)
return p
def get_config_parser(env):
fname = env.get('VDIRSYNCER_CONFIG', _path('~/.vdirsyncer/config'))
c = ConfigParser.SafeConfigParser()
c.read(fname)
return dict((c, c.items(c)) for c in c.sections())
def main():
env = os.environ
cfg = get_config_parser(env)
_main(env, cfg)
def _main(env, file_cfg):
app = argvard.Argvard()
sync = argvard.Command()
@sync_command.main('[accounts...]')
def sync_main(accounts=None):
if accounts is None:

View file

@ -9,10 +9,12 @@
class Error(Exception):
'''Baseclass for all errors.'''
class PreconditionFailed(Error):
'''
- The item doesn't exist although it should
- The item exists although it shouldn't
@ -24,16 +26,20 @@ class PreconditionFailed(Error):
class NotFoundError(PreconditionFailed):
'''Item not found'''
class AlreadyExistingError(PreconditionFailed):
'''Item already exists'''
class WrongEtagError(PreconditionFailed):
'''Wrong etag'''
class StorageError(Error):
'''Internal or initialization errors with storage.'''

View file

@ -7,8 +7,11 @@
:license: MIT, see LICENSE for more details.
'''
class Item(object):
'''should-be-immutable wrapper class for VCALENDAR and VCARD'''
def __init__(self, raw):
self.raw = raw
self._uid = None
@ -23,15 +26,17 @@ class Item(object):
class Storage(object):
'''Superclass of all storages, mainly useful to summarize the interface to
implement.
Terminology:
- UID: Global identifier of the item, across storages.
- HREF: Per-storage identifier of item, might be UID.
- ETAG: Checksum of item, or something similar that changes when the object does
'''
fileext = '.txt'
def __init__(self, item_class=Item):
self.item_class = item_class

View file

@ -17,10 +17,13 @@ import datetime
CALDAV_DT_FORMAT = '%Y%m%dT%H%M%SZ'
class CaldavStorage(Storage):
'''hrefs are full URLs to items'''
_session = None
fileext = '.ics'
def __init__(self, url, username='', password='', start_date=None,
end_date=None, verify=True, auth='basic',
useragent='vdirsyncer', _request_func=None, **kwargs):
@ -106,7 +109,7 @@ class CaldavStorage(Storage):
data = data.format(caldavfilter=caldavfilter)
else:
data = data.format(caldavfilter='')
response = self._request(
'REPORT',
'',
@ -116,7 +119,8 @@ class CaldavStorage(Storage):
response.raise_for_status()
root = etree.XML(response.content)
for element in root.iter('{DAV:}response'):
etag = element.find('{DAV:}propstat').find('{DAV:}prop').find('{DAV:}getetag').text
etag = element.find('{DAV:}propstat').find(
'{DAV:}prop').find('{DAV:}getetag').text
href = self._simplify_href(element.find('{DAV:}href').text)
yield href, etag
@ -214,7 +218,7 @@ class CaldavStorage(Storage):
if response.status_code == 412:
raise exceptions.PreconditionFailed(response.content)
response.raise_for_status()
etag = response.headers.get('etag', None)
if not etag:
obj2, etag = self.get(href)

View file

@ -11,10 +11,13 @@ import os
from vdirsyncer.storage.base import Storage, Item
import vdirsyncer.exceptions as exceptions
class FilesystemStorage(Storage):
'''Saves data in vdir collection
mtime is etag
filename without path is href'''
def __init__(self, path, fileext, **kwargs):
'''
:param path: Absolute path to a *collection* inside a vdir.

View file

@ -8,13 +8,16 @@
'''
import datetime
from vdirsyncer.storage.base import Item, Storage
from vdirsyncer.storage.base import Item, Storage
import vdirsyncer.exceptions as exceptions
class MemoryStorage(Storage):
'''
Saves data in RAM, only useful for testing.
'''
def __init__(self, **kwargs):
self.items = {} # href => (etag, object)
super(MemoryStorage, self).__init__(**kwargs)

View file

@ -16,7 +16,7 @@
def prepare_list(storage, href_to_uid):
for href, etag in storage.list():
props = {'etag': etag}
props = {'etag': etag}
if href in href_to_uid:
props['uid'] = href_to_uid[href]
else:
@ -49,9 +49,12 @@ def sync(storage_a, storage_b, status):
modified by the function and should be passed to it at the next sync.
If this is the first sync, an empty dictionary should be provided.
'''
a_href_to_uid = dict((href_a, uid) for uid, (href_a, etag_a, href_b, etag_b) in status.iteritems())
b_href_to_uid = dict((href_b, uid) for uid, (href_a, etag_a, href_b, etag_b) in status.iteritems())
list_a = dict(prepare_list(storage_a, a_href_to_uid)) # href => {'etag': etag, 'obj': optional object, 'uid': uid}
a_href_to_uid = dict((href_a, uid)
for uid, (href_a, etag_a, href_b, etag_b) in status.iteritems())
b_href_to_uid = dict((href_b, uid)
for uid, (href_a, etag_a, href_b, etag_b) in status.iteritems())
# href => {'etag': etag, 'obj': optional object, 'uid': uid}
list_a = dict(prepare_list(storage_a, a_href_to_uid))
list_b = dict(prepare_list(storage_b, b_href_to_uid))
a_uid_to_href = dict((x['uid'], href) for href, x in list_a.iteritems())
b_uid_to_href = dict((x['uid'], href) for href, x in list_b.iteritems())
@ -59,7 +62,7 @@ def sync(storage_a, storage_b, status):
actions, prefetch_from_a, prefetch_from_b = \
get_actions(list_a, list_b, status, a_uid_to_href, b_uid_to_href)
prefetch(storage_a, list_a, prefetch_from_a)
prefetch(storage_b, list_b, prefetch_from_b)
@ -72,7 +75,7 @@ def sync(storage_a, storage_b, status):
for action, uid, source, dest in actions:
source_storage, source_list, source_uid_to_href = storages[source]
dest_storage, dest_list, dest_uid_to_href = storages[dest]
if action in ('upload', 'update'):
source_href = source_uid_to_href[uid]
source_etag = source_list[source_href]['etag']
@ -86,7 +89,8 @@ def sync(storage_a, storage_b, status):
dest_etag = dest_storage.update(dest_href, obj, old_etag)
source_status = (source_href, source_etag)
dest_status = (dest_href, dest_etag)
status[uid] = source_status + dest_status if source == 'a' else dest_status + source_status
status[uid] = source_status + dest_status if source == 'a' else \
dest_status + source_status
elif action == 'delete':
if dest is not None:
dest_href = dest_uid_to_href[uid]
@ -94,6 +98,7 @@ def sync(storage_a, storage_b, status):
dest_storage.delete(dest_href, dest_etag)
del status[uid]
def get_actions(list_a, list_b, status, a_uid_to_href, b_uid_to_href):
prefetch_from_a = []
prefetch_from_b = []
@ -110,19 +115,21 @@ def get_actions(list_a, list_b, status, a_uid_to_href, b_uid_to_href):
if uid in uids_a and uid in uids_b: # missing status
# TODO: might need some kind of diffing too?
if a['obj'].raw != b['obj'].raw:
1/0
1 / 0
status[uid] = (href_a, a['etag'], href_b, b['etag'])
elif uid in uids_a and uid not in uids_b: # new item was created in a
# new item was created in a
elif uid in uids_a and uid not in uids_b:
prefetch_from_a.append(href_a)
actions.append(('upload', uid, 'a', 'b'))
elif uid not in uids_a and uid in uids_b: # new item was created in b
# new item was created in b
elif uid not in uids_a and uid in uids_b:
prefetch_from_b.append(href_b)
actions.append(('upload', uid, 'b', 'a'))
else:
_, status_etag_a, _, status_etag_b = status[uid]
if uid in uids_a and uid in uids_b:
if a['etag'] != status_etag_a and b['etag'] != status_etag_b:
1/0 # conflict resolution TODO
1 / 0 # conflict resolution TODO
elif a['etag'] != status_etag_a: # item was updated in a
prefetch_from_a.append(href_a)
actions.append(('update', uid, 'a', 'b'))
@ -135,6 +142,7 @@ def get_actions(list_a, list_b, status, a_uid_to_href, b_uid_to_href):
actions.append(('delete', uid, None, 'a'))
elif uid not in uids_a and uid in uids_b: # was deleted from a
actions.append(('delete', uid, None, 'b'))
elif uid not in uids_a and uid not in uids_b: # was deleted from a and b
# was deleted from a and b
elif uid not in uids_a and uid not in uids_b:
actions.append(('delete', uid, None, None))
return actions, prefetch_from_a, prefetch_from_b

View file

@ -18,7 +18,9 @@ from vdirsyncer.storage.memory import MemoryStorage
from vdirsyncer.storage.caldav import CaldavStorage
import vdirsyncer.exceptions as exceptions
class StorageTests(object):
def _create_bogus_item(self, uid):
return Item('''BEGIN:VCALENDAR
VERSION:2.0
@ -33,6 +35,7 @@ UID:{}
END:VTODO
END:VCALENDAR
'''.format(uid))
def _get_storage(self, **kwargs):
raise NotImplementedError()
@ -56,14 +59,17 @@ END:VCALENDAR
def test_update_nonexisting(self):
s = self._get_storage()
item = self._create_bogus_item(1)
self.assertRaises(exceptions.PreconditionFailed, s.update, 'huehue', item, 123)
self.assertRaises(
exceptions.PreconditionFailed, s.update, 'huehue', item, 123)
def test_wrong_etag(self):
s = self._get_storage()
obj = self._create_bogus_item(1)
href, etag = s.upload(obj)
self.assertRaises(exceptions.PreconditionFailed, s.update, href, obj, 'lolnope')
self.assertRaises(exceptions.PreconditionFailed, s.delete, href, 'lolnope')
self.assertRaises(
exceptions.PreconditionFailed, s.update, href, obj, 'lolnope')
self.assertRaises(
exceptions.PreconditionFailed, s.delete, href, 'lolnope')
def test_delete_nonexisting(self):
s = self._get_storage()

View file

@ -33,7 +33,9 @@ radicale.log.start()
class Response(object):
'''Fake API of requests module'''
def __init__(self, x):
self.x = x
self.status_code = x.status_code

View file

@ -18,6 +18,7 @@ from . import StorageTests
class FilesystemStorageTests(TestCase, StorageTests):
tmpdir = None
def _get_storage(self, **kwargs):
path = self.tmpdir = tempfile.mkdtemp()
return FilesystemStorage(path=path, fileext='.txt', **kwargs)

View file

@ -13,6 +13,8 @@ from unittest import TestCase
from vdirsyncer.storage.memory import MemoryStorage
from . import StorageTests
class MemoryStorageTests(TestCase, StorageTests):
def _get_storage(self, **kwargs):
return MemoryStorage(**kwargs)

View file

@ -13,10 +13,13 @@ from vdirsyncer.storage.memory import MemoryStorage
from vdirsyncer.sync import sync
import vdirsyncer.exceptions as exceptions
def empty_storage(x):
return list(x.list()) == []
class SyncTests(TestCase):
def test_irrelevant_status(self):
a = MemoryStorage()
b = MemoryStorage()
@ -101,7 +104,9 @@ class SyncTests(TestCase):
item = Item('UID:1')
a.upload(item)
b.upload(item)
status = {'1': ('1.txt', a.get('1.txt')[1], '1.txt', b.get('1.txt')[1])}
status = {
'1': ('1.txt', a.get('1.txt')[1], '1.txt', b.get('1.txt')[1])
}
old_status = dict(status)
a.update = b.update = a.upload = b.upload = \
lambda *a, **kw: self.fail('Method shouldn\'t have been called.')