From 812d376c5d2c06533f6a0e6b1f8a6ae6f9ed8fc5 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Sat, 1 Mar 2014 23:35:57 +0100 Subject: [PATCH] Add carddav storage Fix #1 This currently breaks the tests, but i think it's radicale being a bitch. We should test this against other servers. --- config.example | 35 +++- vdirsyncer/storage/__init__.py | 2 + vdirsyncer/storage/base.py | 2 +- vdirsyncer/storage/dav/base.py | 141 ++++++++++++++++ vdirsyncer/storage/dav/caldav.py | 166 +++---------------- vdirsyncer/storage/dav/carddav.py | 46 +++++ vdirsyncer/tests/storage/__init__.py | 17 +- vdirsyncer/tests/storage/dav/__init__.py | 9 +- vdirsyncer/tests/storage/dav/test_caldav.py | 6 +- vdirsyncer/tests/storage/dav/test_carddav.py | 38 +++++ 10 files changed, 291 insertions(+), 171 deletions(-) create mode 100644 vdirsyncer/storage/dav/carddav.py create mode 100644 vdirsyncer/tests/storage/dav/test_carddav.py diff --git a/config.example b/config.example index eda4e3d..acb80a4 100644 --- a/config.example +++ b/config.example @@ -2,22 +2,39 @@ status_path=~/.vdirsyncer/status/ #verbose = False -[pair bob] -# This syncronizes only a single collection/calendar/addressbook -a = bob_local -b = bob_remote +# CONTACTS +[pair bob_contacts] +# This syncronizes only a single calendar/addressbook +a = bob_contacts_local +b = bob_contacts_remote # conflict_resolution = None # abort when collisions occur # conflict_resolution = a wins # assume a's items to be more up-to-date # conflict_resolution = b wins # assume b's items to be more up-to-date -[storage bob_local] -# This represents only a single collection/calendar/addressbook +[storage bob_contacts_local] +# This represents only a single calendar/addressbook type = filesystem path = ~/.watdo/tasks/somecalendar/ -#fileext = .ics +fileext = .vcf -[storage bob_remote] -# This represents only a single collection/calendar/addressbook +[storage bob_contacts_remote] +# This represents only a single calendar/addressbook +type = carddav +url = https://owncloud.example.com/remote.php/carddav/addressbooks/bob/default/ +username = # blabla +password = # blabla + +# CALENDAR +[pair bob_calendar] +a = bob_calendar_local +b = bob_calendar_remote + +[storage bob_calendar_local] +type = filesystem +path = ~/.watdo/tasks/somecalendar/ +fileext = .ics + +[storage bob_calendar_remote] type = caldav url = https://owncloud.example.com/remote.php/caldav/calendars/bob/somecalendar/ username = # blabla diff --git a/vdirsyncer/storage/__init__.py b/vdirsyncer/storage/__init__.py index 7b23370..8b09d39 100644 --- a/vdirsyncer/storage/__init__.py +++ b/vdirsyncer/storage/__init__.py @@ -13,9 +13,11 @@ ''' from .dav.caldav import CaldavStorage +from .dav.carddav import CarddavStorage from .filesystem import FilesystemStorage storage_names = { 'caldav': CaldavStorage, + 'carddav': CarddavStorage, 'filesystem': FilesystemStorage } diff --git a/vdirsyncer/storage/base.py b/vdirsyncer/storage/base.py index 2556520..24b1178 100644 --- a/vdirsyncer/storage/base.py +++ b/vdirsyncer/storage/base.py @@ -35,7 +35,7 @@ class Storage(object): object does. ''' fileext = '.txt' - _repr_attributes = set() + _repr_attributes = () def __init__(self, item_class=Item): self.item_class = item_class diff --git a/vdirsyncer/storage/dav/base.py b/vdirsyncer/storage/dav/base.py index b297cc8..eff0396 100644 --- a/vdirsyncer/storage/dav/base.py +++ b/vdirsyncer/storage/dav/base.py @@ -11,10 +11,18 @@ from ..base import Storage import vdirsyncer.exceptions as exceptions import requests import urlparse +from lxml import etree class DavStorage(Storage): + fileext = None + item_mimetype = None + dav_header = None + get_multi_template = None + get_multi_data_query = None + list_xml = None + _session = None _repr_attributes = ('url', 'username') @@ -51,14 +59,30 @@ class DavStorage(Storage): headers = self._default_headers() headers['Depth'] = 1 + response = self._request( + 'OPTIONS', + '', + headers=headers + ) + response.raise_for_status() + if self.dav_header not in response.headers.get('DAV', ''): + raise exceptions.StorageError('URL is not a collection') def _simplify_href(self, href): + '''Used to strip hrefs off the collection's URL, to leave only the + filename.''' href = urlparse.urlparse(href).path if href.startswith(self.parsed_url.path): href = href[len(self.parsed_url.path):] assert '/' not in href, href return href + def _default_headers(self): + return { + 'User-Agent': self.useragent, + 'Content-Type': 'application/xml; charset=UTF-8' + } + def _request(self, method, item, data=None, headers=None): if self._session is None: self._session = requests.session() @@ -77,3 +101,120 @@ class DavStorage(Storage): ((actual_href, obj, etag),) = self.get_multi([href]) assert href == actual_href return obj, etag + + def get_multi(self, hrefs): + if not hrefs: + return () + + href_xml = [] + for href in hrefs: + assert '/' not in href, href + href_xml.append('{}'.format(self.url + href)) + data = self.get_multi_template.format(hrefs='\n'.join(href_xml)) + response = self._request( + 'REPORT', + '', + data=data, + headers=self._default_headers() + ) + response.raise_for_status() + root = etree.XML(response.content) # etree only can handle bytes + rv = [] + hrefs_left = set(hrefs) + for element in root.iter('{DAV:}response'): + href = self._simplify_href( + element.find('{DAV:}href').text.decode(response.encoding)) + obj = element \ + .find('{DAV:}propstat') \ + .find('{DAV:}prop') \ + .find(self.get_multi_data_query).text + etag = element \ + .find('{DAV:}propstat') \ + .find('{DAV:}prop') \ + .find('{DAV:}getetag').text + if isinstance(obj, bytes): + obj = obj.decode(response.encoding) + if isinstance(etag, bytes): + etag = etag.decode(response.encoding) + rv.append((href, Item(obj), etag)) + hrefs_left.remove(href) + for href in hrefs_left: + raise exceptions.NotFoundError(href) + return rv + + def has(self, href): + try: + self.get(href) + except exceptions.PreconditionFailed: + return False + else: + return True + + def update(self, href, obj, etag): + headers = self._default_headers() + headers.update({ + 'Content-Type': self.item_mimetype, + 'If-Match': etag + }) + response = self._request( + 'PUT', + href, + data=obj.raw, + headers=headers + ) + self._check_response(response) + + etag = response.headers.get('etag', None) + if not etag: + obj2, etag = self.get(href) + assert obj2.raw == obj.raw + return href, etag + + def upload(self, obj): + href = self._get_href(obj.uid) + headers = self._default_headers() + headers.update({ + 'Content-Type': self.item_mimetype, + 'If-None-Match': '*' + }) + response = self._request( + 'PUT', + href, + data=obj.raw, + headers=headers + ) + self._check_response(response) + + etag = response.headers.get('etag', None) + if not etag: + obj2, etag = self.get(href) + assert obj2.raw == obj.raw + return href, etag + + def delete(self, href, etag): + headers = self._default_headers() + headers.update({ + 'If-Match': etag + }) + + response = self._request( + 'DELETE', + href, + headers=headers + ) + self._check_response(response) + + def list(self): + response = self._request( + 'REPORT', + '', + data=self.list_xml, + headers=self._default_headers() + ) + response.raise_for_status() + root = etree.XML(response.content) + for element in root.iter('{DAV:}response'): + etag = element.find('{DAV:}propstat').find( + '{DAV:}prop').find('{DAV:}getetag').text + href = self._simplify_href(element.find('{DAV:}href').text) + yield href, etag diff --git a/vdirsyncer/storage/dav/caldav.py b/vdirsyncer/storage/dav/caldav.py index 6c636d5..c14a711 100644 --- a/vdirsyncer/storage/dav/caldav.py +++ b/vdirsyncer/storage/dav/caldav.py @@ -22,9 +22,23 @@ CONFIG_DT_FORMAT = '%Y-%m-%d' class CaldavStorage(DavStorage): fileext = '.ics' + item_mimetype = 'text/calendar' + dav_header = 'calendar-access' start_date = None end_date = None + get_multi_template = ''' + + + + + + {hrefs} + ''' + + get_multi_data_query = '{urn:ietf:params:xml:ns:caldav}calendar-data' + def __init__(self, start_date=None, end_date=None, **kwargs): ''' :param start_date: Start date of timerange to show, default -inf. @@ -43,30 +57,14 @@ class CaldavStorage(DavStorage): (eval(end_date, namespace) if isinstance(end_date, str) else end_date) - headers = self._default_headers() - headers['Depth'] = 1 - response = self._request( - 'OPTIONS', - '', - headers=headers - ) - response.raise_for_status() - if 'calendar-access' not in response.headers.get('DAV', ''): - raise exceptions.StorageError('URL is not a CalDAV collection') - - def _default_headers(self): - return { - 'User-Agent': self.useragent, - 'Content-Type': 'application/xml; charset=UTF-8' - } - - def list(self): + @property + def list_xml(self): data = ''' - + {caldavfilter} @@ -82,131 +80,5 @@ class CaldavStorage(DavStorage): '' '').format(start=start, end=end) - data = data.format(caldavfilter=caldavfilter) - else: - data = data.format(caldavfilter='') - - response = self._request( - 'REPORT', - '', - data=data, - headers=self._default_headers() - ) - response.raise_for_status() - root = etree.XML(response.content) - for element in root.iter('{DAV:}response'): - etag = element.find('{DAV:}propstat').find( - '{DAV:}prop').find('{DAV:}getetag').text - href = self._simplify_href(element.find('{DAV:}href').text) - yield href, etag - - def get_multi(self, hrefs): - if not hrefs: - return () - - data = ''' - - - - - - {hrefs} - ''' - href_xml = [] - for href in hrefs: - assert '/' not in href, href - href_xml.append('{}'.format(self.url + href)) - data = data.format(hrefs='\n'.join(href_xml)) - response = self._request( - 'REPORT', - '', - data=data, - headers=self._default_headers() - ) - response.raise_for_status() - root = etree.XML(response.content) # etree only can handle bytes - rv = [] - hrefs_left = set(hrefs) - for element in root.iter('{DAV:}response'): - href = self._simplify_href( - element.find('{DAV:}href').text.decode(response.encoding)) - obj = element \ - .find('{DAV:}propstat') \ - .find('{DAV:}prop') \ - .find('{urn:ietf:params:xml:ns:caldav}calendar-data').text - etag = element \ - .find('{DAV:}propstat') \ - .find('{DAV:}prop') \ - .find('{DAV:}getetag').text - if isinstance(obj, bytes): - obj = obj.decode(response.encoding) - if isinstance(etag, bytes): - etag = etag.decode(response.encoding) - rv.append((href, Item(obj), etag)) - hrefs_left.remove(href) - for href in hrefs_left: - raise exceptions.NotFoundError(href) - return rv - - def has(self, href): - try: - self.get(href) - except exceptions.PreconditionFailed: - return False - else: - return True - - def upload(self, obj): - href = self._get_href(obj.uid) - headers = self._default_headers() - headers.update({ - 'Content-Type': 'text/calendar', - 'If-None-Match': '*' - }) - response = self._request( - 'PUT', - href, - data=obj.raw, - headers=headers - ) - self._check_response(response) - - etag = response.headers.get('etag', None) - if not etag: - obj2, etag = self.get(href) - assert obj2.raw == obj.raw - return href, etag - - def update(self, href, obj, etag): - headers = self._default_headers() - headers.update({ - 'Content-Type': 'text/calendar', - 'If-Match': etag - }) - response = self._request( - 'PUT', - href, - data=obj.raw, - headers=headers - ) - self._check_response(response) - - etag = response.headers.get('etag', None) - if not etag: - obj2, etag = self.get(href) - assert obj2.raw == obj.raw - return href, etag - - def delete(self, href, etag): - headers = self._default_headers() - headers.update({ - 'If-Match': etag - }) - - response = self._request( - 'DELETE', - href, - headers=headers - ) - self._check_response(response) + return data.format(caldavfilter=caldavfilter) + return data.format(caldavfilter='') diff --git a/vdirsyncer/storage/dav/carddav.py b/vdirsyncer/storage/dav/carddav.py new file mode 100644 index 0000000..87c6ae1 --- /dev/null +++ b/vdirsyncer/storage/dav/carddav.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +''' + vdirsyncer.storage.dav.carddav + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Original version from pycarddav: https://github.com/geier/pycarddav + + :copyright: (c) 2014 Markus Unterwaditzer, Christian Geier and contributors + :license: MIT, see LICENSE for more details. +''' + +from .base import DavStorage +from ..base import Item +import vdirsyncer.exceptions as exceptions +from lxml import etree + + +class CarddavStorage(DavStorage): + + fileext = '.vcf' + item_mimetype = 'text/vcard' + dav_header = 'addressbook' + + get_multi_template = ''' + + + + + + {hrefs} + ''' + + get_multi_data_query = '{urn:ietf:params:xml:ns:carddav}address-data' + + + list_xml = ''' + + + + + + + + ''' diff --git a/vdirsyncer/tests/storage/__init__.py b/vdirsyncer/tests/storage/__init__.py index 90970b5..a50b6cd 100644 --- a/vdirsyncer/tests/storage/__init__.py +++ b/vdirsyncer/tests/storage/__init__.py @@ -25,12 +25,15 @@ class StorageTests(object): for i, item in enumerate(items): assert item.uid == unicode(i + 1), item.raw s = self._get_storage() + hrefs = [] for item in items: - s.upload(item) - hrefs = (href for href, etag in s.list()) - for href in hrefs: + hrefs.append(s.upload(item)) + hrefs.sort() + assert hrefs == sorted(s.list()) + for href, etag in hrefs: assert s.has(href) - obj, etag = s.get(href) + obj, etag2 = s.get(href) + assert etag == etag2 assert 'UID:{}'.format(obj.uid) in obj.raw def test_upload_already_existing(self): @@ -57,3 +60,9 @@ class StorageTests(object): def test_delete_nonexisting(self): s = self._get_storage() self.assertRaises(exceptions.PreconditionFailed, s.delete, '1', 123) + + def test_list(self): + s = self._get_storage() + assert not list(s.list()) + s.upload(self._create_bogus_item('1')) + assert list(s.list()) diff --git a/vdirsyncer/tests/storage/dav/__init__.py b/vdirsyncer/tests/storage/dav/__init__.py index 4151042..28bca69 100644 --- a/vdirsyncer/tests/storage/dav/__init__.py +++ b/vdirsyncer/tests/storage/dav/__init__.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- ''' vdirsyncer.tests.storage.dav - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Using an actual CalDAV/CardDAV server to test the CalDAV and CardDAV storages. Done by using Werkzeug's test client for WSGI apps. While this is @@ -12,7 +12,6 @@ :copyright: (c) 2014 Markus Unterwaditzer :license: MIT, see LICENSE for more details. ''' -__version__ = '0.1.0' import tempfile import shutil @@ -72,6 +71,7 @@ class Response(object): class DavStorageTests(StorageTests): tmpdir = None storage_class = None + radicale_path = None def _get_storage(self, **kwargs): self.tmpdir = tempfile.mkdtemp() @@ -82,12 +82,11 @@ class DavStorageTests(StorageTests): c = Client(app, WerkzeugResponse) server = 'http://127.0.0.1' - calendar_path = '/bob/test.ics/' - full_url = server + calendar_path + full_url = server + self.radicale_path def x(method, item, data=None, headers=None): assert '/' not in item - url = calendar_path + item + url = self.radicale_path + item r = c.open(path=url, method=method, data=data, headers=headers) r = Response(r) return r diff --git a/vdirsyncer/tests/storage/dav/test_caldav.py b/vdirsyncer/tests/storage/dav/test_caldav.py index 205b919..159f87a 100644 --- a/vdirsyncer/tests/storage/dav/test_caldav.py +++ b/vdirsyncer/tests/storage/dav/test_caldav.py @@ -4,11 +4,6 @@ vdirsyncer.tests.storage.test_caldav ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - Using an actual CalDAV server to test the CalDAV storage. Done by using - Werkzeug's test client for WSGI apps. While this is pretty fast, Radicale - has so much global state such that a clean separation of the unit tests is - not guaranteed. - :copyright: (c) 2014 Markus Unterwaditzer :license: MIT, see LICENSE for more details. ''' @@ -23,6 +18,7 @@ from . import DavStorageTests class CaldavStorageTests(TestCase, DavStorageTests): storage_class = CaldavStorage + radicale_path = '/bob/test.ics/' def _create_bogus_item(self, uid): return Item(u'BEGIN:VCALENDAR\n' diff --git a/vdirsyncer/tests/storage/dav/test_carddav.py b/vdirsyncer/tests/storage/dav/test_carddav.py new file mode 100644 index 0000000..32cae74 --- /dev/null +++ b/vdirsyncer/tests/storage/dav/test_carddav.py @@ -0,0 +1,38 @@ + +# -*- coding: utf-8 -*- +''' + vdirsyncer.tests.storage.test_carddav + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' +__version__ = '0.1.0' + +from unittest import TestCase + +from vdirsyncer.storage.base import Item +from vdirsyncer.storage.dav.carddav import CarddavStorage +from . import DavStorageTests + + +class CarddavStorageTests(TestCase, DavStorageTests): + storage_class = CarddavStorage + radicale_path = '/bob/test.vcf/' + + def _create_bogus_item(self, uid): + return Item(u'BEGIN:VCARD\n' + u'VERSION:3.0\n' + u'FN:Cyrus Daboo\n' + u'N:Daboo;Cyrus\n' + u'ADR;TYPE=POSTAL:;2822 Email HQ;' # address continuing + u'Suite 2821;RFCVille;PA;15213;USA\n' # on next line + u'EMAIL;TYPE=INTERNET,PREF:cyrus@example.com\n' + u'NICKNAME:me\n' + u'NOTE:Example VCard.\n' + u'ORG:Self Employed\n' + u'TEL;TYPE=WORK,VOICE:412 605 0499\n' + u'TEL;TYPE=FAX:412 605 0705\n' + u'URL:http://www.example.com\n' + u'UID:{}\n' + u'END:VCARD'.format(uid))