Add carddav storage

Fix #1

This currently breaks the tests, but i think it's radicale being a
bitch. We should test this against other servers.

config.example

@@ -2,22 +2,39 @@
 status_path=~/.vdirsyncer/status/
 #verbose = False
 
-[pair bob]
+# CONTACTS
-# This syncronizes only a single collection/calendar/addressbook
+[pair bob_contacts]
-a = bob_local
+# This syncronizes only a single calendar/addressbook
-b = bob_remote
+a = bob_contacts_local
+b = bob_contacts_remote
 # conflict_resolution = None  # abort when collisions occur
 # conflict_resolution = a wins  # assume a's items to be more up-to-date
 # conflict_resolution = b wins  # assume b's items to be more up-to-date
 
-[storage bob_local]
+[storage bob_contacts_local]
-# This represents only a single collection/calendar/addressbook
+# This represents only a single calendar/addressbook
 type = filesystem
 path = ~/.watdo/tasks/somecalendar/
-#fileext = .ics
+fileext = .vcf
 
-[storage bob_remote]
+[storage bob_contacts_remote]
-# This represents only a single collection/calendar/addressbook
+# This represents only a single calendar/addressbook
+type = carddav
+url = https://owncloud.example.com/remote.php/carddav/addressbooks/bob/default/
+username =  # blabla
+password =  # blabla
+
+# CALENDAR
+[pair bob_calendar]
+a = bob_calendar_local
+b = bob_calendar_remote
+
+[storage bob_calendar_local]
+type = filesystem
+path = ~/.watdo/tasks/somecalendar/
+fileext = .ics
+
+[storage bob_calendar_remote]
 type = caldav
 url = https://owncloud.example.com/remote.php/caldav/calendars/bob/somecalendar/
 username =  # blabla

vdirsyncer/storage/__init__.py

@@ -13,9 +13,11 @@
 '''
 
 from .dav.caldav import CaldavStorage
+from .dav.carddav import CarddavStorage
 from .filesystem import FilesystemStorage
 
 storage_names = {
     'caldav': CaldavStorage,
+    'carddav': CarddavStorage,
     'filesystem': FilesystemStorage
 }

vdirsyncer/storage/base.py

@@ -35,7 +35,7 @@ class Storage(object):
               object does.
     '''
     fileext = '.txt'
-    _repr_attributes = set()
+    _repr_attributes = ()
 
     def __init__(self, item_class=Item):
         self.item_class = item_class

vdirsyncer/storage/dav/base.py

@@ -11,10 +11,18 @@ from ..base import Storage
 import vdirsyncer.exceptions as exceptions
 import requests
 import urlparse
+from lxml import etree
 
 
 class DavStorage(Storage):
 
+    fileext = None
+    item_mimetype = None
+    dav_header = None
+    get_multi_template = None
+    get_multi_data_query = None
+    list_xml = None
+
     _session = None
     _repr_attributes = ('url', 'username')
 
@@ -51,14 +59,30 @@ class DavStorage(Storage):
 
         headers = self._default_headers()
         headers['Depth'] = 1
+        response = self._request(
+            'OPTIONS',
+            '',
+            headers=headers
+        )
+        response.raise_for_status()
+        if self.dav_header not in response.headers.get('DAV', ''):
+            raise exceptions.StorageError('URL is not a collection')
 
     def _simplify_href(self, href):
+        '''Used to strip hrefs off the collection's URL, to leave only the
+        filename.'''
         href = urlparse.urlparse(href).path
         if href.startswith(self.parsed_url.path):
             href = href[len(self.parsed_url.path):]
         assert '/' not in href, href
         return href
 
+    def _default_headers(self):
+        return {
+            'User-Agent': self.useragent,
+            'Content-Type': 'application/xml; charset=UTF-8'
+        }
+
     def _request(self, method, item, data=None, headers=None):
         if self._session is None:
             self._session = requests.session()
@@ -77,3 +101,120 @@ class DavStorage(Storage):
         ((actual_href, obj, etag),) = self.get_multi([href])
         assert href == actual_href
         return obj, etag
+
+    def get_multi(self, hrefs):
+        if not hrefs:
+            return ()
+
+        href_xml = []
+        for href in hrefs:
+            assert '/' not in href, href
+            href_xml.append('<D:href>{}</D:href>'.format(self.url + href))
+        data = self.get_multi_template.format(hrefs='\n'.join(href_xml))
+        response = self._request(
+            'REPORT',
+            '',
+            data=data,
+            headers=self._default_headers()
+        )
+        response.raise_for_status()
+        root = etree.XML(response.content)  # etree only can handle bytes
+        rv = []
+        hrefs_left = set(hrefs)
+        for element in root.iter('{DAV:}response'):
+            href = self._simplify_href(
+                element.find('{DAV:}href').text.decode(response.encoding))
+            obj = element \
+                .find('{DAV:}propstat') \
+                .find('{DAV:}prop') \
+                .find(self.get_multi_data_query).text
+            etag = element \
+                .find('{DAV:}propstat') \
+                .find('{DAV:}prop') \
+                .find('{DAV:}getetag').text
+            if isinstance(obj, bytes):
+                obj = obj.decode(response.encoding)
+            if isinstance(etag, bytes):
+                etag = etag.decode(response.encoding)
+            rv.append((href, Item(obj), etag))
+            hrefs_left.remove(href)
+        for href in hrefs_left:
+            raise exceptions.NotFoundError(href)
+        return rv
+
+    def has(self, href):
+        try:
+            self.get(href)
+        except exceptions.PreconditionFailed:
+            return False
+        else:
+            return True
+
+    def update(self, href, obj, etag):
+        headers = self._default_headers()
+        headers.update({
+            'Content-Type': self.item_mimetype,
+            'If-Match': etag
+        })
+        response = self._request(
+            'PUT',
+            href,
+            data=obj.raw,
+            headers=headers
+        )
+        self._check_response(response)
+
+        etag = response.headers.get('etag', None)
+        if not etag:
+            obj2, etag = self.get(href)
+            assert obj2.raw == obj.raw
+        return href, etag
+
+    def upload(self, obj):
+        href = self._get_href(obj.uid)
+        headers = self._default_headers()
+        headers.update({
+            'Content-Type': self.item_mimetype,
+            'If-None-Match': '*'
+        })
+        response = self._request(
+            'PUT',
+            href,
+            data=obj.raw,
+            headers=headers
+        )
+        self._check_response(response)
+
+        etag = response.headers.get('etag', None)
+        if not etag:
+            obj2, etag = self.get(href)
+            assert obj2.raw == obj.raw
+        return href, etag
+
+    def delete(self, href, etag):
+        headers = self._default_headers()
+        headers.update({
+            'If-Match': etag
+        })
+
+        response = self._request(
+            'DELETE',
+            href,
+            headers=headers
+        )
+        self._check_response(response)
+
+    def list(self):
+        response = self._request(
+            'REPORT',
+            '',
+            data=self.list_xml,
+            headers=self._default_headers()
+        )
+        response.raise_for_status()
+        root = etree.XML(response.content)
+        for element in root.iter('{DAV:}response'):
+            etag = element.find('{DAV:}propstat').find(
+                '{DAV:}prop').find('{DAV:}getetag').text
+            href = self._simplify_href(element.find('{DAV:}href').text)
+            yield href, etag

vdirsyncer/storage/dav/caldav.py

@@ -22,9 +22,23 @@ CONFIG_DT_FORMAT = '%Y-%m-%d'
 class CaldavStorage(DavStorage):
 
     fileext = '.ics'
+    item_mimetype = 'text/calendar'
+    dav_header = 'calendar-access'
     start_date = None
     end_date = None
 
+    get_multi_template = '''<?xml version="1.0" encoding="utf-8" ?>
+        <C:calendar-multiget xmlns:D="DAV:"
+            xmlns:C="urn:ietf:params:xml:ns:caldav">
+            <D:prop>
+                <D:getetag/>
+                <C:calendar-data/>
+            </D:prop>
+            {hrefs}
+        </C:calendar-multiget>'''
+
+    get_multi_data_query = '{urn:ietf:params:xml:ns:caldav}calendar-data'
+
     def __init__(self, start_date=None, end_date=None, **kwargs):
         '''
         :param start_date: Start date of timerange to show, default -inf.
@@ -43,24 +57,8 @@ class CaldavStorage(DavStorage):
                 (eval(end_date, namespace) if isinstance(end_date, str)
                  else end_date)
 
-        headers = self._default_headers()
+    @property
-        headers['Depth'] = 1
+    def list_xml(self):
-        response = self._request(
-            'OPTIONS',
-            '',
-            headers=headers
-        )
-        response.raise_for_status()
-        if 'calendar-access' not in response.headers.get('DAV', ''):
-            raise exceptions.StorageError('URL is not a CalDAV collection')
-
-    def _default_headers(self):
-        return {
-            'User-Agent': self.useragent,
-            'Content-Type': 'application/xml; charset=UTF-8'
-        }
-
-    def list(self):
         data = '''<?xml version="1.0" encoding="utf-8" ?>
             <C:calendar-query xmlns:D="DAV:"
                 xmlns:C="urn:ietf:params:xml:ns:caldav">
@@ -82,131 +80,5 @@ class CaldavStorage(DavStorage):
                             '<C:time-range start="{start}" end="{end}"/>'
                             '</C:comp-filter>').format(start=start,
                                                        end=end)
-            data = data.format(caldavfilter=caldavfilter)
+            return data.format(caldavfilter=caldavfilter)
-        else:
+        return data.format(caldavfilter='')
-            data = data.format(caldavfilter='')
-
-        response = self._request(
-            'REPORT',
-            '',
-            data=data,
-            headers=self._default_headers()
-        )
-        response.raise_for_status()
-        root = etree.XML(response.content)
-        for element in root.iter('{DAV:}response'):
-            etag = element.find('{DAV:}propstat').find(
-                '{DAV:}prop').find('{DAV:}getetag').text
-            href = self._simplify_href(element.find('{DAV:}href').text)
-            yield href, etag
-
-    def get_multi(self, hrefs):
-        if not hrefs:
-            return ()
-
-        data = '''<?xml version="1.0" encoding="utf-8" ?>
-            <C:calendar-multiget xmlns:D="DAV:"
-                xmlns:C="urn:ietf:params:xml:ns:caldav">
-                <D:prop>
-                    <D:getetag/>
-                    <C:calendar-data/>
-                </D:prop>
-                {hrefs}
-            </C:calendar-multiget>'''
-        href_xml = []
-        for href in hrefs:
-            assert '/' not in href, href
-            href_xml.append('<D:href>{}</D:href>'.format(self.url + href))
-        data = data.format(hrefs='\n'.join(href_xml))
-        response = self._request(
-            'REPORT',
-            '',
-            data=data,
-            headers=self._default_headers()
-        )
-        response.raise_for_status()
-        root = etree.XML(response.content)  # etree only can handle bytes
-        rv = []
-        hrefs_left = set(hrefs)
-        for element in root.iter('{DAV:}response'):
-            href = self._simplify_href(
-                element.find('{DAV:}href').text.decode(response.encoding))
-            obj = element \
-                .find('{DAV:}propstat') \
-                .find('{DAV:}prop') \
-                .find('{urn:ietf:params:xml:ns:caldav}calendar-data').text
-            etag = element \
-                .find('{DAV:}propstat') \
-                .find('{DAV:}prop') \
-                .find('{DAV:}getetag').text
-            if isinstance(obj, bytes):
-                obj = obj.decode(response.encoding)
-            if isinstance(etag, bytes):
-                etag = etag.decode(response.encoding)
-            rv.append((href, Item(obj), etag))
-            hrefs_left.remove(href)
-        for href in hrefs_left:
-            raise exceptions.NotFoundError(href)
-        return rv
-
-    def has(self, href):
-        try:
-            self.get(href)
-        except exceptions.PreconditionFailed:
-            return False
-        else:
-            return True
-
-    def upload(self, obj):
-        href = self._get_href(obj.uid)
-        headers = self._default_headers()
-        headers.update({
-            'Content-Type': 'text/calendar',
-            'If-None-Match': '*'
-        })
-        response = self._request(
-            'PUT',
-            href,
-            data=obj.raw,
-            headers=headers
-        )
-        self._check_response(response)
-
-        etag = response.headers.get('etag', None)
-        if not etag:
-            obj2, etag = self.get(href)
-            assert obj2.raw == obj.raw
-        return href, etag
-
-    def update(self, href, obj, etag):
-        headers = self._default_headers()
-        headers.update({
-            'Content-Type': 'text/calendar',
-            'If-Match': etag
-        })
-        response = self._request(
-            'PUT',
-            href,
-            data=obj.raw,
-            headers=headers
-        )
-        self._check_response(response)
-
-        etag = response.headers.get('etag', None)
-        if not etag:
-            obj2, etag = self.get(href)
-            assert obj2.raw == obj.raw
-        return href, etag
-
-    def delete(self, href, etag):
-        headers = self._default_headers()
-        headers.update({
-            'If-Match': etag
-        })
-
-        response = self._request(
-            'DELETE',
-            href,
-            headers=headers
-        )
-        self._check_response(response)

vdirsyncer/storage/dav/carddav.py

@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+'''
+    vdirsyncer.storage.dav.carddav
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Original version from pycarddav: https://github.com/geier/pycarddav
+
+    :copyright: (c) 2014 Markus Unterwaditzer, Christian Geier and contributors
+    :license: MIT, see LICENSE for more details.
+'''
+
+from .base import DavStorage
+from ..base import Item
+import vdirsyncer.exceptions as exceptions
+from lxml import etree
+
+
+class CarddavStorage(DavStorage):
+
+    fileext = '.vcf'
+    item_mimetype = 'text/vcard'
+    dav_header = 'addressbook'
+
+    get_multi_template = '''<?xml version="1.0" encoding="utf-8" ?>
+            <C:addressbook-multiget xmlns:D="DAV:"
+                    xmlns:C="urn:ietf:params:xml:ns:carddav">
+                <D:prop>
+                    <D:getetag/>
+                    <C:address-data/>
+                </D:prop>
+                {hrefs}
+            </C:addressbook-multiget>'''
+
+    get_multi_data_query = '{urn:ietf:params:xml:ns:carddav}address-data'
+
+
+    list_xml = '''<?xml version="1.0" encoding="utf-8" ?>
+        <C:addressbook-query xmlns:D="DAV:"
+                xmlns:C="urn:ietf:params:xml:ns:carddav">
+            <D:prop>
+                <D:getetag/>
+            </D:prop>
+            <C:filter>
+                <C:comp-filter name="VCARD"/>
+            </C:filter>
+        </C:addressbook-query>'''

vdirsyncer/tests/storage/__init__.py

@@ -25,12 +25,15 @@ class StorageTests(object):
         for i, item in enumerate(items):
             assert item.uid == unicode(i + 1), item.raw
         s = self._get_storage()
+        hrefs = []
         for item in items:
-            s.upload(item)
+            hrefs.append(s.upload(item))
-        hrefs = (href for href, etag in s.list())
+        hrefs.sort()
-        for href in hrefs:
+        assert hrefs == sorted(s.list())
+        for href, etag in hrefs:
             assert s.has(href)
-            obj, etag = s.get(href)
+            obj, etag2 = s.get(href)
+            assert etag == etag2
             assert 'UID:{}'.format(obj.uid) in obj.raw
 
     def test_upload_already_existing(self):
@@ -57,3 +60,9 @@ class StorageTests(object):
     def test_delete_nonexisting(self):
         s = self._get_storage()
         self.assertRaises(exceptions.PreconditionFailed, s.delete, '1', 123)
+
+    def test_list(self):
+        s = self._get_storage()
+        assert not list(s.list())
+        s.upload(self._create_bogus_item('1'))
+        assert list(s.list())

vdirsyncer/tests/storage/dav/__init__.py

@@ -2,7 +2,7 @@
 # -*- coding: utf-8 -*-
 '''
     vdirsyncer.tests.storage.dav
-    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
     Using an actual CalDAV/CardDAV server to test the CalDAV and CardDAV
     storages. Done by using Werkzeug's test client for WSGI apps. While this is
@@ -12,7 +12,6 @@
     :copyright: (c) 2014 Markus Unterwaditzer
     :license: MIT, see LICENSE for more details.
 '''
-__version__ = '0.1.0'
 
 import tempfile
 import shutil
@@ -72,6 +71,7 @@ class Response(object):
 class DavStorageTests(StorageTests):
     tmpdir = None
     storage_class = None
+    radicale_path = None
 
     def _get_storage(self, **kwargs):
         self.tmpdir = tempfile.mkdtemp()
@@ -82,12 +82,11 @@ class DavStorageTests(StorageTests):
 
         c = Client(app, WerkzeugResponse)
         server = 'http://127.0.0.1'
-        calendar_path = '/bob/test.ics/'
+        full_url = server + self.radicale_path
-        full_url = server + calendar_path
 
         def x(method, item, data=None, headers=None):
             assert '/' not in item
-            url = calendar_path + item
+            url = self.radicale_path + item
             r = c.open(path=url, method=method, data=data, headers=headers)
             r = Response(r)
             return r

vdirsyncer/tests/storage/dav/test_caldav.py

@@ -4,11 +4,6 @@
     vdirsyncer.tests.storage.test_caldav
     ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-    Using an actual CalDAV server to test the CalDAV storage. Done by using
-    Werkzeug's test client for WSGI apps. While this is pretty fast, Radicale
-    has so much global state such that a clean separation of the unit tests is
-    not guaranteed.
-
     :copyright: (c) 2014 Markus Unterwaditzer
     :license: MIT, see LICENSE for more details.
 '''
@@ -23,6 +18,7 @@ from . import DavStorageTests
 
 class CaldavStorageTests(TestCase, DavStorageTests):
     storage_class = CaldavStorage
+    radicale_path = '/bob/test.ics/'
 
     def _create_bogus_item(self, uid):
         return Item(u'BEGIN:VCALENDAR\n'

vdirsyncer/tests/storage/dav/test_carddav.py

@@ -0,0 +1,38 @@
+
+# -*- coding: utf-8 -*-
+'''
+    vdirsyncer.tests.storage.test_carddav
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: (c) 2014 Markus Unterwaditzer
+    :license: MIT, see LICENSE for more details.
+'''
+__version__ = '0.1.0'
+
+from unittest import TestCase
+
+from vdirsyncer.storage.base import Item
+from vdirsyncer.storage.dav.carddav import CarddavStorage
+from . import DavStorageTests
+
+
+class CarddavStorageTests(TestCase, DavStorageTests):
+    storage_class = CarddavStorage
+    radicale_path = '/bob/test.vcf/'
+
+    def _create_bogus_item(self, uid):
+        return Item(u'BEGIN:VCARD\n'
+                    u'VERSION:3.0\n'
+                    u'FN:Cyrus Daboo\n'
+                    u'N:Daboo;Cyrus\n'
+                    u'ADR;TYPE=POSTAL:;2822 Email HQ;'  # address continuing
+                    u'Suite 2821;RFCVille;PA;15213;USA\n'  # on next line
+                    u'EMAIL;TYPE=INTERNET,PREF:cyrus@example.com\n'
+                    u'NICKNAME:me\n'
+                    u'NOTE:Example VCard.\n'
+                    u'ORG:Self Employed\n'
+                    u'TEL;TYPE=WORK,VOICE:412 605 0499\n'
+                    u'TEL;TYPE=FAX:412 605 0705\n'
+                    u'URL:http://www.example.com\n'
+                    u'UID:{}\n'
+                    u'END:VCARD'.format(uid))