Merge branch 'dav_discovery'

Fix #11

tests/storage/__init__.py

@@ -21,9 +21,12 @@ class StorageTests(object):
         item_template = item_template or self.item_template
         return Item(item_template.format(uid=uid, r=r))
 
-    def _get_storage(self, **kwargs):
+    def get_storage_args(self, collection=None):
         raise NotImplementedError()
 
+    def _get_storage(self):
+        return self.storage_class(**self.get_storage_args())
+
     def test_generic(self):
         items = map(self._create_bogus_item, range(1, 10))
         for i, item in enumerate(items):
@@ -88,3 +91,21 @@ class StorageTests(object):
         assert not list(s.list())
         s.upload(self._create_bogus_item('1'))
         assert list(s.list())
+
+    def test_discover(self):
+        items = []
+        for i in range(4):
+            s = self.storage_class(**self.get_storage_args(collection=str(i)))
+            items.append(self._create_bogus_item(str(i)))
+            s.upload(items[-1])
+
+        d = self.storage_class.discover(
+            **self.get_storage_args(collection=None))
+        for s in d:
+            ((href, etag),) = s.list()
+            item, etag = s.get(href)
+            assert item.raw in set(x.raw for x in items)
+
+    def test_collection_arg(self):
+        s = self.storage_class(**self.get_storage_args(collection='asd'))
+        assert s.collection == 'asd'

tests/storage/conftest.py

@@ -0,0 +1,6 @@
+import pytest
+
+
+@pytest.fixture
+def class_tmpdir(request, tmpdir):
+    request.instance.tmpdir = str(tmpdir)

tests/storage/dav/__init__.py

@@ -13,11 +13,11 @@
     :license: MIT, see LICENSE for more details.
 '''
 
-import tempfile
-import shutil
 import sys
 import os
 import urlparse
+import tempfile
+import shutil
 import mock
 
 from werkzeug.test import Client
@@ -104,6 +104,7 @@ class Response(object):
         self.x = x
         self.status_code = x.status_code
         self.content = x.get_data(as_text=False)
+        self.text = x.get_data(as_text=True)
         self.headers = x.headers
         self.encoding = x.charset
 
@@ -116,24 +117,20 @@ class Response(object):
 
 class DavStorageTests(StorageTests):
     '''hrefs are paths without scheme or netloc'''
-    tmpdir = None
     storage_class = None
-    radicale_path = None
     patcher = None
+    tmpdir = None
 
-    def _get_storage(self, **kwargs):
+    def setup_method(self, method):
         self.tmpdir = tempfile.mkdtemp()
-
         do_the_radicale_dance(self.tmpdir)
         from radicale import Application
         app = Application()
 
         c = Client(app, WerkzeugResponse)
-        server = 'http://127.0.0.1'
-        full_url = server + self.radicale_path
 
         def x(session, method, url, data=None, headers=None, **kw):
-            path = urlparse.urlparse(url).path or self.radicale_path
+            path = urlparse.urlparse(url).path
             assert isinstance(data, bytes) or data is None
             r = c.open(path=path, method=method, data=data, headers=headers)
             r = Response(r)
@@ -142,7 +139,9 @@ class DavStorageTests(StorageTests):
         self.patcher = p = mock.patch('requests.Session.request', new=x)
         p.start()
 
-        return self.storage_class(url=full_url, **kwargs)
+    def get_storage_args(self, collection=None):
+        url = 'http://127.0.0.1/bob/'
+        return {'url': url, 'collection': collection}
 
     def teardown_method(self, method):
         self.app = None

tests/storage/dav/test_caldav.py

@@ -10,7 +10,6 @@
 
 from unittest import TestCase
 
-from vdirsyncer.storage.base import Item
 from vdirsyncer.storage.dav.caldav import CaldavStorage
 from . import DavStorageTests
 
@@ -46,7 +45,6 @@ END:VCALENDAR'''
 
 class CaldavStorageTests(TestCase, DavStorageTests):
     storage_class = CaldavStorage
-    radicale_path = '/bob/test.ics/'
 
     item_template = TASK_TEMPLATE
 

tests/storage/dav/test_carddav.py

@@ -10,14 +10,12 @@
 
 from unittest import TestCase
 
-from vdirsyncer.storage.base import Item
 from vdirsyncer.storage.dav.carddav import CarddavStorage
 from . import DavStorageTests
 
 
 class CarddavStorageTests(TestCase, DavStorageTests):
     storage_class = CarddavStorage
-    radicale_path = '/bob/test.vcf/'
 
     item_template = (u'BEGIN:VCARD\n'
                      u'VERSION:3.0\n'

tests/storage/test_filesystem.py

@@ -9,20 +9,18 @@
 '''
 
 from unittest import TestCase
-import tempfile
+import pytest
-import shutil
+import os
 from vdirsyncer.storage.filesystem import FilesystemStorage
 from . import StorageTests
 
 
+@pytest.mark.usefixtures('class_tmpdir')
 class FilesystemStorageTests(TestCase, StorageTests):
-    tmpdir = None
+    storage_class = FilesystemStorage
 
-    def _get_storage(self, **kwargs):
+    def get_storage_args(self, collection=None):
-        path = self.tmpdir = tempfile.mkdtemp()
+        path = self.tmpdir
-        return FilesystemStorage(path=path, fileext='.txt', **kwargs)
+        if collection is not None:
-
+            os.makedirs(os.path.join(path, collection))
-    def teardown_method(self, method):
+        return {'path': path, 'fileext': '.txt', 'collection': collection}
-        if self.tmpdir is not None:
-            shutil.rmtree(self.tmpdir)
-            self.tmpdir = None

tests/storage/test_http.py

@@ -8,12 +8,10 @@
 '''
 
 import mock
-import requests
 from unittest import TestCase
-from . import StorageTests
 from .. import assert_item_equals
 from textwrap import dedent
-from vdirsyncer.storage.http import HttpStorage, Item, split_collection
+from vdirsyncer.storage.http import HttpStorage, Item
 
 
 class HttpStorageTests(TestCase):
@@ -31,7 +29,6 @@ class HttpStorageTests(TestCase):
                 METHOD:PUBLISH
                 BEGIN:VEVENT
                 UID:461092315540@example.com
-                ORGANIZER;CN="Alice Balder, Example Inc.":MAILTO:alice@example.com
                 LOCATION:Somewhere
                 SUMMARY:Eine Kurzinfo
                 DESCRIPTION:Beschreibung des Termines
@@ -48,7 +45,7 @@ class HttpStorageTests(TestCase):
                 BEGIN:VALARM
                 ACTION:AUDIO
                 TRIGGER:19980403T120000
-                ATTACH;FMTTYPE=audio/basic:http://host.com/pub/audio-files/ssbanner.aud
+                ATTACH;FMTTYPE=audio/basic:http://host.com/pub/ssbanner.aud
                 REPEAT:4
                 DURATION:PT1H
                 END:VALARM
@@ -68,7 +65,6 @@ class HttpStorageTests(TestCase):
             assert_item_equals(item, Item(dedent(u'''
                 BEGIN:VEVENT
                 UID:461092315540@example.com
-                ORGANIZER;CN="Alice Balder, Example Inc.":MAILTO:alice@example.com
                 LOCATION:Somewhere
                 SUMMARY:Eine Kurzinfo
                 DESCRIPTION:Beschreibung des Termines
@@ -89,7 +85,7 @@ class HttpStorageTests(TestCase):
                 BEGIN:VALARM
                 ACTION:AUDIO
                 TRIGGER:19980403T120000
-                ATTACH;FMTTYPE=audio/basic:http://host.com/pub/audio-files/ssbanner.aud
+                ATTACH;FMTTYPE=audio/basic:http://host.com/pub/ssbanner.aud
                 REPEAT:4
                 DURATION:PT1H
                 END:VALARM

tests/storage/test_memory.py

@@ -15,5 +15,13 @@ from . import StorageTests
 
 class MemoryStorageTests(TestCase, StorageTests):
 
-    def _get_storage(self, **kwargs):
+    storage_class = MemoryStorage
-        return MemoryStorage(**kwargs)
+
+    def get_storage_args(self, **kwargs):
+        return kwargs
+
+    def test_discover(self):
+        '''This test doesn't make any sense here.'''
+
+    def test_collection_arg(self):
+        '''This test doesn't make any sense here.'''

vdirsyncer/cli.py

@@ -144,7 +144,8 @@ def _main(env, file_cfg):
         actions = []
         for pair_name in pairs:
             try:
-                a_name, b_name, pair_options, storage_defaults = all_pairs[pair_name]
+                a_name, b_name, pair_options, storage_defaults = \
+                    all_pairs[pair_name]
             except KeyError:
                 cli_logger.critical('Pair not found: {}'.format(pair_name))
                 cli_logger.critical('These are the pairs found: ')

vdirsyncer/storage/base.py

@@ -30,15 +30,31 @@ class Storage(object):
 
     Terminology:
       - UID: Global identifier of the item, across storages.
-      - HREF: Per-storage identifier of item, might be UID.
+      - HREF: Per-storage identifier of item, might be UID. The reason items
+          aren't just referenced by their UID is because the CalDAV and CardDAV
+          specifications make this imperformant to implement.
       - ETAG: Checksum of item, or something similar that changes when the
-              object does.
+          object does.
+
+    :param collection: If None, the given URL or path is already directly
+        referring to a collection. Otherwise it will be treated as a basepath
+        to many collections (e.g. a vdir) and the given collection name will be
+        looked for.
     '''
     fileext = '.txt'
     _repr_attributes = ()
 
-    def __init__(self, item_class=Item):
+    @classmethod
-        self.item_class = item_class
+    def discover(cls, **kwargs):
+        '''
+        Discover collections given a basepath or -URL to many collections.
+        :param **kwargs: Keyword arguments to additionally pass to the storage
+            instances returned. You shouldn't pass `collection` here, otherwise
+            TypeError will be raised.
+        :returns: Iterable of storages which represent the discovered
+            collections, all of which are passed kwargs during initialization.
+        '''
+        raise NotImplementedError()
 
     def _get_href(self, uid):
         return uid + self.fileext

vdirsyncer/storage/dav/base.py

@@ -16,12 +16,20 @@ from lxml import etree
 
 class DavStorage(Storage):
 
+    # the file extension of items. Useful for testing against radicale.
     fileext = None
+    # mimetype of items
     item_mimetype = None
+    # The expected header for resource validation.
     dav_header = None
+    # XML to use when fetching multiple hrefs.
     get_multi_template = None
+    # The LXML query for extracting results in get_multi
     get_multi_data_query = None
-    list_xml = None
+    # The leif class to use for autodiscovery
+    # This should be the class *name* (i.e. "module attribute name") instead of
+    # the class, because leif is an optional dependency
+    leif_class = None
 
     _session = None
     _repr_attributes = ('url', 'username')
@@ -29,12 +37,13 @@ class DavStorage(Storage):
     def __init__(self, url, username='', password='', collection=None,
                  verify=True, auth='basic', useragent='vdirsyncer', **kwargs):
         '''
-        :param url: Direct URL for the CalDAV collection. No autodiscovery.
+        :param url: Base URL or an URL to a collection. Autodiscovery should be
+            done via :py:meth:`DavStorage.discover`.
         :param username: Username for authentication.
         :param password: Password for authentication.
         :param verify: Verify SSL certificate, default True.
         :param auth: Authentication method, from {'basic', 'digest'}, default
-                     'basic'.
+            'basic'.
         :param useragent: Default 'vdirsyncer'.
         '''
         super(DavStorage, self).__init__(**kwargs)
@@ -56,6 +65,7 @@ class DavStorage(Storage):
             url = urlparse.urljoin(url, collection)
         self.url = url.rstrip('/') + '/'
         self.parsed_url = urlparse.urlparse(self.url)
+        self.collection = collection
 
         headers = self._default_headers()
         headers['Depth'] = 1
@@ -68,6 +78,25 @@ class DavStorage(Storage):
         if self.dav_header not in response.headers.get('DAV', ''):
             raise exceptions.StorageError('URL is not a collection')
 
+    @classmethod
+    def discover(cls, url, **kwargs):
+        if kwargs.pop('collection', None) is not None:
+            raise TypeError('collection argument must not be given.')
+        from leif import leif
+        d = getattr(leif, cls.leif_class)(
+            url,
+            user=kwargs.get('username', None),
+            password=kwargs.get('password', None),
+            ssl_verify=kwargs.get('verify', True)
+        )
+        for c in d.discover():
+            collection = c['href']
+            if collection.startswith(url):
+                collection = collection[len(url):]
+            s = cls(url=url, collection=collection, **kwargs)
+            s.displayname = c['displayname']
+            yield s
+
     def _normalize_href(self, href):
         '''Normalize the href to be a path only relative to hostname and
         schema.'''
@@ -180,6 +209,8 @@ class DavStorage(Storage):
 
     def update(self, href, obj, etag):
         href = self._normalize_href(href)
+        if etag is None:
+            raise ValueError('etag must be given and must not be None.')
         return self._put(href, obj, etag)
 
     def upload(self, obj):

vdirsyncer/storage/dav/caldav.py

@@ -21,9 +21,10 @@ class CaldavStorage(DavStorage):
     fileext = '.ics'
     item_mimetype = 'text/calendar'
     dav_header = 'calendar-access'
+    leif_class = 'CalDiscover'
+
     start_date = None
     end_date = None
-    item_types = None
 
     get_multi_template = '''<?xml version="1.0" encoding="utf-8" ?>
         <C:calendar-multiget xmlns:D="DAV:"
@@ -42,6 +43,10 @@ class CaldavStorage(DavStorage):
         '''
         :param start_date: Start date of timerange to show, default -inf.
         :param end_date: End date of timerange to show, default +inf.
+        :param item_types: The item types to show from the server. Dependent on
+            server functionality, no clientside validation of results. This
+            currently only affects the `list` method, but this shouldn't cause
+            problems in the normal usecase.
         '''
         super(CaldavStorage, self).__init__(**kwargs)
         if isinstance(item_types, str):

vdirsyncer/storage/dav/carddav.py

@@ -17,6 +17,7 @@ class CarddavStorage(DavStorage):
     fileext = '.vcf'
     item_mimetype = 'text/vcard'
     dav_header = 'addressbook'
+    leif_class = 'CardDiscover'
 
     get_multi_template = '''<?xml version="1.0" encoding="utf-8" ?>
             <C:addressbook-multiget xmlns:D="DAV:"

vdirsyncer/storage/filesystem.py

@@ -23,16 +23,34 @@ class FilesystemStorage(Storage):
 
     _repr_attributes = ('path',)
 
-    def __init__(self, path, fileext, collection=None, encoding='utf-8', **kwargs):
+    def __init__(self, path, fileext, collection=None, encoding='utf-8',
+                 **kwargs):
         '''
-        :param path: Absolute path to a *collection* inside a vdir.
+        :param path: Absolute path to a vdir or collection, depending on the
+            collection parameter (see
+            :py:class:`vdirsyncer.storage.base.Storage`).
+        :param fileext: The file extension to use (e.g. `".txt"`). Contained in
+            the href, so if you change the file extension after a sync, this
+            will trigger a re-download of everything (but *should* not cause
+            data-loss of any kind).
+        :param encoding: File encoding for items.
         '''
-        self.path = expand_path(path)
+        super(FilesystemStorage, self).__init__(**kwargs)
         if collection is not None:
-            self.path = os.path.join(self.path, collection)
+            path = os.path.join(path, collection)
+        self.collection = collection
+        self.path = expand_path(path)
         self.encoding = encoding
         self.fileext = fileext
-        super(FilesystemStorage, self).__init__(**kwargs)
+
+    @classmethod
+    def discover(cls, path, **kwargs):
+        if kwargs.pop('collection', None) is not None:
+            raise TypeError('collection argument must not be given.')
+        for collection in os.listdir(path):
+            s = cls(path=path, collection=collection, **kwargs)
+            if next(s.list(), None) is not None:
+                yield s
 
     def _get_filepath(self, href):
         return os.path.join(self.path, href)