Merge pull request #68 from untitaker/autodiscovery

Autodiscovery

docs/api.rst

@@ -48,7 +48,14 @@ Pair Section
 
 - ``collections``: Optional, a comma-separated list of collections to
   synchronize. If this parameter is omitted, it is assumed the storages are
-  already directly pointing to one collection each.
+  already directly pointing to one collection each. Specifying a collection
+  multiple times won't make vdirsyncer sync that collection more than once.
+
+  Furthermore, there are the special values ``from a`` and ``from b``, which
+  tell vdirsyncer to try autodiscovery on a specific storage::
+
+      collections = from b,foo,bar  # all in storage b + "foo" + "bar"
+      collections = from b,from a  # all in storage a + all in storage b
 
 - ``conflict_resolution``: Optional, define how conflicts should be handled.  A
   conflict occurs when one item changed on both sides since the last sync.

tests/test_cli.py

@@ -66,6 +66,48 @@ def test_storage_instance_from_config(monkeypatch):
     assert cli.storage_instance_from_config(config) == 'OK'
 
 
+def test_expand_collection(monkeypatch):
+    x = lambda *a: list(cli.expand_collection(*a))
+    assert x(None, 'foo', None, None) == ['foo']
+    assert x(None, 'from lol', None, None) == ['from lol']
+
+    all_pairs = {'mypair': ('my_a', 'my_b', None, {'lol': True})}
+    all_storages = {'my_a': {'type': 'mytype_a', 'is_a': True},
+                    'my_b': {'type': 'mytype_b', 'is_b': True}}
+
+    class TypeA(object):
+        @classmethod
+        def discover(cls, **config):
+            assert config == {
+                'is_a': True,
+                'lol': True
+            }
+            for i in range(1, 4):
+                s = cls()
+                s.collection = 'a{}'.format(i)
+                yield s
+
+    class TypeB(object):
+        @classmethod
+        def discover(cls, **config):
+            assert config == {
+                'is_b': True,
+                'lol': True
+            }
+            for i in range(1, 4):
+                s = cls()
+                s.collection = 'b{}'.format(i)
+                yield s
+
+    import vdirsyncer.storage
+    monkeypatch.setitem(vdirsyncer.storage.storage_names, 'mytype_a', TypeA)
+    monkeypatch.setitem(vdirsyncer.storage.storage_names, 'mytype_b', TypeB)
+
+    assert x('mypair', 'mycoll', all_pairs, all_storages) == ['mycoll']
+    assert x('mypair', 'from a', all_pairs, all_storages) == ['a1', 'a2', 'a3']
+    assert x('mypair', 'from b', all_pairs, all_storages) == ['b1', 'b2', 'b3']
+
+
 def test_parse_pairs_args():
     pairs = {
         'foo': ('bar', 'baz', {'conflict_resolution': 'a wins'},

vdirsyncer/cli.py

@@ -111,6 +111,15 @@ def save_status(path, status_name, status):
             f.write('\n')
 
 
+def storage_class_from_config(config):
+    config = dict(config)
+    storage_name = config.pop('type')
+    cls = storage_names.get(storage_name, None)
+    if cls is None:
+        raise KeyError('Unknown storage: {}'.format(storage_name))
+    return cls, config
+
+
 def storage_instance_from_config(config, description=None):
     '''
     :param config: A configuration dictionary to pass as kwargs to the class
@@ -118,9 +127,8 @@ def storage_instance_from_config(config, description=None):
     :param description: A name for the storage for debugging purposes
     '''
 
-    config = dict(config)
+    cls, config = storage_class_from_config(config)
-    storage_name = config.pop('type')
+
-    cls = storage_names[storage_name]
     try:
         return cls(**config)
     except Exception:
@@ -130,7 +138,7 @@ def storage_instance_from_config(config, description=None):
         invalid = given - all
 
         cli_logger.critical('error: Failed to initialize {}'
-                            .format(description or storage_name))
+                            .format(description or cls.storage_name))
 
         if not missing and not invalid:
             cli_logger.exception('')
@@ -138,16 +146,30 @@ def storage_instance_from_config(config, description=None):
         if missing:
             cli_logger.critical(
                 u'error: {} storage requires the parameters: {}'
-                .format(storage_name, u', '.join(missing)))
+                .format(cls.storage_name, u', '.join(missing)))
 
         if invalid:
             cli_logger.critical(
                 u'error: {} storage doesn\'t take the parameters: {}'
-                .format(storage_name, u', '.join(invalid)))
+                .format(cls.storage_name, u', '.join(invalid)))
 
         sys.exit(1)
 
 
+def expand_collection(pair, collection, all_pairs, all_storages):
+    if collection in ('from a', 'from b'):
+        a_name, b_name, _, storage_defaults = all_pairs[pair]
+        config = dict(storage_defaults)
+        if collection == 'from a':
+            config.update(all_storages[a_name])
+        else:
+            config.update(all_storages[b_name])
+        cls, config = storage_class_from_config(config)
+        return (s.collection for s in cls.discover(**config))
+    else:
+        return [collection]
+
+
 def main():
     env = os.environ
 
@@ -231,28 +253,35 @@ def _main(env, file_cfg):
         from the pair "bob".
         '''
         actions = []
+        handled_collections = set()
         force_delete = context.get('force_delete', set())
-        for pair_name, collection in parse_pairs_args(pairs, all_pairs):
+        for pair_name, _collection in parse_pairs_args(pairs, all_pairs):
-            a_name, b_name, pair_options, storage_defaults = \
+            for collection in expand_collection(pair_name, _collection,
-                all_pairs[pair_name]
+                                                all_pairs, all_storages):
+                if (pair_name, collection) in handled_collections:
+                    continue
+                handled_collections.add((pair_name, collection))
 
-            config_a = dict(storage_defaults)
+                a_name, b_name, pair_options, storage_defaults = \
-            config_a['collection'] = collection
+                    all_pairs[pair_name]
-            config_a.update(all_storages[a_name])
 
-            config_b = dict(storage_defaults)
+                config_a = dict(storage_defaults)
-            config_b['collection'] = collection
+                config_a['collection'] = collection
-            config_b.update(all_storages[b_name])
+                config_a.update(all_storages[a_name])
 
-            actions.append({
+                config_b = dict(storage_defaults)
-                'config_a': config_a,
+                config_b['collection'] = collection
-                'config_b': config_b,
+                config_b.update(all_storages[b_name])
-                'pair_name': pair_name,
+
-                'collection': collection,
+                actions.append({
-                'pair_options': pair_options,
+                    'config_a': config_a,
-                'general': general,
+                    'config_b': config_b,
-                'force_delete': force_delete
+                    'pair_name': pair_name,
-            })
+                    'collection': collection,
+                    'pair_options': pair_options,
+                    'general': general,
+                    'force_delete': force_delete
+                })
 
         processes = general.get('processes', 0) or len(actions)
         cli_logger.debug('Using {} processes.'.format(processes))

vdirsyncer/storage/__init__.py

@@ -17,10 +17,23 @@ from .filesystem import FilesystemStorage
 from .http import HttpStorage
 from .singlefile import SingleFileStorage
 
-storage_names = {
+
-    'caldav': CaldavStorage,
+def _generate_storage_dict(*classes):
-    'carddav': CarddavStorage,
+    rv = {}
-    'filesystem': FilesystemStorage,
+    for cls in classes:
-    'http': HttpStorage,
+        key = cls.storage_name
-    'singlefile': SingleFileStorage
+        assert key
-}
+        assert isinstance(key, str)
+        assert key not in rv
+        rv[key] = cls
+    return rv
+
+storage_names = _generate_storage_dict(
+    CaldavStorage,
+    CarddavStorage,
+    FilesystemStorage,
+    HttpStorage,
+    SingleFileStorage
+)
+
+del _generate_storage_dict

vdirsyncer/storage/base.py

@@ -69,6 +69,7 @@ class Storage(object):
         looked for.
     '''
     fileext = '.txt'
+    storage_name = None  # the name used in the config file
     _repr_attributes = ()
 
     @classmethod

vdirsyncer/storage/dav.py

@@ -287,6 +287,7 @@ class CaldavStorage(DavStorage):
         in the normal usecase.
     '''
 
+    storage_name = 'caldav'
     fileext = '.ics'
     item_mimetype = 'text/calendar'
     dav_header = 'calendar-access'
@@ -383,6 +384,7 @@ class CarddavStorage(DavStorage):
     CardDAV. Usable as ``carddav`` in the config file.
     ''' + DavStorage.__doc__
 
+    storage_name = 'carddav'
     fileext = '.vcf'
     item_mimetype = 'text/vcard'
     dav_header = 'addressbook'

vdirsyncer/storage/filesystem.py

@@ -36,6 +36,7 @@ class FilesystemStorage(Storage):
     :param create: Create directories if they don't exist.
     '''
 
+    storage_name = 'filesystem'
     _repr_attributes = ('path',)
 
     def __init__(self, path, fileext, collection=None, encoding='utf-8',
@@ -54,6 +55,7 @@ class FilesystemStorage(Storage):
     def discover(cls, path, **kwargs):
         if kwargs.pop('collection', None) is not None:
             raise TypeError('collection argument must not be given.')
+        path = expand_path(path)
         for collection in os.listdir(path):
             s = cls(path=path, collection=collection, **kwargs)
             yield s

vdirsyncer/storage/http.py

@@ -74,6 +74,7 @@ class HttpStorage(Storage):
         url = https://example.com/holidays_from_hicksville.ics
     '''
 
+    storage_name = 'http'
     _repr_attributes = ('username', 'url')
     _items = None
 

vdirsyncer/storage/singlefile.py

@@ -49,6 +49,7 @@ class SingleFileStorage(Storage):
 
     '''
 
+    storage_name = 'singlefile'
     _repr_attributes = ('path',)
 
     _write_mode = 'wb'