Autodiscovery

Fix #11

docs/api.rst

@@ -50,6 +50,12 @@ Pair Section
   synchronize. If this parameter is omitted, it is assumed the storages are
   already directly pointing to one collection each.
 
+  Furthermore, there are the special values ``from a`` and ``from b``, which
+  tell vdirsyncer to try autodiscovery on a specific storage::
+
+      collections = from b,foo,bar  # all in storage b + "foo" + "bar"
+      collections = from b,from a  # all in both storages
+
 - ``conflict_resolution``: Optional, define how conflicts should be handled.  A
   conflict occurs when one item changed on both sides since the last sync.
   Valid values are ``a wins`` and ``b wins``. By default, vdirsyncer will show

tests/test_cli.py

@@ -66,6 +66,50 @@ def test_storage_instance_from_config(monkeypatch):
     assert cli.storage_instance_from_config(config) == 'OK'
 
 
+def test_expand_collection(monkeypatch):
+    x = lambda *a: list(cli.expand_collection(*a))
+    assert x(None, 'foo', None, None) == ['foo']
+    assert x(None, 'from lol', None, None) == ['from lol']
+
+    all_pairs = {'mypair': ('my_a', 'my_b', None, {'lol': True})}
+    all_storages = {'my_a': {'type': 'mytype_a', 'is_a': True},
+                    'my_b': {'type': 'mytype_b', 'is_b': True}}
+
+    class TypeA(object):
+        @classmethod
+        def discover(cls, **config):
+            assert config == {
+                'is_a': True,
+                'lol': True
+            }
+            for i in range(1, 4):
+                s = cls()
+                s.collection = 'a{}'.format(i)
+                yield s
+
+
+    class TypeB(object):
+        @classmethod
+        def discover(cls, **config):
+            assert config == {
+                'is_b': True,
+                'lol': True
+            }
+            for i in range(1, 4):
+                s = cls()
+                s.collection = 'b{}'.format(i)
+                yield s
+
+
+    import vdirsyncer.storage
+    monkeypatch.setitem(vdirsyncer.storage.storage_names, 'mytype_a', TypeA)
+    monkeypatch.setitem(vdirsyncer.storage.storage_names, 'mytype_b', TypeB)
+
+    assert x('mypair', 'mycoll', all_pairs, all_storages) == ['mycoll']
+    assert x('mypair', 'from a', all_pairs, all_storages) == ['a1', 'a2', 'a3']
+    assert x('mypair', 'from b', all_pairs, all_storages) == ['b1', 'b2', 'b3']
+
+
 def test_parse_pairs_args():
     pairs = {
         'foo': ('bar', 'baz', {'conflict_resolution': 'a wins'},

vdirsyncer/cli.py

@@ -111,6 +111,15 @@ def save_status(path, status_name, status):
             f.write('\n')
 
 
+def storage_class_from_config(config):
+    config = dict(config)
+    storage_name = config.pop('type')
+    cls = storage_names.get(storage_name, None)
+    if cls is None:
+        raise KeyError('Unknown storage: {}'.format(storage_name))
+    return cls, config
+
+
 def storage_instance_from_config(config, description=None):
     '''
     :param config: A configuration dictionary to pass as kwargs to the class
@@ -118,9 +127,8 @@ def storage_instance_from_config(config, description=None):
     :param description: A name for the storage for debugging purposes
     '''
 
-    config = dict(config)
-    storage_name = config.pop('type')
-    cls = storage_names[storage_name]
+    cls, config = storage_class_from_config(config)
+
     try:
         return cls(**config)
     except Exception:
@@ -148,6 +156,20 @@ def storage_instance_from_config(config, description=None):
         sys.exit(1)
 
 
+def expand_collection(pair, collection, all_pairs, all_storages):
+    if collection in ('from a', 'from b'):
+        a_name, b_name, _, storage_defaults = all_pairs[pair]
+        config = dict(storage_defaults)
+        if collection == 'from a':
+            config.update(all_storages[a_name])
+        else:
+            config.update(all_storages[b_name])
+        cls, config = storage_class_from_config(config)
+        return (s.collection for s in cls.discover(**config))
+    else:
+        return [collection]
+
+
 def main():
     env = os.environ
 
@@ -231,28 +253,35 @@ def _main(env, file_cfg):
         from the pair "bob".
         '''
         actions = []
+        handled_collections = set()
         force_delete = context.get('force_delete', set())
-        for pair_name, collection in parse_pairs_args(pairs, all_pairs):
-            a_name, b_name, pair_options, storage_defaults = \
-                all_pairs[pair_name]
+        for pair_name, _collection in parse_pairs_args(pairs, all_pairs):
+            for collection in expand_collection(pair_name, _collection,
+                                                all_pairs, all_storages):
+                if (pair_name, collection) in handled_collections:
+                    continue
+                handled_collections.add((pair_name, collection))
 
-            config_a = dict(storage_defaults)
-            config_a['collection'] = collection
-            config_a.update(all_storages[a_name])
+                a_name, b_name, pair_options, storage_defaults = \
+                    all_pairs[pair_name]
 
-            config_b = dict(storage_defaults)
-            config_b['collection'] = collection
-            config_b.update(all_storages[b_name])
+                config_a = dict(storage_defaults)
+                config_a['collection'] = collection
+                config_a.update(all_storages[a_name])
 
-            actions.append({
-                'config_a': config_a,
-                'config_b': config_b,
-                'pair_name': pair_name,
-                'collection': collection,
-                'pair_options': pair_options,
-                'general': general,
-                'force_delete': force_delete
-            })
+                config_b = dict(storage_defaults)
+                config_b['collection'] = collection
+                config_b.update(all_storages[b_name])
+
+                actions.append({
+                    'config_a': config_a,
+                    'config_b': config_b,
+                    'pair_name': pair_name,
+                    'collection': collection,
+                    'pair_options': pair_options,
+                    'general': general,
+                    'force_delete': force_delete
+                })
 
         processes = general.get('processes', 0) or len(actions)
         cli_logger.debug('Using {} processes.'.format(processes))

vdirsyncer/storage/filesystem.py

@@ -54,6 +54,7 @@ class FilesystemStorage(Storage):
     def discover(cls, path, **kwargs):
         if kwargs.pop('collection', None) is not None:
             raise TypeError('collection argument must not be given.')
+        path = expand_path(path)
         for collection in os.listdir(path):
             s = cls(path=path, collection=collection, **kwargs)
             yield s