get_multi now ignores duplicate input

tests/storage/__init__.py

@@ -63,6 +63,12 @@ class BaseStorageTests(object):
     def test_empty_get_multi(self, s):
         assert list(s.get_multi([])) == []
 
+    def test_get_multi_duplicates(self, s, get_item):
+        href, etag = s.upload(get_item())
+        (href2, item, etag2), = s.get_multi([href] * 2)
+        assert href2 == href
+        assert etag2 == etag
+
     def test_upload_already_existing(self, s, get_item):
         item = get_item()
         s.upload(item)

vdirsyncer/storage/base.py

@@ -9,7 +9,8 @@
 
 
 from .. import exceptions
-from vdirsyncer.utils.vobject import Item  # noqa
+from ..utils import uniq
+from ..utils.vobject import Item  # noqa
 
 
 class Storage(object):
@@ -101,7 +102,7 @@ class Storage(object):
         raise NotImplementedError()
 
     def get_multi(self, hrefs):
-        '''Fetch multiple items.
+        '''Fetch multiple items. Duplicate hrefs must be ignored.
 
         Functionally similar to :py:meth:`get`, but might bring performance
         benefits on some storages when used cleverly.
@@ -111,7 +112,7 @@ class Storage(object):
             items couldn't be found.
         :returns: iterable of (href, item, etag)
         '''
-        for href in hrefs:
+        for href in uniq(hrefs):
             item, etag = self.get(href)
             yield href, item, etag
 

vdirsyncer/storage/dav.py

@@ -16,6 +16,7 @@ from requests import session as requests_session
 from .base import Item, Storage
 from .http import USERAGENT, prepare_auth, prepare_verify
 from .. import exceptions, log, utils
+from ..utils import uniq
 
 
 dav_logger = log.get(__name__)
@@ -300,13 +301,12 @@ class DavStorage(Storage):
         return item, etag
 
     def get_multi(self, hrefs):
-        if not hrefs:
-            return ()
-        hrefs = [self._normalize_href(href) for href in hrefs]
-
         href_xml = []
-        for href in hrefs:
+        for href in uniq(hrefs):
             href_xml.append('<D:href>{}</D:href>'.format(href))
+        if not href_xml:
+            return ()
+
         data = self.get_multi_template.format(hrefs='\n'.join(href_xml))
         response = self.session.request(
             'REPORT',

vdirsyncer/utils/__init__.py

@@ -57,6 +57,14 @@ def split_sequence(s, f):
     return a, b
 
 
+def uniq(s):
+    d = set()
+    for x in s:
+        if x not in d:
+            d.add(x)
+            yield x
+
+
 def parse_config_value(value):
     if value in ('on', 'yes'):
         logger.warning('{} is deprecated for the config, please use true.\n'