Merge branch 'owncloud'

Fix #16

.gitignore

@@ -4,3 +4,7 @@ htmlcov
 .coverage
 build
 env
+owncloud-testserver
+*.egg-info
+.cache
+.xprocess

.travis.yml

@@ -1,12 +1,14 @@
 language: python
 python: "2.7"
 env:
-    - DAV_SERVER=radicale RADICALE_STORAGE=filesystem
+    global:
-    - DAV_SERVER=radicale RADICALE_STORAGE=database
+        - IS_TRAVIS=true
-    - DAV_SERVER=radicale_git RADICALE_STORAGE=filesystem
+    matrix:
-    - DAV_SERVER=radicale_git RADICALE_STORAGE=database
+        - DAV_SERVER=radicale_filesystem REQUIREMENTS=release
+        - DAV_SERVER=radicale_filesystem REQUIREMENTS=devel
+        - DAV_SERVER=radicale_database REQUIREMENTS=release
+        - DAV_SERVER=radicale_database REQUIREMENTS=devel
+        - DAV_SERVER=owncloud REQUIREMENTS=release
 
-install:
+install: "./install-deps.sh"
-    - "./install-deps.sh"
+script: "./run-tests.sh tests/"
-
-script: py.test

README.rst

@@ -30,5 +30,17 @@ How to run the tests
 
 ::
 
-    sh install_deps.sh
+    sh install-deps.sh
-    py.test
+    sh run-tests.sh
+
+The environment variable ``DAV_SERVER`` specifies which CalDAV/CardDAV server
+to test against. It has to be set for both scripts, ``install-deps.sh`` and
+``run-tests.sh``.
+
+  - ``DAV_SERVER=radicale``: The default, installs the latest Radicale release
+    from PyPI. Very fast, because no additional processes are needed.
+  - ``DAV_SERVER=radicale_git``: Same as ``radicale``, except that the
+    installation happens from their git repo. ``install-deps.sh`` is slightly
+    slower with this.
+  - ``DAV_SERVER=owncloud``: Uses latest ownCloud release. Very slow
+    installation, very slow tests.

install-deps.sh

@@ -1,23 +1,41 @@
 #!/bin/sh
-pip install --use-mirrors .
+echo "The shell is $SHELL"
+set -e
+pip install --use-mirrors --editable .
 pip install --use-mirrors -r requirements.txt
-[[ -z "$DAV_SERVER" ]] && DAV_SERVER=radicale
+[ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem
-[[ -z "$RADICALE_STORAGE" ]] && RADICALE_STORAGE=filesystem
 
-davserver_radicale() {
+davserver_radicale_filesystem() {
-    pip install --use-mirrors radicale
     radicale_deps
 }
 
-davserver_radicale_git() {
+davserver_radicale_database() {
-    pip install git+https://github.com/Kozea/Radicale.git
     radicale_deps
+    pip install --use-mirrors sqlalchemy pysqlite
 }
 
-radicale_deps() { radicale_storage_$RADICALE_STORAGE; }
+radicale_deps() {
+    if [ "$REQUIREMENTS" = "release" ]; then
+        radicale_pkg="radicale"
+    elif [ "$REQUIREMENTS" = "devel" ]; then
+        radicale_pkg="git+https://github.com/Kozea/Radicale.git"
+    else
+        false
+    fi
+    pip install --use-mirrors werkzeug $radicale_pkg
+}
 
-radicale_storage_database() { pip install --use-mirrors sqlalchemy pysqlite; }
+davserver_owncloud() {
-radicale_storage_filesystem() { true; }
+    # Maybe tmpfs is mounted on /tmp/, can't harm anyway.
+    if [ ! -d ./owncloud-testserver/ ]; then
+        git clone --depth=1 \
+            https://github.com/untitaker/owncloud-testserver.git \
+            /tmp/owncloud-testserver
+        ln -s /tmp/owncloud-testserver .
+    fi
+    cd ./owncloud-testserver/
+    sh install.sh
+}
 
 
 davserver_$DAV_SERVER

requirements.txt

@@ -1,4 +1,4 @@
-Werkzeug
 pytest
+pytest-xprocess
 mock
 git+https://github.com/geier/leif

run-tests.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+set -e
+[ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem
+exec py.test $@

tests/__init__.py

@@ -1,7 +1,16 @@
-def normalize_item(x):
+def normalize_item(item):
-    return set(x for x in x.raw.splitlines() if
+    # - X-RADICALE-NAME is used by radicale, because hrefs don't really exist
-               not x.startswith('X-RADICALE-NAME') and
+    #   in their filesystem backend
-               not x.startswith('PRODID'))
+    # - PRODID is changed by radicale for some reason after upload, but nobody
+    #   cares about that anyway
+    rv = set()
+    for line in item.raw.splitlines():
+        line = line.strip()
+        line = line.strip().split(u':', 1)
+        if line[0] in ('X-RADICALE-NAME', 'PRODID', 'REV'):
+            continue
+        rv.add(u':'.join(line))
+    return rv
 
 
 def assert_item_equals(a, b):

tests/storage/__init__.py

@@ -26,7 +26,9 @@ class StorageTests(object):
         raise NotImplementedError()
 
     def _get_storage(self):
-        return self.storage_class(**self.get_storage_args())
+        s = self.storage_class(**self.get_storage_args())
+        assert not list(s.list())
+        return s
 
     def test_generic(self):
         items = map(self._create_bogus_item, range(1, 10))
@@ -98,7 +100,8 @@ class StorageTests(object):
     def test_discover(self):
         items = []
         for i in range(4):
-            s = self.storage_class(**self.get_storage_args(collection=str(i)))
+            i += 1
+            s = self.storage_class(**self.get_storage_args(collection='test' + str(i)))
             items.append(self._create_bogus_item(str(i)))
             s.upload(items[-1])
 
@@ -110,7 +113,7 @@ class StorageTests(object):
             assert item.raw in set(x.raw for x in items)
 
     def test_collection_arg(self):
-        s = self.storage_class(**self.get_storage_args(collection='asd'))
+        s = self.storage_class(**self.get_storage_args(collection='test2'))
         # Can't do stronger assertion because of radicale, which needs a
         # fileextension to guess the collection type.
-        assert 'asd' in s.collection
+        assert 'test2' in s.collection

tests/storage/dav/__init__.py

@@ -13,11 +13,14 @@ import os
 from .. import StorageTests
 import vdirsyncer.exceptions as exceptions
 from vdirsyncer.storage.base import Item
+import requests.exceptions
 
 
-dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale'
+dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale_filesystem'
-if dav_server in ('radicale', 'radicale_git'):
+if dav_server.startswith('radicale_'):
     from ._radicale import ServerMixin
+elif dav_server == 'owncloud':
+    from ._owncloud import ServerMixin
 else:
     raise RuntimeError('{} is not a known DAV server.'.format(dav_server))
 
@@ -28,6 +31,6 @@ class DavStorageTests(ServerMixin, StorageTests):
         s = self._get_storage()
         try:
             s.upload(item)
-        except exceptions.Error:
+        except (exceptions.Error, requests.exceptions.HTTPError):
             pass
         assert not list(s.list())

tests/storage/dav/_owncloud.py

@@ -0,0 +1,39 @@
+# -*- coding: utf-8 -*-
+'''
+    vdirsyncer.tests.storage.dav._owncloud
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    Using utilities from paste to wrap the PHP application into WSGI.
+
+    :copyright: (c) 2014 Markus Unterwaditzer
+    :license: MIT, see LICENSE for more details.
+'''
+
+from vdirsyncer.utils import expand_path
+import subprocess
+import os
+
+owncloud_repo = expand_path(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/'))
+
+class ServerMixin(object):
+    storage_class = None
+    wsgi_teardown = None
+
+    def setup_method(self, method):
+        subprocess.check_call([os.path.join(owncloud_repo, 'install.sh')])
+
+    def get_storage_args(self, collection='test'):
+        url = 'http://127.0.0.1:8080'
+        if self.storage_class.fileext == '.vcf':
+            url += '/remote.php/carddav/addressbooks/asdf/'
+        elif self.storage_class.fileext == '.ics':
+            url += '/remote.php/caldav/calendars/asdf/'
+        else:
+            raise RuntimeError(self.storage_class.fileext)
+        if collection is not None:
+            # the following collections are setup in ownCloud
+            assert collection in ('test', 'test1', 'test2', 'test3', 'test4',
+                                  'test5', 'test6', 'test7', 'test8', 'test9',
+                                  'test10')
+
+        return {'url': url, 'collection': collection, 'username': 'asdf', 'password': 'asdf'}

tests/storage/dav/_radicale.py

@@ -57,6 +57,8 @@ create table property (
        primary key (name, collection_path));
 '''
 
+dav_server = os.environ.get('DAV_SERVER', 'radicale_filesystem')
+
 
 def do_the_radicale_dance(tmpdir):
     # All of radicale is already global state, the cleanliness of the code and
@@ -75,7 +77,7 @@ def do_the_radicale_dance(tmpdir):
     # Now we can set some basic configuration.
     radicale.config.set('rights', 'type', 'None')
 
-    if os.environ.get('RADICALE_STORAGE', 'filesystem') == 'filesystem':
+    if dav_server == 'radicale_filesystem':
         radicale.config.set('storage', 'type', 'filesystem')
         radicale.config.set('storage', 'filesystem_folder', tmpdir)
     else:
@@ -105,6 +107,7 @@ class Response(object):
         self.text = x.get_data(as_text=True)
         self.headers = x.headers
         self.encoding = x.charset
+        self.reason = str(x.status)
 
     def raise_for_status(self):
         '''copied from requests itself'''
@@ -113,10 +116,24 @@ class Response(object):
             raise HTTPError(str(self.status_code))
 
 
+def wsgi_setup(app):
+    c = Client(app, WerkzeugResponse)
+    def x(session, method, url, data=None, headers=None, **kw):
+        path = urlparse.urlparse(url).path
+        assert isinstance(data, bytes) or data is None
+        r = c.open(path=path, method=method, data=data, headers=headers)
+        r = Response(r)
+        return r
+
+    p = mock.patch('requests.Session.request', new=x)
+    p.start()
+    return p.stop
+
+
 class ServerMixin(object):
     '''hrefs are paths without scheme or netloc'''
     storage_class = None
-    patcher = None
+    wsgi_teardown = None
     tmpdir = None
 
     def setup_method(self, method):
@@ -124,18 +141,7 @@ class ServerMixin(object):
         do_the_radicale_dance(self.tmpdir)
         from radicale import Application
         app = Application()
-
+        self.wsgi_teardown = wsgi_setup(app)
-        c = Client(app, WerkzeugResponse)
-
-        def x(session, method, url, data=None, headers=None, **kw):
-            path = urlparse.urlparse(url).path
-            assert isinstance(data, bytes) or data is None
-            r = c.open(path=path, method=method, data=data, headers=headers)
-            r = Response(r)
-            return r
-
-        self.patcher = p = mock.patch('requests.Session.request', new=x)
-        p.start()
 
     def get_storage_args(self, collection='test'):
         url = 'http://127.0.0.1/bob/'
@@ -148,15 +154,6 @@ class ServerMixin(object):
         if self.tmpdir is not None:
             shutil.rmtree(self.tmpdir)
             self.tmpdir = None
-        if self.patcher is not None:
+        if self.wsgi_teardown is not None:
-            self.patcher.stop()
+            self.wsgi_teardown()
-            self.patcher = None
+            self.wsgi_teardown = None
-
-    def test_dav_broken_item(self):
-        item = Item(u'UID:1')
-        s = self._get_storage()
-        try:
-            s.upload(item)
-        except exceptions.Error:
-            pass
-        assert not list(s.list())

tests/storage/dav/conftest.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+'''
+    vdirsyncer.tests.storage.dav.conftest
+    ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+    :copyright: (c) 2014 Markus Unterwaditzer
+    :license: MIT, see LICENSE for more details.
+'''
+
+import os
+import pytest
+
+dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale_filesystem'
+php_sh = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/php.sh'))
+
+if dav_server == 'owncloud':
+    @pytest.fixture(autouse=True)
+    def start_owncloud_server(xprocess):
+        def preparefunc(cwd):
+            return 'Listening on', ['sh', php_sh]
+
+        xprocess.ensure('owncloud_server', preparefunc)

tests/storage/dav/test_caldav.py

@@ -9,7 +9,9 @@
 '''
 
 
+import requests.exceptions
 from vdirsyncer.storage.dav.caldav import CaldavStorage
+import vdirsyncer.exceptions as exceptions
 from . import DavStorageTests
 
 
@@ -61,8 +63,11 @@ class TestCaldavStorage(DavStorageTests):
     def test_item_types(self):
         kw = self.get_storage_args()
         s = self.storage_class(item_types=('VTODO',), **kw)
-        s.upload(self._create_bogus_item(1, item_template=EVENT_TEMPLATE))
+        try:
-        s.upload(self._create_bogus_item(5, item_template=EVENT_TEMPLATE))
+            s.upload(self._create_bogus_item(1, item_template=EVENT_TEMPLATE))
+            s.upload(self._create_bogus_item(5, item_template=EVENT_TEMPLATE))
+        except (exceptions.Error, requests.exceptions.HTTPError):
+            pass
         href, etag = \
             s.upload(self._create_bogus_item(3, item_template=TASK_TEMPLATE))
         ((href2, etag2),) = s.list()

tests/storage/dav/test_carddav.py

@@ -21,11 +21,11 @@ class TestCarddavStorage(DavStorageTests):
                      u'N:Daboo;Cyrus\n'
                      u'ADR;TYPE=POSTAL:;2822 Email HQ;'  # address continuing
                      u'Suite 2821;RFCVille;PA;15213;USA\n'  # on next line
-                     u'EMAIL;TYPE=INTERNET,PREF:cyrus@example.com\n'
+                     u'EMAIL;TYPE=INTERNET;TYPE=PREF:cyrus@example.com\n'
                      u'NICKNAME:me\n'
                      u'NOTE:Example VCard.\n'
                      u'ORG:Self Employed\n'
-                     u'TEL;TYPE=WORK,VOICE:412 605 0499\n'
+                     u'TEL;TYPE=WORK;TYPE=VOICE:412 605 0499\n'
                      u'TEL;TYPE=FAX:412 605 0705\n'
                      u'URL:http://www.example.com\n'
                      u'UID:{uid}\n'

vdirsyncer/storage/dav/base.py

@@ -127,7 +127,7 @@ class DavStorage(Storage):
     @staticmethod
     def _check_response(response):
         if response.status_code == 412:
-            raise exceptions.PreconditionFailed()
+            raise exceptions.PreconditionFailed(response.reason)
         response.raise_for_status()
 
     def get(self, href):
@@ -206,7 +206,7 @@ class DavStorage(Storage):
         etag = response.headers.get('etag', None)
         if not etag:
             obj2, etag = self.get(href)
-            assert obj2.raw == obj.raw
+            assert obj2.uid == obj.uid
         return href, etag
 
     def update(self, href, obj, etag):
@@ -232,6 +232,8 @@ class DavStorage(Storage):
             href,
             headers=headers
         )
+        if response.status_code == 404:
+            raise exceptions.NotFoundError(href)
         self._check_response(response)
 
     def _list(self, xml):