diff --git a/.gitignore b/.gitignore index dbb3ea0..4398cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,7 @@ htmlcov .coverage build env +owncloud-testserver +*.egg-info +.cache +.xprocess diff --git a/.travis.yml b/.travis.yml index d55a82c..dcae517 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,14 @@ language: python python: "2.7" env: - - DAV_SERVER=radicale RADICALE_STORAGE=filesystem - - DAV_SERVER=radicale RADICALE_STORAGE=database - - DAV_SERVER=radicale_git RADICALE_STORAGE=filesystem - - DAV_SERVER=radicale_git RADICALE_STORAGE=database + global: + - IS_TRAVIS=true + matrix: + - DAV_SERVER=radicale_filesystem REQUIREMENTS=release + - DAV_SERVER=radicale_filesystem REQUIREMENTS=devel + - DAV_SERVER=radicale_database REQUIREMENTS=release + - DAV_SERVER=radicale_database REQUIREMENTS=devel + - DAV_SERVER=owncloud REQUIREMENTS=release -install: - - "./install-deps.sh" - -script: py.test +install: "./install-deps.sh" +script: "./run-tests.sh tests/" diff --git a/README.rst b/README.rst index 56bd500..5d7059b 100644 --- a/README.rst +++ b/README.rst @@ -30,5 +30,17 @@ How to run the tests :: - sh install_deps.sh - py.test + sh install-deps.sh + sh run-tests.sh + +The environment variable ``DAV_SERVER`` specifies which CalDAV/CardDAV server +to test against. It has to be set for both scripts, ``install-deps.sh`` and +``run-tests.sh``. + + - ``DAV_SERVER=radicale``: The default, installs the latest Radicale release + from PyPI. Very fast, because no additional processes are needed. + - ``DAV_SERVER=radicale_git``: Same as ``radicale``, except that the + installation happens from their git repo. ``install-deps.sh`` is slightly + slower with this. + - ``DAV_SERVER=owncloud``: Uses latest ownCloud release. Very slow + installation, very slow tests. diff --git a/install-deps.sh b/install-deps.sh index b0e9048..0cea15c 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -1,23 +1,41 @@ #!/bin/sh -pip install --use-mirrors . +echo "The shell is $SHELL" +set -e +pip install --use-mirrors --editable . pip install --use-mirrors -r requirements.txt -[[ -z "$DAV_SERVER" ]] && DAV_SERVER=radicale -[[ -z "$RADICALE_STORAGE" ]] && RADICALE_STORAGE=filesystem +[ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem -davserver_radicale() { - pip install --use-mirrors radicale +davserver_radicale_filesystem() { radicale_deps } -davserver_radicale_git() { - pip install git+https://github.com/Kozea/Radicale.git +davserver_radicale_database() { radicale_deps + pip install --use-mirrors sqlalchemy pysqlite } -radicale_deps() { radicale_storage_$RADICALE_STORAGE; } +radicale_deps() { + if [ "$REQUIREMENTS" = "release" ]; then + radicale_pkg="radicale" + elif [ "$REQUIREMENTS" = "devel" ]; then + radicale_pkg="git+https://github.com/Kozea/Radicale.git" + else + false + fi + pip install --use-mirrors werkzeug $radicale_pkg +} -radicale_storage_database() { pip install --use-mirrors sqlalchemy pysqlite; } -radicale_storage_filesystem() { true; } +davserver_owncloud() { + # Maybe tmpfs is mounted on /tmp/, can't harm anyway. + if [ ! -d ./owncloud-testserver/ ]; then + git clone --depth=1 \ + https://github.com/untitaker/owncloud-testserver.git \ + /tmp/owncloud-testserver + ln -s /tmp/owncloud-testserver . + fi + cd ./owncloud-testserver/ + sh install.sh +} davserver_$DAV_SERVER diff --git a/requirements.txt b/requirements.txt index a9d3dfb..4395388 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -Werkzeug pytest +pytest-xprocess mock git+https://github.com/geier/leif diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 0000000..1c82674 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,4 @@ +#!/bin/sh +set -e +[ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem +exec py.test $@ diff --git a/tests/__init__.py b/tests/__init__.py index ef37da5..1b5706b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,16 @@ -def normalize_item(x): - return set(x for x in x.raw.splitlines() if - not x.startswith('X-RADICALE-NAME') and - not x.startswith('PRODID')) +def normalize_item(item): + # - X-RADICALE-NAME is used by radicale, because hrefs don't really exist + # in their filesystem backend + # - PRODID is changed by radicale for some reason after upload, but nobody + # cares about that anyway + rv = set() + for line in item.raw.splitlines(): + line = line.strip() + line = line.strip().split(u':', 1) + if line[0] in ('X-RADICALE-NAME', 'PRODID', 'REV'): + continue + rv.add(u':'.join(line)) + return rv def assert_item_equals(a, b): diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index e859914..3cdfab7 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -26,7 +26,9 @@ class StorageTests(object): raise NotImplementedError() def _get_storage(self): - return self.storage_class(**self.get_storage_args()) + s = self.storage_class(**self.get_storage_args()) + assert not list(s.list()) + return s def test_generic(self): items = map(self._create_bogus_item, range(1, 10)) @@ -98,7 +100,8 @@ class StorageTests(object): def test_discover(self): items = [] for i in range(4): - s = self.storage_class(**self.get_storage_args(collection=str(i))) + i += 1 + s = self.storage_class(**self.get_storage_args(collection='test' + str(i))) items.append(self._create_bogus_item(str(i))) s.upload(items[-1]) @@ -110,7 +113,7 @@ class StorageTests(object): assert item.raw in set(x.raw for x in items) def test_collection_arg(self): - s = self.storage_class(**self.get_storage_args(collection='asd')) + s = self.storage_class(**self.get_storage_args(collection='test2')) # Can't do stronger assertion because of radicale, which needs a # fileextension to guess the collection type. - assert 'asd' in s.collection + assert 'test2' in s.collection diff --git a/tests/storage/dav/__init__.py b/tests/storage/dav/__init__.py index 9d14194..a7bbd43 100644 --- a/tests/storage/dav/__init__.py +++ b/tests/storage/dav/__init__.py @@ -13,11 +13,14 @@ import os from .. import StorageTests import vdirsyncer.exceptions as exceptions from vdirsyncer.storage.base import Item +import requests.exceptions -dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale' -if dav_server in ('radicale', 'radicale_git'): +dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale_filesystem' +if dav_server.startswith('radicale_'): from ._radicale import ServerMixin +elif dav_server == 'owncloud': + from ._owncloud import ServerMixin else: raise RuntimeError('{} is not a known DAV server.'.format(dav_server)) @@ -28,6 +31,6 @@ class DavStorageTests(ServerMixin, StorageTests): s = self._get_storage() try: s.upload(item) - except exceptions.Error: + except (exceptions.Error, requests.exceptions.HTTPError): pass assert not list(s.list()) diff --git a/tests/storage/dav/_owncloud.py b/tests/storage/dav/_owncloud.py new file mode 100644 index 0000000..34f5e09 --- /dev/null +++ b/tests/storage/dav/_owncloud.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +''' + vdirsyncer.tests.storage.dav._owncloud + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Using utilities from paste to wrap the PHP application into WSGI. + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' + +from vdirsyncer.utils import expand_path +import subprocess +import os + +owncloud_repo = expand_path(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/')) + +class ServerMixin(object): + storage_class = None + wsgi_teardown = None + + def setup_method(self, method): + subprocess.check_call([os.path.join(owncloud_repo, 'install.sh')]) + + def get_storage_args(self, collection='test'): + url = 'http://127.0.0.1:8080' + if self.storage_class.fileext == '.vcf': + url += '/remote.php/carddav/addressbooks/asdf/' + elif self.storage_class.fileext == '.ics': + url += '/remote.php/caldav/calendars/asdf/' + else: + raise RuntimeError(self.storage_class.fileext) + if collection is not None: + # the following collections are setup in ownCloud + assert collection in ('test', 'test1', 'test2', 'test3', 'test4', + 'test5', 'test6', 'test7', 'test8', 'test9', + 'test10') + + return {'url': url, 'collection': collection, 'username': 'asdf', 'password': 'asdf'} diff --git a/tests/storage/dav/_radicale.py b/tests/storage/dav/_radicale.py index ab7b323..7aea147 100644 --- a/tests/storage/dav/_radicale.py +++ b/tests/storage/dav/_radicale.py @@ -57,6 +57,8 @@ create table property ( primary key (name, collection_path)); ''' +dav_server = os.environ.get('DAV_SERVER', 'radicale_filesystem') + def do_the_radicale_dance(tmpdir): # All of radicale is already global state, the cleanliness of the code and @@ -75,7 +77,7 @@ def do_the_radicale_dance(tmpdir): # Now we can set some basic configuration. radicale.config.set('rights', 'type', 'None') - if os.environ.get('RADICALE_STORAGE', 'filesystem') == 'filesystem': + if dav_server == 'radicale_filesystem': radicale.config.set('storage', 'type', 'filesystem') radicale.config.set('storage', 'filesystem_folder', tmpdir) else: @@ -105,6 +107,7 @@ class Response(object): self.text = x.get_data(as_text=True) self.headers = x.headers self.encoding = x.charset + self.reason = str(x.status) def raise_for_status(self): '''copied from requests itself''' @@ -113,10 +116,24 @@ class Response(object): raise HTTPError(str(self.status_code)) +def wsgi_setup(app): + c = Client(app, WerkzeugResponse) + def x(session, method, url, data=None, headers=None, **kw): + path = urlparse.urlparse(url).path + assert isinstance(data, bytes) or data is None + r = c.open(path=path, method=method, data=data, headers=headers) + r = Response(r) + return r + + p = mock.patch('requests.Session.request', new=x) + p.start() + return p.stop + + class ServerMixin(object): '''hrefs are paths without scheme or netloc''' storage_class = None - patcher = None + wsgi_teardown = None tmpdir = None def setup_method(self, method): @@ -124,18 +141,7 @@ class ServerMixin(object): do_the_radicale_dance(self.tmpdir) from radicale import Application app = Application() - - c = Client(app, WerkzeugResponse) - - def x(session, method, url, data=None, headers=None, **kw): - path = urlparse.urlparse(url).path - assert isinstance(data, bytes) or data is None - r = c.open(path=path, method=method, data=data, headers=headers) - r = Response(r) - return r - - self.patcher = p = mock.patch('requests.Session.request', new=x) - p.start() + self.wsgi_teardown = wsgi_setup(app) def get_storage_args(self, collection='test'): url = 'http://127.0.0.1/bob/' @@ -148,15 +154,6 @@ class ServerMixin(object): if self.tmpdir is not None: shutil.rmtree(self.tmpdir) self.tmpdir = None - if self.patcher is not None: - self.patcher.stop() - self.patcher = None - - def test_dav_broken_item(self): - item = Item(u'UID:1') - s = self._get_storage() - try: - s.upload(item) - except exceptions.Error: - pass - assert not list(s.list()) + if self.wsgi_teardown is not None: + self.wsgi_teardown() + self.wsgi_teardown = None diff --git a/tests/storage/dav/conftest.py b/tests/storage/dav/conftest.py new file mode 100644 index 0000000..3dead48 --- /dev/null +++ b/tests/storage/dav/conftest.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +''' + vdirsyncer.tests.storage.dav.conftest + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' + +import os +import pytest + +dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale_filesystem' +php_sh = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/php.sh')) + +if dav_server == 'owncloud': + @pytest.fixture(autouse=True) + def start_owncloud_server(xprocess): + def preparefunc(cwd): + return 'Listening on', ['sh', php_sh] + + xprocess.ensure('owncloud_server', preparefunc) diff --git a/tests/storage/dav/test_caldav.py b/tests/storage/dav/test_caldav.py index c9049ea..757981e 100644 --- a/tests/storage/dav/test_caldav.py +++ b/tests/storage/dav/test_caldav.py @@ -9,7 +9,9 @@ ''' +import requests.exceptions from vdirsyncer.storage.dav.caldav import CaldavStorage +import vdirsyncer.exceptions as exceptions from . import DavStorageTests @@ -61,8 +63,11 @@ class TestCaldavStorage(DavStorageTests): def test_item_types(self): kw = self.get_storage_args() s = self.storage_class(item_types=('VTODO',), **kw) - s.upload(self._create_bogus_item(1, item_template=EVENT_TEMPLATE)) - s.upload(self._create_bogus_item(5, item_template=EVENT_TEMPLATE)) + try: + s.upload(self._create_bogus_item(1, item_template=EVENT_TEMPLATE)) + s.upload(self._create_bogus_item(5, item_template=EVENT_TEMPLATE)) + except (exceptions.Error, requests.exceptions.HTTPError): + pass href, etag = \ s.upload(self._create_bogus_item(3, item_template=TASK_TEMPLATE)) ((href2, etag2),) = s.list() diff --git a/tests/storage/dav/test_carddav.py b/tests/storage/dav/test_carddav.py index e70c017..99c1771 100644 --- a/tests/storage/dav/test_carddav.py +++ b/tests/storage/dav/test_carddav.py @@ -21,11 +21,11 @@ class TestCarddavStorage(DavStorageTests): u'N:Daboo;Cyrus\n' u'ADR;TYPE=POSTAL:;2822 Email HQ;' # address continuing u'Suite 2821;RFCVille;PA;15213;USA\n' # on next line - u'EMAIL;TYPE=INTERNET,PREF:cyrus@example.com\n' + u'EMAIL;TYPE=INTERNET;TYPE=PREF:cyrus@example.com\n' u'NICKNAME:me\n' u'NOTE:Example VCard.\n' u'ORG:Self Employed\n' - u'TEL;TYPE=WORK,VOICE:412 605 0499\n' + u'TEL;TYPE=WORK;TYPE=VOICE:412 605 0499\n' u'TEL;TYPE=FAX:412 605 0705\n' u'URL:http://www.example.com\n' u'UID:{uid}\n' diff --git a/vdirsyncer/storage/dav/base.py b/vdirsyncer/storage/dav/base.py index 9e57aca..4a73a4e 100644 --- a/vdirsyncer/storage/dav/base.py +++ b/vdirsyncer/storage/dav/base.py @@ -127,7 +127,7 @@ class DavStorage(Storage): @staticmethod def _check_response(response): if response.status_code == 412: - raise exceptions.PreconditionFailed() + raise exceptions.PreconditionFailed(response.reason) response.raise_for_status() def get(self, href): @@ -206,7 +206,7 @@ class DavStorage(Storage): etag = response.headers.get('etag', None) if not etag: obj2, etag = self.get(href) - assert obj2.raw == obj.raw + assert obj2.uid == obj.uid return href, etag def update(self, href, obj, etag): @@ -232,6 +232,8 @@ class DavStorage(Storage): href, headers=headers ) + if response.status_code == 404: + raise exceptions.NotFoundError(href) self._check_response(response) def _list(self, xml):