From d61c4fc31b3e8bad9102e8675eaba5d8ced47848 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Mon, 17 Mar 2014 22:43:41 +0100 Subject: [PATCH 01/20] WIP --- install-deps.sh | 8 ++++++ tests/storage/dav/__init__.py | 2 ++ tests/storage/dav/_owncloud.py | 50 ++++++++++++++++++++++++++++++++++ tests/storage/dav/_radicale.py | 44 +++++++++++++----------------- 4 files changed, 79 insertions(+), 25 deletions(-) create mode 100644 tests/storage/dav/_owncloud.py diff --git a/install-deps.sh b/install-deps.sh index b0e9048..e2b4446 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -20,4 +20,12 @@ radicale_storage_database() { pip install --use-mirrors sqlalchemy pysqlite; } radicale_storage_filesystem() { true; } +davserver_owncloud() { + pip install paste + git clone git@github.com:untitaker/owncloud-testserver.git + cd ./owncloud-testserver/ + sh install.sh +} + + davserver_$DAV_SERVER diff --git a/tests/storage/dav/__init__.py b/tests/storage/dav/__init__.py index 9d14194..5a26664 100644 --- a/tests/storage/dav/__init__.py +++ b/tests/storage/dav/__init__.py @@ -18,6 +18,8 @@ from vdirsyncer.storage.base import Item dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale' if dav_server in ('radicale', 'radicale_git'): from ._radicale import ServerMixin +elif dav_server == 'owncloud': + from ._owncloud import ServerMixin else: raise RuntimeError('{} is not a known DAV server.'.format(dav_server)) diff --git a/tests/storage/dav/_owncloud.py b/tests/storage/dav/_owncloud.py new file mode 100644 index 0000000..a42601e --- /dev/null +++ b/tests/storage/dav/_owncloud.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +''' + vdirsyncer.tests.storage.dav._owncloud + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + Using utilities from paste to wrap the PHP application into WSGI. + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' + +from paste.cgiapp import CGIApplication +from vdirsyncer.utils import expand_path +from ._radicale import wsgi_setup +import subprocess +import os + +owncloud_repo = expand_path(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/')) +app = CGIApplication(None, 'php.cgi', [owncloud_repo], include_os_environ=False, query_string=None) + +def middleware(environ, start_response): + print(environ) + environ['REQUEST_URI'] = 'http://127.0.0.1' + environ['PATH_INFO'] + return app(environ, start_response) + +class ServerMixin(object): + storage_class = None + wsgi_teardown = None + + def setup_method(self, method): + subprocess.call([os.path.join(owncloud_repo, 'install.sh')]) + self.wsgi_teardown = wsgi_setup(middleware) + + def get_storage_args(self, collection='test'): + assert self.storage_class.fileext in ('.ics', '.vcf') + if self.storage_class.fileext == '.vcf': + url = 'http://127.0.0.1/remote.php/carddav/addressbooks/asdf/' + else: + url = 'http://127.0.0.1/remote.php/carddav/addressbooks/asdf/' + if collection is not None: + assert collection in ('test', 'test1', 'test2', 'test3', 'test4', + 'test5', 'test6', 'test7', 'test8', 'test9', + 'test10') + + return {'url': url, 'collection': collection} + + def teardown_method(self, method): + if self.wsgi_teardown is not None: + self.wsgi_teardown() + self.wsgi_teardown = None diff --git a/tests/storage/dav/_radicale.py b/tests/storage/dav/_radicale.py index ab7b323..80317f9 100644 --- a/tests/storage/dav/_radicale.py +++ b/tests/storage/dav/_radicale.py @@ -113,10 +113,24 @@ class Response(object): raise HTTPError(str(self.status_code)) +def wsgi_setup(app): + c = Client(app, WerkzeugResponse) + def x(session, method, url, data=None, headers=None, **kw): + path = urlparse.urlparse(url).path + assert isinstance(data, bytes) or data is None + r = c.open(path=path, method=method, data=data, headers=headers) + r = Response(r) + return r + + p = mock.patch('requests.Session.request', new=x) + p.start() + return p.stop + + class ServerMixin(object): '''hrefs are paths without scheme or netloc''' storage_class = None - patcher = None + wsgi_teardown = None tmpdir = None def setup_method(self, method): @@ -124,18 +138,7 @@ class ServerMixin(object): do_the_radicale_dance(self.tmpdir) from radicale import Application app = Application() - - c = Client(app, WerkzeugResponse) - - def x(session, method, url, data=None, headers=None, **kw): - path = urlparse.urlparse(url).path - assert isinstance(data, bytes) or data is None - r = c.open(path=path, method=method, data=data, headers=headers) - r = Response(r) - return r - - self.patcher = p = mock.patch('requests.Session.request', new=x) - p.start() + self.wsgi_teardown = wsgi_setup(app) def get_storage_args(self, collection='test'): url = 'http://127.0.0.1/bob/' @@ -148,15 +151,6 @@ class ServerMixin(object): if self.tmpdir is not None: shutil.rmtree(self.tmpdir) self.tmpdir = None - if self.patcher is not None: - self.patcher.stop() - self.patcher = None - - def test_dav_broken_item(self): - item = Item(u'UID:1') - s = self._get_storage() - try: - s.upload(item) - except exceptions.Error: - pass - assert not list(s.list()) + if self.wsgi_teardown is not None: + self.wsgi_teardown() + self.wsgi_teardown = None From 21e694c98c2df4df005c0e2e55ca167b1b286584 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 18 Mar 2014 16:59:30 +0100 Subject: [PATCH 02/20] More fixes --- install-deps.sh | 1 - run-tests.sh | 17 +++++++++++++++++ tests/storage/__init__.py | 6 +++--- tests/storage/dav/__init__.py | 3 ++- tests/storage/dav/_owncloud.py | 22 +++++----------------- 5 files changed, 27 insertions(+), 22 deletions(-) create mode 100644 run-tests.sh diff --git a/install-deps.sh b/install-deps.sh index e2b4446..cbd1524 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -21,7 +21,6 @@ radicale_storage_filesystem() { true; } davserver_owncloud() { - pip install paste git clone git@github.com:untitaker/owncloud-testserver.git cd ./owncloud-testserver/ sh install.sh diff --git a/run-tests.sh b/run-tests.sh new file mode 100644 index 0000000..046fb68 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,17 @@ +#!/bin/sh +[[ -z "$DAV_SERVER" ]] && DAV_SERVER=radicale +[[ -z "$RADICALE_STORAGE" ]] && RADICALE_STORAGE=filesystem + +davserver_radicale() { true; } +davserver_radicale_git() { true; } + +davserver_owncloud() { + sh ./owncloud-testserver/php.sh +} + + +davserver_$DAV_SERVER & +DAVSERVER_PID=$! +py.test ./tests/ +kill -9 $DAVSERVER_PID +wait diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index e859914..38e24c9 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -98,7 +98,7 @@ class StorageTests(object): def test_discover(self): items = [] for i in range(4): - s = self.storage_class(**self.get_storage_args(collection=str(i))) + s = self.storage_class(**self.get_storage_args(collection='test' + str(i+1))) items.append(self._create_bogus_item(str(i))) s.upload(items[-1]) @@ -110,7 +110,7 @@ class StorageTests(object): assert item.raw in set(x.raw for x in items) def test_collection_arg(self): - s = self.storage_class(**self.get_storage_args(collection='asd')) + s = self.storage_class(**self.get_storage_args(collection='test2')) # Can't do stronger assertion because of radicale, which needs a # fileextension to guess the collection type. - assert 'asd' in s.collection + assert 'test2' in s.collection diff --git a/tests/storage/dav/__init__.py b/tests/storage/dav/__init__.py index 5a26664..e8a31bb 100644 --- a/tests/storage/dav/__init__.py +++ b/tests/storage/dav/__init__.py @@ -13,6 +13,7 @@ import os from .. import StorageTests import vdirsyncer.exceptions as exceptions from vdirsyncer.storage.base import Item +import requests.exceptions dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale' @@ -30,6 +31,6 @@ class DavStorageTests(ServerMixin, StorageTests): s = self._get_storage() try: s.upload(item) - except exceptions.Error: + except (exceptions.Error, requests.exceptions.HTTPError): pass assert not list(s.list()) diff --git a/tests/storage/dav/_owncloud.py b/tests/storage/dav/_owncloud.py index a42601e..a9ce3ae 100644 --- a/tests/storage/dav/_owncloud.py +++ b/tests/storage/dav/_owncloud.py @@ -9,19 +9,11 @@ :license: MIT, see LICENSE for more details. ''' -from paste.cgiapp import CGIApplication from vdirsyncer.utils import expand_path -from ._radicale import wsgi_setup import subprocess import os owncloud_repo = expand_path(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/')) -app = CGIApplication(None, 'php.cgi', [owncloud_repo], include_os_environ=False, query_string=None) - -def middleware(environ, start_response): - print(environ) - environ['REQUEST_URI'] = 'http://127.0.0.1' + environ['PATH_INFO'] - return app(environ, start_response) class ServerMixin(object): storage_class = None @@ -29,22 +21,18 @@ class ServerMixin(object): def setup_method(self, method): subprocess.call([os.path.join(owncloud_repo, 'install.sh')]) - self.wsgi_teardown = wsgi_setup(middleware) def get_storage_args(self, collection='test'): assert self.storage_class.fileext in ('.ics', '.vcf') + url = 'http://127.0.0.1:8080' if self.storage_class.fileext == '.vcf': - url = 'http://127.0.0.1/remote.php/carddav/addressbooks/asdf/' + url += '/remote.php/carddav/addressbooks/asdf/' else: - url = 'http://127.0.0.1/remote.php/carddav/addressbooks/asdf/' + url += '/remote.php/carddav/addressbooks/asdf/' if collection is not None: + # the following collections are setup in ownCloud assert collection in ('test', 'test1', 'test2', 'test3', 'test4', 'test5', 'test6', 'test7', 'test8', 'test9', 'test10') - return {'url': url, 'collection': collection} - - def teardown_method(self, method): - if self.wsgi_teardown is not None: - self.wsgi_teardown() - self.wsgi_teardown = None + return {'url': url, 'collection': collection, 'username': 'asdf', 'password': 'asdf'} From a1e3ff7fbbc24807e058a4fdf93cf47df2bef19a Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 18 Mar 2014 16:59:49 +0100 Subject: [PATCH 03/20] Add owncloud testserver to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dbb3ea0..50e5c1d 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ htmlcov .coverage build env +owncloud-testserver From f341c4a294b8febf0df706ec59b06263bde3dddc Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 18 Mar 2014 17:13:40 +0100 Subject: [PATCH 04/20] More fixes for owncloud --- run-tests.sh | 7 +++---- tests/storage/__init__.py | 3 ++- tests/storage/dav/_owncloud.py | 5 +++-- vdirsyncer/storage/dav/base.py | 2 ++ 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/run-tests.sh b/run-tests.sh index 046fb68..fb3f78d 100644 --- a/run-tests.sh +++ b/run-tests.sh @@ -10,8 +10,7 @@ davserver_owncloud() { } +# while it would be nice if the server was cleanly shut down, it's not really a +# problem either davserver_$DAV_SERVER & -DAVSERVER_PID=$! -py.test ./tests/ -kill -9 $DAVSERVER_PID -wait +py.test $@ diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 38e24c9..90646fb 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -98,7 +98,8 @@ class StorageTests(object): def test_discover(self): items = [] for i in range(4): - s = self.storage_class(**self.get_storage_args(collection='test' + str(i+1))) + i += 1 + s = self.storage_class(**self.get_storage_args(collection='test' + str(i))) items.append(self._create_bogus_item(str(i))) s.upload(items[-1]) diff --git a/tests/storage/dav/_owncloud.py b/tests/storage/dav/_owncloud.py index a9ce3ae..b059d65 100644 --- a/tests/storage/dav/_owncloud.py +++ b/tests/storage/dav/_owncloud.py @@ -23,12 +23,13 @@ class ServerMixin(object): subprocess.call([os.path.join(owncloud_repo, 'install.sh')]) def get_storage_args(self, collection='test'): - assert self.storage_class.fileext in ('.ics', '.vcf') url = 'http://127.0.0.1:8080' if self.storage_class.fileext == '.vcf': url += '/remote.php/carddav/addressbooks/asdf/' - else: + elif self.storage_class.fileext == '.ics': url += '/remote.php/carddav/addressbooks/asdf/' + else: + raise RuntimeError(self.storage_class.fileext) if collection is not None: # the following collections are setup in ownCloud assert collection in ('test', 'test1', 'test2', 'test3', 'test4', diff --git a/vdirsyncer/storage/dav/base.py b/vdirsyncer/storage/dav/base.py index 9e57aca..d7a9c4e 100644 --- a/vdirsyncer/storage/dav/base.py +++ b/vdirsyncer/storage/dav/base.py @@ -232,6 +232,8 @@ class DavStorage(Storage): href, headers=headers ) + if response.status_code == 404: + raise exceptions.NotFoundError(href) self._check_response(response) def _list(self, xml): From a4557ef5cff144cf6c616d495eec37ffbecdd474 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 18 Mar 2014 17:21:33 +0100 Subject: [PATCH 05/20] Fix typos --- tests/storage/dav/_owncloud.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage/dav/_owncloud.py b/tests/storage/dav/_owncloud.py index b059d65..bb822e8 100644 --- a/tests/storage/dav/_owncloud.py +++ b/tests/storage/dav/_owncloud.py @@ -27,7 +27,7 @@ class ServerMixin(object): if self.storage_class.fileext == '.vcf': url += '/remote.php/carddav/addressbooks/asdf/' elif self.storage_class.fileext == '.ics': - url += '/remote.php/carddav/addressbooks/asdf/' + url += '/remote.php/caldav/calendars/asdf/' else: raise RuntimeError(self.storage_class.fileext) if collection is not None: @@ -36,4 +36,4 @@ class ServerMixin(object): 'test5', 'test6', 'test7', 'test8', 'test9', 'test10') - return {'url': url, 'collection': collection, 'username': 'asdf', 'password': 'asdf'} + return {'url': url, 'collection': collection, 'username': 'asdf', 'password': 'asdf'} From 84217392161d33cc7bdec04f64446edde0164ea9 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 18 Mar 2014 17:21:42 +0100 Subject: [PATCH 06/20] Loosen up assertions for owncloud a bit --- vdirsyncer/storage/dav/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vdirsyncer/storage/dav/base.py b/vdirsyncer/storage/dav/base.py index d7a9c4e..78cf3f0 100644 --- a/vdirsyncer/storage/dav/base.py +++ b/vdirsyncer/storage/dav/base.py @@ -206,7 +206,7 @@ class DavStorage(Storage): etag = response.headers.get('etag', None) if not etag: obj2, etag = self.get(href) - assert obj2.raw == obj.raw + assert obj2.uid == obj.uid return href, etag def update(self, href, obj, etag): From 4f51b31405b77c387a80c9733c8ccef5d91c8610 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 18 Mar 2014 17:28:07 +0100 Subject: [PATCH 07/20] Loosen up testsuite again --- tests/storage/dav/test_caldav.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/storage/dav/test_caldav.py b/tests/storage/dav/test_caldav.py index c9049ea..757981e 100644 --- a/tests/storage/dav/test_caldav.py +++ b/tests/storage/dav/test_caldav.py @@ -9,7 +9,9 @@ ''' +import requests.exceptions from vdirsyncer.storage.dav.caldav import CaldavStorage +import vdirsyncer.exceptions as exceptions from . import DavStorageTests @@ -61,8 +63,11 @@ class TestCaldavStorage(DavStorageTests): def test_item_types(self): kw = self.get_storage_args() s = self.storage_class(item_types=('VTODO',), **kw) - s.upload(self._create_bogus_item(1, item_template=EVENT_TEMPLATE)) - s.upload(self._create_bogus_item(5, item_template=EVENT_TEMPLATE)) + try: + s.upload(self._create_bogus_item(1, item_template=EVENT_TEMPLATE)) + s.upload(self._create_bogus_item(5, item_template=EVENT_TEMPLATE)) + except (exceptions.Error, requests.exceptions.HTTPError): + pass href, etag = \ s.upload(self._create_bogus_item(3, item_template=TASK_TEMPLATE)) ((href2, etag2),) = s.list() From 5527602a6284d2160b067f58bafa9d403f81e755 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 00:15:07 +0100 Subject: [PATCH 08/20] Fix git url for owncloud --- install-deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-deps.sh b/install-deps.sh index cbd1524..41bd2a8 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -21,7 +21,7 @@ radicale_storage_filesystem() { true; } davserver_owncloud() { - git clone git@github.com:untitaker/owncloud-testserver.git + git clone https://github.com/untitaker/owncloud-testserver.git cd ./owncloud-testserver/ sh install.sh } From d6b3c6d3288c27b7f9d95ce4a2acbb72245e7102 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 15:31:49 +0100 Subject: [PATCH 09/20] Extra sanity checks and debugging --- tests/storage/__init__.py | 4 +++- vdirsyncer/storage/dav/base.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 90646fb..3cdfab7 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -26,7 +26,9 @@ class StorageTests(object): raise NotImplementedError() def _get_storage(self): - return self.storage_class(**self.get_storage_args()) + s = self.storage_class(**self.get_storage_args()) + assert not list(s.list()) + return s def test_generic(self): items = map(self._create_bogus_item, range(1, 10)) diff --git a/vdirsyncer/storage/dav/base.py b/vdirsyncer/storage/dav/base.py index 78cf3f0..4a73a4e 100644 --- a/vdirsyncer/storage/dav/base.py +++ b/vdirsyncer/storage/dav/base.py @@ -127,7 +127,7 @@ class DavStorage(Storage): @staticmethod def _check_response(response): if response.status_code == 412: - raise exceptions.PreconditionFailed() + raise exceptions.PreconditionFailed(response.reason) response.raise_for_status() def get(self, href): From 3bff81303ccf7f7fee40f13e4e748f2aa1933787 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 15:55:08 +0100 Subject: [PATCH 10/20] Loosen up assertions even more --- tests/__init__.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index ef37da5..8effcb0 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,7 +1,13 @@ -def normalize_item(x): - return set(x for x in x.raw.splitlines() if - not x.startswith('X-RADICALE-NAME') and - not x.startswith('PRODID')) +def normalize_item(item): + rv = set() + for line in item.raw.splitlines(): + line = line.strip() + line = line.strip().split(u':', 1) + line[0] = line[0].split(';')[0] + if line[0] in ('X-RADICALE-NAME', 'PRODID', 'REV'): + continue + rv.add(u':'.join(line)) + return rv def assert_item_equals(a, b): From ff688ca2b9278a3d2447aa93e27878f9e1a8b4ce Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 16:03:57 +0100 Subject: [PATCH 11/20] Make assertions a bit narrower again... and use ownCloud/SabreDAV normalized form for item template instead --- tests/__init__.py | 5 ++++- tests/storage/dav/test_carddav.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/__init__.py b/tests/__init__.py index 8effcb0..1b5706b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,9 +1,12 @@ def normalize_item(item): + # - X-RADICALE-NAME is used by radicale, because hrefs don't really exist + # in their filesystem backend + # - PRODID is changed by radicale for some reason after upload, but nobody + # cares about that anyway rv = set() for line in item.raw.splitlines(): line = line.strip() line = line.strip().split(u':', 1) - line[0] = line[0].split(';')[0] if line[0] in ('X-RADICALE-NAME', 'PRODID', 'REV'): continue rv.add(u':'.join(line)) diff --git a/tests/storage/dav/test_carddav.py b/tests/storage/dav/test_carddav.py index e70c017..99c1771 100644 --- a/tests/storage/dav/test_carddav.py +++ b/tests/storage/dav/test_carddav.py @@ -21,11 +21,11 @@ class TestCarddavStorage(DavStorageTests): u'N:Daboo;Cyrus\n' u'ADR;TYPE=POSTAL:;2822 Email HQ;' # address continuing u'Suite 2821;RFCVille;PA;15213;USA\n' # on next line - u'EMAIL;TYPE=INTERNET,PREF:cyrus@example.com\n' + u'EMAIL;TYPE=INTERNET;TYPE=PREF:cyrus@example.com\n' u'NICKNAME:me\n' u'NOTE:Example VCard.\n' u'ORG:Self Employed\n' - u'TEL;TYPE=WORK,VOICE:412 605 0499\n' + u'TEL;TYPE=WORK;TYPE=VOICE:412 605 0499\n' u'TEL;TYPE=FAX:412 605 0705\n' u'URL:http://www.example.com\n' u'UID:{uid}\n' From ceb764b67dc93c3f7badad6605ecda57e4c7b7c5 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 16:09:31 +0100 Subject: [PATCH 12/20] add information on DAV_SERVER envvar to readme --- .travis.yml | 7 +++---- README.rst | 16 ++++++++++++++-- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index d55a82c..7f15f3e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,8 +5,7 @@ env: - DAV_SERVER=radicale RADICALE_STORAGE=database - DAV_SERVER=radicale_git RADICALE_STORAGE=filesystem - DAV_SERVER=radicale_git RADICALE_STORAGE=database + - DAV_SERVER=owncloud -install: - - "./install-deps.sh" - -script: py.test +install: "./install-deps.sh" +script: "./run-tests.sh tests/" diff --git a/README.rst b/README.rst index 56bd500..5d7059b 100644 --- a/README.rst +++ b/README.rst @@ -30,5 +30,17 @@ How to run the tests :: - sh install_deps.sh - py.test + sh install-deps.sh + sh run-tests.sh + +The environment variable ``DAV_SERVER`` specifies which CalDAV/CardDAV server +to test against. It has to be set for both scripts, ``install-deps.sh`` and +``run-tests.sh``. + + - ``DAV_SERVER=radicale``: The default, installs the latest Radicale release + from PyPI. Very fast, because no additional processes are needed. + - ``DAV_SERVER=radicale_git``: Same as ``radicale``, except that the + installation happens from their git repo. ``install-deps.sh`` is slightly + slower with this. + - ``DAV_SERVER=owncloud``: Uses latest ownCloud release. Very slow + installation, very slow tests. From b92ae6d001769804637e0672c243a3ceabf00de4 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 16:13:14 +0100 Subject: [PATCH 13/20] Add status attribute to Response mock --- tests/storage/dav/_radicale.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/storage/dav/_radicale.py b/tests/storage/dav/_radicale.py index 80317f9..d10a022 100644 --- a/tests/storage/dav/_radicale.py +++ b/tests/storage/dav/_radicale.py @@ -105,6 +105,7 @@ class Response(object): self.text = x.get_data(as_text=True) self.headers = x.headers self.encoding = x.charset + self.reason = str(x.status) def raise_for_status(self): '''copied from requests itself''' From bce581df37cbc25d8c6dd32e9d6b37efda8cd925 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 16:17:50 +0100 Subject: [PATCH 14/20] Don't install werkzeug when it is not needed --- install-deps.sh | 5 ++++- requirements.txt | 1 - 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/install-deps.sh b/install-deps.sh index 41bd2a8..9ee8ddb 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -14,7 +14,10 @@ davserver_radicale_git() { radicale_deps } -radicale_deps() { radicale_storage_$RADICALE_STORAGE; } +radicale_deps() { + pip install --use-mirrors werkzeug + radicale_storage_$RADICALE_STORAGE +} radicale_storage_database() { pip install --use-mirrors sqlalchemy pysqlite; } radicale_storage_filesystem() { true; } diff --git a/requirements.txt b/requirements.txt index a9d3dfb..1929a86 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,3 @@ -Werkzeug pytest mock git+https://github.com/geier/leif From f7690ad96ab2a8bd63221b782b62718687b90f1f Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 16:23:56 +0100 Subject: [PATCH 15/20] Argh, Travis --- .gitignore | 1 + .travis.yml | 13 ++++++++----- install-deps.sh | 28 +++++++++++++++------------- run-tests.sh | 10 +++++----- tests/storage/dav/__init__.py | 2 +- tests/storage/dav/_radicale.py | 4 +++- 6 files changed, 33 insertions(+), 25 deletions(-) mode change 100644 => 100755 run-tests.sh diff --git a/.gitignore b/.gitignore index 50e5c1d..7e0efb4 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ htmlcov build env owncloud-testserver +*.egg-info diff --git a/.travis.yml b/.travis.yml index 7f15f3e..dcae517 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,14 @@ language: python python: "2.7" env: - - DAV_SERVER=radicale RADICALE_STORAGE=filesystem - - DAV_SERVER=radicale RADICALE_STORAGE=database - - DAV_SERVER=radicale_git RADICALE_STORAGE=filesystem - - DAV_SERVER=radicale_git RADICALE_STORAGE=database - - DAV_SERVER=owncloud + global: + - IS_TRAVIS=true + matrix: + - DAV_SERVER=radicale_filesystem REQUIREMENTS=release + - DAV_SERVER=radicale_filesystem REQUIREMENTS=devel + - DAV_SERVER=radicale_database REQUIREMENTS=release + - DAV_SERVER=radicale_database REQUIREMENTS=devel + - DAV_SERVER=owncloud REQUIREMENTS=release install: "./install-deps.sh" script: "./run-tests.sh tests/" diff --git a/install-deps.sh b/install-deps.sh index 9ee8ddb..f570be5 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -1,28 +1,30 @@ #!/bin/sh -pip install --use-mirrors . +echo "The shell is $SHELL" +set -e +pip install --use-mirrors --editable . pip install --use-mirrors -r requirements.txt -[[ -z "$DAV_SERVER" ]] && DAV_SERVER=radicale -[[ -z "$RADICALE_STORAGE" ]] && RADICALE_STORAGE=filesystem +[ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem -davserver_radicale() { - pip install --use-mirrors radicale +davserver_radicale_filesystem() { radicale_deps } -davserver_radicale_git() { - pip install git+https://github.com/Kozea/Radicale.git +davserver_radicale_database() { radicale_deps + pip install --use-mirrors sqlalchemy pysqlite } radicale_deps() { - pip install --use-mirrors werkzeug - radicale_storage_$RADICALE_STORAGE + if [ "$REQUIREMENTS" == "release" ]; then + radicale_pkg="radicale" + elif [ "$REQUIREMENTS" == "devel" ]; then + radicale_pkg="git+https://github.com/Kozea/Radicale.git" + else + false + fi + pip install --use-mirrors werkzeug $radicale_pkg } -radicale_storage_database() { pip install --use-mirrors sqlalchemy pysqlite; } -radicale_storage_filesystem() { true; } - - davserver_owncloud() { git clone https://github.com/untitaker/owncloud-testserver.git cd ./owncloud-testserver/ diff --git a/run-tests.sh b/run-tests.sh old mode 100644 new mode 100755 index fb3f78d..515772a --- a/run-tests.sh +++ b/run-tests.sh @@ -1,12 +1,12 @@ #!/bin/sh -[[ -z "$DAV_SERVER" ]] && DAV_SERVER=radicale -[[ -z "$RADICALE_STORAGE" ]] && RADICALE_STORAGE=filesystem +set -e +[ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem -davserver_radicale() { true; } -davserver_radicale_git() { true; } +davserver_radicale_database() { true; } +davserver_radicale_filesystem() { true; } davserver_owncloud() { - sh ./owncloud-testserver/php.sh + sh ./owncloud-testserver/php.sh > /dev/null } diff --git a/tests/storage/dav/__init__.py b/tests/storage/dav/__init__.py index e8a31bb..33c8417 100644 --- a/tests/storage/dav/__init__.py +++ b/tests/storage/dav/__init__.py @@ -17,7 +17,7 @@ import requests.exceptions dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale' -if dav_server in ('radicale', 'radicale_git'): +if dav_server.startswith('radicale_'): from ._radicale import ServerMixin elif dav_server == 'owncloud': from ._owncloud import ServerMixin diff --git a/tests/storage/dav/_radicale.py b/tests/storage/dav/_radicale.py index d10a022..7aea147 100644 --- a/tests/storage/dav/_radicale.py +++ b/tests/storage/dav/_radicale.py @@ -57,6 +57,8 @@ create table property ( primary key (name, collection_path)); ''' +dav_server = os.environ.get('DAV_SERVER', 'radicale_filesystem') + def do_the_radicale_dance(tmpdir): # All of radicale is already global state, the cleanliness of the code and @@ -75,7 +77,7 @@ def do_the_radicale_dance(tmpdir): # Now we can set some basic configuration. radicale.config.set('rights', 'type', 'None') - if os.environ.get('RADICALE_STORAGE', 'filesystem') == 'filesystem': + if dav_server == 'radicale_filesystem': radicale.config.set('storage', 'type', 'filesystem') radicale.config.set('storage', 'filesystem_folder', tmpdir) else: From 2a13ca4c292b448fdaf3050a89a03a06788c56a3 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 22:31:04 +0100 Subject: [PATCH 16/20] Strict POSIX scripts Ubuntu uses dash as default shell, wish somebody told me :/ --- install-deps.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-deps.sh b/install-deps.sh index f570be5..3c7c36e 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -15,9 +15,9 @@ davserver_radicale_database() { } radicale_deps() { - if [ "$REQUIREMENTS" == "release" ]; then + if [ "$REQUIREMENTS" = "release" ]; then radicale_pkg="radicale" - elif [ "$REQUIREMENTS" == "devel" ]; then + elif [ "$REQUIREMENTS" = "devel" ]; then radicale_pkg="git+https://github.com/Kozea/Radicale.git" else false From bf21978e56db1f778fc1276a422e3d55db894f10 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 19 Mar 2014 23:03:12 +0100 Subject: [PATCH 17/20] Fix dav server default --- tests/storage/dav/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage/dav/__init__.py b/tests/storage/dav/__init__.py index 33c8417..a7bbd43 100644 --- a/tests/storage/dav/__init__.py +++ b/tests/storage/dav/__init__.py @@ -16,7 +16,7 @@ from vdirsyncer.storage.base import Item import requests.exceptions -dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale' +dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale_filesystem' if dav_server.startswith('radicale_'): from ._radicale import ServerMixin elif dav_server == 'owncloud': From c492813487b0e66c6df37b77577bb5f321089367 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Thu, 20 Mar 2014 14:07:06 +0100 Subject: [PATCH 18/20] This is useless We need more sophisticated output control, which PHP doesn't provide --- run-tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run-tests.sh b/run-tests.sh index 515772a..d47fe8b 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -6,7 +6,7 @@ davserver_radicale_database() { true; } davserver_radicale_filesystem() { true; } davserver_owncloud() { - sh ./owncloud-testserver/php.sh > /dev/null + sh ./owncloud-testserver/php.sh } From 414b986450703840441d0828cf48495a67b00733 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Thu, 20 Mar 2014 15:01:55 +0100 Subject: [PATCH 19/20] Use pytest-xprocess for process management --- .gitignore | 2 ++ install-deps.sh | 4 +++- requirements.txt | 1 + run-tests.sh | 14 +------------- tests/storage/dav/_owncloud.py | 2 +- tests/storage/dav/conftest.py | 22 ++++++++++++++++++++++ 6 files changed, 30 insertions(+), 15 deletions(-) create mode 100644 tests/storage/dav/conftest.py diff --git a/.gitignore b/.gitignore index 7e0efb4..4398cdf 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,5 @@ build env owncloud-testserver *.egg-info +.cache +.xprocess diff --git a/install-deps.sh b/install-deps.sh index 3c7c36e..7b47c22 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -26,7 +26,9 @@ radicale_deps() { } davserver_owncloud() { - git clone https://github.com/untitaker/owncloud-testserver.git + # Maybe tmpfs is mounted on /tmp/, can't harm anyway. + git clone --depth=1 https://github.com/untitaker/owncloud-testserver.git /tmp/owncloud-testserver + ln -s /tmp/owncloud-testserver . cd ./owncloud-testserver/ sh install.sh } diff --git a/requirements.txt b/requirements.txt index 1929a86..4395388 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ pytest +pytest-xprocess mock git+https://github.com/geier/leif diff --git a/run-tests.sh b/run-tests.sh index d47fe8b..1c82674 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -1,16 +1,4 @@ #!/bin/sh set -e [ -n "$DAV_SERVER" ] || DAV_SERVER=radicale_filesystem - -davserver_radicale_database() { true; } -davserver_radicale_filesystem() { true; } - -davserver_owncloud() { - sh ./owncloud-testserver/php.sh -} - - -# while it would be nice if the server was cleanly shut down, it's not really a -# problem either -davserver_$DAV_SERVER & -py.test $@ +exec py.test $@ diff --git a/tests/storage/dav/_owncloud.py b/tests/storage/dav/_owncloud.py index bb822e8..34f5e09 100644 --- a/tests/storage/dav/_owncloud.py +++ b/tests/storage/dav/_owncloud.py @@ -20,7 +20,7 @@ class ServerMixin(object): wsgi_teardown = None def setup_method(self, method): - subprocess.call([os.path.join(owncloud_repo, 'install.sh')]) + subprocess.check_call([os.path.join(owncloud_repo, 'install.sh')]) def get_storage_args(self, collection='test'): url = 'http://127.0.0.1:8080' diff --git a/tests/storage/dav/conftest.py b/tests/storage/dav/conftest.py new file mode 100644 index 0000000..3dead48 --- /dev/null +++ b/tests/storage/dav/conftest.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +''' + vdirsyncer.tests.storage.dav.conftest + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' + +import os +import pytest + +dav_server = os.environ.get('DAV_SERVER', '').strip() or 'radicale_filesystem' +php_sh = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../../owncloud-testserver/php.sh')) + +if dav_server == 'owncloud': + @pytest.fixture(autouse=True) + def start_owncloud_server(xprocess): + def preparefunc(cwd): + return 'Listening on', ['sh', php_sh] + + xprocess.ensure('owncloud_server', preparefunc) From 3f25d20fbd0665fbbfa71ba77a73e8c8d37c1485 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Thu, 20 Mar 2014 15:18:11 +0100 Subject: [PATCH 20/20] Don't try to clone if owncloud repo already exists --- install-deps.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install-deps.sh b/install-deps.sh index 7b47c22..0cea15c 100755 --- a/install-deps.sh +++ b/install-deps.sh @@ -27,8 +27,12 @@ radicale_deps() { davserver_owncloud() { # Maybe tmpfs is mounted on /tmp/, can't harm anyway. - git clone --depth=1 https://github.com/untitaker/owncloud-testserver.git /tmp/owncloud-testserver - ln -s /tmp/owncloud-testserver . + if [ ! -d ./owncloud-testserver/ ]; then + git clone --depth=1 \ + https://github.com/untitaker/owncloud-testserver.git \ + /tmp/owncloud-testserver + ln -s /tmp/owncloud-testserver . + fi cd ./owncloud-testserver/ sh install.sh }