From f1f51ac3cf8a913295bcec0136c0301060a38119 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 20:28:32 +0200 Subject: [PATCH 01/13] Don't fix EOF in travis.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since this is auto-generated, it... conflicts in complicated ways. 😞 --- .pre-commit-config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fc1c641..a403707 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -5,6 +5,7 @@ repos: - id: trailing-whitespace args: [--markdown-linebreak-ext=md] - id: end-of-file-fixer + exclude: '.travis.yml' - id: check-toml - id: check-added-large-files - id: debug-statements From c410fbf331c2b3521078ba2791ceb97f02235c6a Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 20:29:12 +0200 Subject: [PATCH 02/13] Enable docker on travis --- .travis.yml | 3 +++ scripts/make_travisconf.py | 2 ++ 2 files changed, 5 insertions(+) diff --git a/.travis.yml b/.travis.yml index 6c2c178..b566d0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,5 +70,8 @@ "script": [ "make -e $BUILD" ], + "services": [ + "docker" + ], "sudo": true } \ No newline at end of file diff --git a/scripts/make_travisconf.py b/scripts/make_travisconf.py index 6505ba4..6250462 100644 --- a/scripts/make_travisconf.py +++ b/scripts/make_travisconf.py @@ -12,6 +12,8 @@ cfg['dist'] = 'trusty' cfg['language'] = 'python' cfg['cache'] = 'pip' +cfg['services'] = ['docker'] + cfg['git'] = { 'submodules': False } From 0f3b2e74c0b1930e8526d369e418f5d2052bf688 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 20:26:19 +0200 Subject: [PATCH 03/13] Test xandikos running it in docker The main advantage here is that its dependencies are TOTALLY separate from vdirsyncer's, keeping the runtime environment for vdirsyncer cleaner. It also makes testing locally not only possible, but fast and pleasant. --- docker-compose.yaml | 7 +++++ docker/xandikos/Dockerfile | 13 ++++++++++ setup.cfg | 3 ++- tests/storage/servers/xandikos/__init__.py | 30 ++++------------------ tests/storage/servers/xandikos/install.sh | 14 ++-------- 5 files changed, 29 insertions(+), 38 deletions(-) create mode 100644 docker-compose.yaml create mode 100644 docker/xandikos/Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..f295471 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,7 @@ +version: '3' + +services: + xandikos: + build: docker/xandikos/ + ports: + - '8000:8000' diff --git a/docker/xandikos/Dockerfile b/docker/xandikos/Dockerfile new file mode 100644 index 0000000..464450c --- /dev/null +++ b/docker/xandikos/Dockerfile @@ -0,0 +1,13 @@ +# Original file copyright 2017 Jelmer Vernooij + +FROM ubuntu:bionic +RUN apt-get update && apt-get -y install xandikos locales +EXPOSE 8000 + +RUN locale-gen en_US.UTF-8 +ENV PYTHONIOENCODING=utf-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 + +CMD xandikos -d /tmp/dav -l 0.0.0.0 -p 8000 --autocreate diff --git a/setup.cfg b/setup.cfg index ec67785..fb86724 100644 --- a/setup.cfg +++ b/setup.cfg @@ -10,7 +10,8 @@ addopts = --tb=short # E743: Ambiguous function definition ignore = E731, E743 # E503: Line break occurred before a binary operator -extend-ignore = W503 +extend-ignore = E203, W503 +max-line-length = 88 select = C,E,F,W,B,B9 exclude = .eggs, tests/storage/servers/owncloud/, tests/storage/servers/nextcloud/, tests/storage/servers/baikal/, build/ application-package-names = tests,vdirsyncer diff --git a/tests/storage/servers/xandikos/__init__.py b/tests/storage/servers/xandikos/__init__.py index 899564b..b0e6451 100644 --- a/tests/storage/servers/xandikos/__init__.py +++ b/tests/storage/servers/xandikos/__init__.py @@ -1,35 +1,15 @@ import pytest -from xandikos.web import XandikosApp, XandikosBackend, WellknownRedirector - -import wsgi_intercept -import wsgi_intercept.requests_intercept - class ServerMixin: @pytest.fixture def get_storage_args(self, request, tmpdir, slow_create_collection): - tmpdir.mkdir('xandikos') - backend = XandikosBackend(path=str(tmpdir)) - cup = '/user/' - backend.create_principal(cup, create_defaults=True) - app = XandikosApp(backend, cup) - - app = WellknownRedirector(app, '/') - - wsgi_intercept.requests_intercept.install() - wsgi_intercept.add_wsgi_intercept('127.0.0.1', 8080, lambda: app) - - def teardown(): - wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 8080) - wsgi_intercept.requests_intercept.uninstall() - request.addfinalizer(teardown) - - def inner(collection='test'): - url = 'http://127.0.0.1:8080/' - args = {'url': url, 'collection': collection} + def inner(collection="test"): + url = "http://127.0.0.1:8000/" + args = {"url": url} if collection is not None: - args = self.storage_class.create_collection(**args) + args = slow_create_collection(self.storage_class, args, collection) return args + return inner diff --git a/tests/storage/servers/xandikos/install.sh b/tests/storage/servers/xandikos/install.sh index df5ad85..b01f6ea 100644 --- a/tests/storage/servers/xandikos/install.sh +++ b/tests/storage/servers/xandikos/install.sh @@ -1,14 +1,4 @@ #!/bin/sh -set -e -pip install wsgi_intercept - -if [ "$REQUIREMENTS" = "release" ] || [ "$REQUIREMENTS" = "minimal" ]; then - # XXX: This is the last version to support Python 3.5 - pip install -U "xandikos==0.0.11" -elif [ "$REQUIREMENTS" = "devel" ]; then - pip install -U git+https://github.com/jelmer/xandikos -else - echo "Invalid REQUIREMENTS value" - false -fi +docker-compose build xandikos +docker-compose up -d xandikos From b0d8fd34dc08cc9d98e1a2929f96847e3e80ebdf Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 20:58:16 +0200 Subject: [PATCH 04/13] Test radicale with docker too --- docker-compose.yaml | 5 ++ docker/radicale/Dockerfile | 5 ++ tests/storage/servers/radicale/__init__.py | 58 ++++------------------ tests/storage/servers/radicale/install.sh | 12 +---- 4 files changed, 22 insertions(+), 58 deletions(-) create mode 100644 docker/radicale/Dockerfile diff --git a/docker-compose.yaml b/docker-compose.yaml index f295471..26a0d99 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -5,3 +5,8 @@ services: build: docker/xandikos/ ports: - '8000:8000' + + radicale: + build: docker/radicale/ + ports: + - '8001:8001' diff --git a/docker/radicale/Dockerfile b/docker/radicale/Dockerfile new file mode 100644 index 0000000..462c5e2 --- /dev/null +++ b/docker/radicale/Dockerfile @@ -0,0 +1,5 @@ +FROM python:3.8 + +RUN pip install radicale + +CMD radicale --storage-filesystem-folder /tmp/dav -H 0.0.0.0:8001 -D diff --git a/tests/storage/servers/radicale/__init__.py b/tests/storage/servers/radicale/__init__.py index 6e4ae29..b3b59bb 100644 --- a/tests/storage/servers/radicale/__init__.py +++ b/tests/storage/servers/radicale/__init__.py @@ -1,57 +1,19 @@ -import logging import pytest -import radicale -import radicale.config - -from pkg_resources import parse_version as ver - -import wsgi_intercept -import wsgi_intercept.requests_intercept - -logger = logging.getLogger(__name__) - class ServerMixin: - - @pytest.fixture(autouse=True) - def setup(self, request, tmpdir): - if ver(radicale.VERSION) < ver('2.0.0-pre'): - raise RuntimeError('Testing against Radicale only works with ' - 'Radicale >= 2.0.0') - - def get_app(): - config = radicale.config.load(()) - config.set('storage', 'filesystem_folder', str(tmpdir)) - config.set('rights', 'type', 'owner_only') - - app = radicale.Application(config, logger) - - def is_authenticated(user, password): - return user == 'bob' and password == 'bob' - - app.is_authenticated = is_authenticated - return app - - wsgi_intercept.requests_intercept.install() - wsgi_intercept.add_wsgi_intercept('127.0.0.1', 80, get_app) - - def teardown(): - wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 80) - wsgi_intercept.requests_intercept.uninstall() - request.addfinalizer(teardown) - @pytest.fixture - def get_storage_args(self, get_item): - def inner(collection='test'): - url = 'http://127.0.0.1/' - rv = {'url': url, 'username': 'bob', 'password': 'bob'} + def get_storage_args(self, request, tmpdir, slow_create_collection): + def inner(collection="test"): + url = "http://127.0.0.1:8001/" + args = { + "url": url, + "username": "radicale", + "password": "radicale", + } if collection is not None: - collection = collection + self.storage_class.fileext - rv = self.storage_class.create_collection(collection, **rv) - s = self.storage_class(**rv) - assert not list(s.list()) + args = slow_create_collection(self.storage_class, args, collection) + return args - return rv return inner diff --git a/tests/storage/servers/radicale/install.sh b/tests/storage/servers/radicale/install.sh index c2a1ec9..51732d8 100644 --- a/tests/storage/servers/radicale/install.sh +++ b/tests/storage/servers/radicale/install.sh @@ -1,12 +1,4 @@ #!/bin/sh -set -e -if [ "$REQUIREMENTS" = "release" ] || [ "$REQUIREMENTS" = "minimal" ]; then - radicale_pkg="radicale==2.1.10" -elif [ "$REQUIREMENTS" = "devel" ]; then - radicale_pkg="git+https://github.com/Kozea/Radicale.git" -else - echo "Invalid requirements envvar" - false -fi -pip install wsgi_intercept $radicale_pkg +docker-compose build radicale +docker-compose up -d radicale From 6c6da2f61383a3e2f506a73e83eaf1144fdaaef3 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 22:25:34 +0200 Subject: [PATCH 05/13] Test baikal using docker --- .gitmodules | 3 - docker-compose.yaml | 5 + docker/baikal/Dockerfile | 26 +++++ docker/baikal/apache.conf | 25 ++++ docker/baikal/baikal.yaml | 18 +++ docker/baikal/configure.sql | 139 +++++++++++++++++++++++ docker/baikal/start.sh | 16 +++ tests/storage/servers/baikal | 1 - tests/storage/servers/baikal/__init__.py | 24 ++++ tests/storage/servers/baikal/install.sh | 4 + 10 files changed, 257 insertions(+), 4 deletions(-) create mode 100644 docker/baikal/Dockerfile create mode 100644 docker/baikal/apache.conf create mode 100644 docker/baikal/baikal.yaml create mode 100644 docker/baikal/configure.sql create mode 100644 docker/baikal/start.sh delete mode 160000 tests/storage/servers/baikal create mode 100644 tests/storage/servers/baikal/__init__.py create mode 100644 tests/storage/servers/baikal/install.sh diff --git a/.gitmodules b/.gitmodules index cc13351..0f685d2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,3 @@ -[submodule "tests/storage/servers/baikal"] - path = tests/storage/servers/baikal - url = https://github.com/vdirsyncer/baikal-testserver [submodule "tests/storage/servers/owncloud"] path = tests/storage/servers/owncloud url = https://github.com/vdirsyncer/owncloud-testserver diff --git a/docker-compose.yaml b/docker-compose.yaml index 26a0d99..17779bb 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -10,3 +10,8 @@ services: build: docker/radicale/ ports: - '8001:8001' + + baikal: + build: docker/baikal/ + ports: + - '8002:80' diff --git a/docker/baikal/Dockerfile b/docker/baikal/Dockerfile new file mode 100644 index 0000000..6b0f1ff --- /dev/null +++ b/docker/baikal/Dockerfile @@ -0,0 +1,26 @@ +# Based on https://github.com/ckulka/baikal-docker +# Sadly, we can't override the VOLUME it has set, and we want some static +# config. +FROM php:7.4-apache +ENV VERSION 0.7.0 + +ADD https://github.com/sabre-io/Baikal/releases/download/$VERSION/baikal-$VERSION.zip . +RUN apt-get update && apt-get install -y sqlite3 unzip +RUN unzip -q baikal-$VERSION.zip -d /var/www/ + +RUN chown -R www-data:www-data /var/www/baikal && \ + docker-php-ext-install pdo pdo_mysql + +COPY apache.conf /etc/apache2/sites-enabled/000-default.conf +COPY start.sh /opt/ +RUN a2enmod rewrite + +COPY baikal.yaml /var/www/baikal/config/baikal.yaml +COPY configure.sql /configure.sql + +RUN touch /var/www/baikal/Specific/INSTALL_DISABLED +RUN cat /configure.sql | sqlite3 /var/www/baikal/Specific/db/db.sqlite + +RUN chmod -R 777 /var/www/baikal/Specific/ /var/www/baikal/config/ + +CMD [ "sh", "/opt/start.sh" ] diff --git a/docker/baikal/apache.conf b/docker/baikal/apache.conf new file mode 100644 index 0000000..a6b35f5 --- /dev/null +++ b/docker/baikal/apache.conf @@ -0,0 +1,25 @@ +# Shameless copied from https://github.com/ckulka/baikal-docker/blob/master/files/apache.conf + + + + # InjectedServerAlias dav.example.org dav.example.io + DocumentRoot /var/www/baikal/html + + RewriteEngine On + RewriteRule /.well-known/carddav /dav.php [R,L] + RewriteRule /.well-known/caldav /dav.php [R,L] + + + Options None + Options +FollowSymlinks + AllowOverride All + + # Confiugration for apache-2.2: + Order allow,deny + Allow from all + + # Confiugration for apache-2.4: + Require all granted + + + diff --git a/docker/baikal/baikal.yaml b/docker/baikal/baikal.yaml new file mode 100644 index 0000000..459e455 --- /dev/null +++ b/docker/baikal/baikal.yaml @@ -0,0 +1,18 @@ +system: + configured_version: 0.7.0 + timezone: Europe/Paris + card_enabled: true + cal_enabled: true + dav_auth_type: Basic + admin_passwordhash: 6a890c3aa185845a4bee1e1caed92e1faaf2dec6772291dca301cef6782e3bce + auth_realm: BaikalDAV + invite_from: noreply@localhost +database: + sqlite_file: /var/www/baikal/Specific/db/db.sqlite + mysql: false + mysql_host: '' + mysql_dbname: '' + mysql_username: '' + mysql_password: '' + encryption_key: bdf3bec969736e122e6d5f72c282c49e + configured_version: '' diff --git a/docker/baikal/configure.sql b/docker/baikal/configure.sql new file mode 100644 index 0000000..55ec586 --- /dev/null +++ b/docker/baikal/configure.sql @@ -0,0 +1,139 @@ +PRAGMA foreign_keys=OFF; +BEGIN TRANSACTION; +CREATE TABLE addressbooks ( + id integer primary key asc NOT NULL, + principaluri text NOT NULL, + displayname text, + uri text NOT NULL, + description text, + synctoken integer DEFAULT 1 NOT NULL +); +INSERT INTO addressbooks VALUES(1,'principals/baikal','Default Address Book','default','Default Address Book for Baikal',1); +CREATE TABLE cards ( + id integer primary key asc NOT NULL, + addressbookid integer NOT NULL, + carddata blob, + uri text NOT NULL, + lastmodified integer, + etag text, + size integer +); +CREATE TABLE addressbookchanges ( + id integer primary key asc NOT NULL, + uri text, + synctoken integer NOT NULL, + addressbookid integer NOT NULL, + operation integer NOT NULL +); +CREATE TABLE calendarobjects ( + id integer primary key asc NOT NULL, + calendardata blob NOT NULL, + uri text NOT NULL, + calendarid integer NOT NULL, + lastmodified integer NOT NULL, + etag text NOT NULL, + size integer NOT NULL, + componenttype text, + firstoccurence integer, + lastoccurence integer, + uid text +); +CREATE TABLE calendars ( + id integer primary key asc NOT NULL, + synctoken integer DEFAULT 1 NOT NULL, + components text NOT NULL +); +INSERT INTO calendars VALUES(1,1,'VEVENT,VTODO'); +CREATE TABLE calendarinstances ( + id integer primary key asc NOT NULL, + calendarid integer, + principaluri text, + access integer, + displayname text, + uri text NOT NULL, + description text, + calendarorder integer, + calendarcolor text, + timezone text, + transparent bool, + share_href text, + share_displayname text, + share_invitestatus integer DEFAULT '2', + UNIQUE (principaluri, uri), + UNIQUE (calendarid, principaluri), + UNIQUE (calendarid, share_href) +); +INSERT INTO calendarinstances VALUES(1,1,'principals/baikal',NULL,'Default calendar','default','Default calendar',0,'','Europe/Paris',NULL,NULL,NULL,2); +CREATE TABLE calendarchanges ( + id integer primary key asc NOT NULL, + uri text, + synctoken integer NOT NULL, + calendarid integer NOT NULL, + operation integer NOT NULL +); +CREATE TABLE calendarsubscriptions ( + id integer primary key asc NOT NULL, + uri text NOT NULL, + principaluri text NOT NULL, + source text NOT NULL, + displayname text, + refreshrate text, + calendarorder integer, + calendarcolor text, + striptodos bool, + stripalarms bool, + stripattachments bool, + lastmodified int +); +CREATE TABLE schedulingobjects ( + id integer primary key asc NOT NULL, + principaluri text NOT NULL, + calendardata blob, + uri text NOT NULL, + lastmodified integer, + etag text NOT NULL, + size integer NOT NULL +); +CREATE TABLE locks ( + id integer primary key asc NOT NULL, + owner text, + timeout integer, + created integer, + token text, + scope integer, + depth integer, + uri text +); +CREATE TABLE principals ( + id INTEGER PRIMARY KEY ASC NOT NULL, + uri TEXT NOT NULL, + email TEXT, + displayname TEXT, + UNIQUE(uri) +); +INSERT INTO principals VALUES(1,'principals/baikal','baikal@example.com','Baikal'); +CREATE TABLE groupmembers ( + id INTEGER PRIMARY KEY ASC NOT NULL, + principal_id INTEGER NOT NULL, + member_id INTEGER NOT NULL, + UNIQUE(principal_id, member_id) +); +CREATE TABLE propertystorage ( + id integer primary key asc NOT NULL, + path text NOT NULL, + name text NOT NULL, + valuetype integer NOT NULL, + value string +); +CREATE TABLE users ( + id integer primary key asc NOT NULL, + username TEXT NOT NULL, + digesta1 TEXT NOT NULL, + UNIQUE(username) +); +INSERT INTO users VALUES(1,'baikal','3b0845b235b7e985ce5905ab8df45e1a'); +CREATE INDEX addressbookid_synctoken ON addressbookchanges (addressbookid, synctoken); +CREATE INDEX calendarid_synctoken ON calendarchanges (calendarid, synctoken); +CREATE INDEX principaluri_uri ON calendarsubscriptions (principaluri, uri); +CREATE UNIQUE INDEX path_property ON propertystorage (path, name); +COMMIT; diff --git a/docker/baikal/start.sh b/docker/baikal/start.sh new file mode 100644 index 0000000..cd41bba --- /dev/null +++ b/docker/baikal/start.sh @@ -0,0 +1,16 @@ +#!/bin/sh +# Shameless copied from https://raw.githubusercontent.com/ckulka/baikal-docker/master/files/start.sh + +# Inject ServerName and ServerAlias if specified +APACHE_CONFIG="/etc/apache2/sites-available/000-default.conf" +if [ ! -z ${BAIKAL_SERVERNAME+x} ] +then + sed -i "s/# InjectedServerName .*/ServerName $BAIKAL_SERVERNAME/g" $APACHE_CONFIG +fi + +if [ ! -z ${BAIKAL_SERVERALIAS+x} ] +then + sed -i "s/# InjectedServerAlias .*/ServerAlias $BAIKAL_SERVERALIAS/g" $APACHE_CONFIG +fi + +apache2-foreground diff --git a/tests/storage/servers/baikal b/tests/storage/servers/baikal deleted file mode 160000 index 6c8c379..0000000 --- a/tests/storage/servers/baikal +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6c8c379f1ee8bf4ab0ac54fc4eec3e4a6349c237 diff --git a/tests/storage/servers/baikal/__init__.py b/tests/storage/servers/baikal/__init__.py new file mode 100644 index 0000000..2647819 --- /dev/null +++ b/tests/storage/servers/baikal/__init__.py @@ -0,0 +1,24 @@ +import pytest + + +class ServerMixin: + @pytest.fixture + def get_storage_args(self, request, tmpdir, slow_create_collection): + def inner(collection="test"): + base_url = "http://127.0.0.1:8002/" + args = { + "url": base_url, + "username": "baikal", + "password": "baikal", + } + + if self.storage_class.fileext == '.vcf': + args['url'] = base_url + "card.php/" + else: + args['url'] = base_url + "cal.php/" + + if collection is not None: + args = slow_create_collection(self.storage_class, args, collection) + return args + + return inner diff --git a/tests/storage/servers/baikal/install.sh b/tests/storage/servers/baikal/install.sh new file mode 100644 index 0000000..99fcef9 --- /dev/null +++ b/tests/storage/servers/baikal/install.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +docker-compose build baikal +docker-compose up -d baikal From 03190356886ce0a58d3371c46c374ca361f4ebf7 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 22:39:26 +0200 Subject: [PATCH 06/13] Don't run unit/system tests for each storage The current storage (DAV_SERVER) has no impact on unit tests and system tests, so rather than run all these tests for each server, just run them once per python. --- .travis.yml | 34 +++++++++++++++++++++++++--------- Makefile | 10 ++++++++-- scripts/make_travisconf.py | 7 ++++++- 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/.travis.yml b/.travis.yml index b566d0b..70479a9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,40 +25,56 @@ "python": "3.6" }, { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ", + "env": "BUILD=test REQUIREMENTS=release", "python": "3.5" }, { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ", + "env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=release ", "python": "3.5" }, { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ", + "env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=release ", "python": "3.5" }, { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ", + "env": "BUILD=test REQUIREMENTS=minimal", "python": "3.5" }, { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ", + "env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=minimal ", + "python": "3.5" + }, + { + "env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=minimal ", + "python": "3.5" + }, + { + "env": "BUILD=test REQUIREMENTS=release", "python": "3.6" }, { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ", + "env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=release ", "python": "3.6" }, { - "env": "BUILD=test DAV_SERVER=fastmail REQUIREMENTS=release ", + "env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=release ", + "python": "3.6" + }, + { + "env": "BUILD=test-storage DAV_SERVER=fastmail REQUIREMENTS=release ", "if": "NOT (type IN (pull_request))", "python": "3.6" }, { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ", + "env": "BUILD=test REQUIREMENTS=minimal", "python": "3.6" }, { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ", + "env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=minimal ", + "python": "3.6" + }, + { + "env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=minimal ", "python": "3.6" }, { diff --git a/Makefile b/Makefile index bbbceeb..27bbb63 100644 --- a/Makefile +++ b/Makefile @@ -46,14 +46,16 @@ export TESTSERVER_BASE := ./tests/storage/servers/ CODECOV_PATH = /tmp/codecov.sh ifeq ($(CI), true) +test-storage: + curl -s https://codecov.io/bash > $(CODECOV_PATH) + $(PYTEST) tests/storage/ + bash $(CODECOV_PATH) -c -F storage test: curl -s https://codecov.io/bash > $(CODECOV_PATH) $(PYTEST) tests/unit/ bash $(CODECOV_PATH) -c -F unit $(PYTEST) tests/system/ bash $(CODECOV_PATH) -c -F system - $(PYTEST) tests/storage/ - bash $(CODECOV_PATH) -c -F storage else test: $(PYTEST) @@ -81,6 +83,10 @@ install-test: install-servers install-dev fi [ -z "$(TEST_EXTRA_PACKAGES)" ] || pip install $(TEST_EXTRA_PACKAGES) +install-test-storage: install-test + # This is just an alias + true + install-style: install-docs install-dev pip install -U flake8 flake8-import-order flake8-bugbear diff --git a/scripts/make_travisconf.py b/scripts/make_travisconf.py index 6250462..56025ad 100644 --- a/scripts/make_travisconf.py +++ b/scripts/make_travisconf.py @@ -45,13 +45,18 @@ for python, requirements in itertools.product( ): dav_servers = ("radicale", "xandikos") + matrix.append({ + 'python': python, + 'env': f"BUILD=test REQUIREMENTS={requirements}", + }) + if python == latest_python and requirements == "release": dav_servers += ("fastmail",) for dav_server in dav_servers: job = { 'python': python, - 'env': ("BUILD=test " + 'env': ("BUILD=test-storage " "DAV_SERVER={dav_server} " "REQUIREMENTS={requirements} " .format(dav_server=dav_server, From 4f894e04ddc0dbbb4ec3ee2b1043fc4303000a9b Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Mon, 8 Jun 2020 22:40:12 +0200 Subject: [PATCH 07/13] Empty noop script --- tests/storage/servers/skip/install.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/storage/servers/skip/install.sh b/tests/storage/servers/skip/install.sh index 1a24852..e69de29 100755 --- a/tests/storage/servers/skip/install.sh +++ b/tests/storage/servers/skip/install.sh @@ -1 +0,0 @@ -#!/bin/sh From 88f2cd5b53415aeca72a6e798fad7d939e62713c Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 9 Jun 2020 09:16:07 +0200 Subject: [PATCH 08/13] Skip VTODO items for Fastmail See #824 --- tests/storage/servers/fastmail/__init__.py | 26 +++++++++++++--------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/tests/storage/servers/fastmail/__init__.py b/tests/storage/servers/fastmail/__init__.py index 6471ef3..1c21c6d 100644 --- a/tests/storage/servers/fastmail/__init__.py +++ b/tests/storage/servers/fastmail/__init__.py @@ -4,24 +4,28 @@ import pytest class ServerMixin: - @pytest.fixture - def get_storage_args(self, slow_create_collection): - def inner(collection='test'): + def get_storage_args(self, item_type, slow_create_collection): + if item_type == "VTODO": + # Fastmail has non-standard support for TODOs + # See https://github.com/pimutils/vdirsyncer/issues/824 + pytest.skip("Fastmail has non-standard VTODO support.") + + def inner(collection="test"): args = { - 'username': os.environ['FASTMAIL_USERNAME'], - 'password': os.environ['FASTMAIL_PASSWORD'] + "username": os.environ["FASTMAIL_USERNAME"], + "password": os.environ["FASTMAIL_PASSWORD"], } - if self.storage_class.fileext == '.ics': - args['url'] = 'https://caldav.fastmail.com/' - elif self.storage_class.fileext == '.vcf': - args['url'] = 'https://carddav.fastmail.com/' + if self.storage_class.fileext == ".ics": + args["url"] = "https://caldav.fastmail.com/" + elif self.storage_class.fileext == ".vcf": + args["url"] = "https://carddav.fastmail.com/" else: raise RuntimeError() if collection is not None: - args = slow_create_collection(self.storage_class, args, - collection) + args = slow_create_collection(self.storage_class, args, collection) return args + return inner From 53331fedee9e3b77f40a8dfbca54e91ca0a29fd5 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 9 Jun 2020 09:34:03 +0200 Subject: [PATCH 09/13] Skip unsupported radicale test --- tests/storage/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 9add313..dd0d0b3 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -201,6 +201,8 @@ class StorageTests: if getattr(self, 'dav_server', '') in \ ('icloud', 'fastmail', 'davical'): pytest.skip('Manual cleanup would be necessary.') + if getattr(self, 'dav_server', '') == "radicale": + pytest.skip("Radicale does not support collection creation") args = get_storage_args(collection=None) args['collection'] = 'test' From 50eefa18163857315f0d42be44d8c051cf3a0cc1 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 9 Jun 2020 10:28:28 +0200 Subject: [PATCH 10/13] Use pre-commit for CI style checks Have a single, uniform definition everywhere. --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 27bbb63..2141017 100644 --- a/Makefile +++ b/Makefile @@ -88,10 +88,10 @@ install-test-storage: install-test true install-style: install-docs install-dev - pip install -U flake8 flake8-import-order flake8-bugbear + pip install pre-commit style: - flake8 + pre-commit run --all ! git grep -i syncroniz */* ! git grep -i 'text/icalendar' */* sphinx-build -W -b html ./docs/ ./docs/_build/html/ From af3659ac1a573a80a25954177023335cb644825d Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 9 Jun 2020 10:42:26 +0200 Subject: [PATCH 11/13] Update contributing instructions --- docs/contributing.rst | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/docs/contributing.rst b/docs/contributing.rst index deb48f5..449638a 100644 --- a/docs/contributing.rst +++ b/docs/contributing.rst @@ -84,8 +84,8 @@ virtualenv_ and run this inside of it:: # - stylecheckers (flake8) and code formatters (autopep8) make install-dev - # Install git commit hook for the stylechecker - make install-git-hooks + # Install git commit hook for some extra linting and checking + pre-commit install # install test dependencies make install-test @@ -100,9 +100,13 @@ The ``Makefile`` has a lot of options that allow you to control which tests are run, and which servers are tested. Take a look at its code where they are all initialized and documented. -For example, to test xandikos, run:: +For example, to test xandikos, first run the server itself:: + + docker-compose build xandikos + docker-compose up -d xandikos + +Then run the tests specifying this ``DAV_SERVER``, run:: - make DAV_SERVER=xandikos install-test make DAV_SERVER=xandikos test If you have any questions, feel free to open issues about it. From 354aaec2e010205fbdaeb3472743c10ab4f4da03 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 9 Jun 2020 10:46:17 +0200 Subject: [PATCH 12/13] Only run CI for branch `master` For PRs, CI will run on the PR itself, so no need to run it for the branch (otherwise we get duplicate jobs, and it's sooooo slow!). --- .travis.yml | 5 +---- scripts/make_travisconf.py | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 70479a9..f538aeb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,7 @@ { "branches": { "only": [ - "auto", - "next", - "master", - "/^.*-maintenance$/" + "master" ] }, "cache": "pip", diff --git a/scripts/make_travisconf.py b/scripts/make_travisconf.py index 56025ad..65b2876 100644 --- a/scripts/make_travisconf.py +++ b/scripts/make_travisconf.py @@ -19,7 +19,7 @@ cfg['git'] = { } cfg['branches'] = { - 'only': ['auto', 'next', 'master', '/^.*-maintenance$/'] + 'only': ['master'] } cfg['install'] = """ From cd86ea7a6229716faa6b665eed08fcc2eca1dbd1 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Tue, 9 Jun 2020 11:29:54 +0200 Subject: [PATCH 13/13] Run storage tests for ETESYNC too --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 2141017..55d6322 100644 --- a/Makefile +++ b/Makefile @@ -56,6 +56,7 @@ test: bash $(CODECOV_PATH) -c -F unit $(PYTEST) tests/system/ bash $(CODECOV_PATH) -c -F system + [ "$(ETESYNC_TESTS)" = "false" ] || make test-storage else test: $(PYTEST)