sjs/vdirsyncer
1
0
Fork 0
mirror of https://github.com/samsonjs/vdirsyncer.git synced 2026-03-29 09:35:50 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #823 from pimutils/next

Browse source 
Run test servers with Docker
This commit is contained in:
Hugo Barrera 2020-06-09 09:47:54 +00:00 committed by GitHub
commit 82375f20aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
25 changed files with 383 additions and 135 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitmodules vendored
View file

@ -1,6 +1,3 @@
[submodule "tests/storage/servers/baikal"]
path = tests/storage/servers/baikal
url = https://github.com/vdirsyncer/baikal-testserver
[submodule "tests/storage/servers/owncloud"]
path = tests/storage/servers/owncloud
url = https://github.com/vdirsyncer/owncloud-testserver

1
.pre-commit-config.yaml
View file

@ -5,6 +5,7 @@ repos:
- id: trailing-whitespace
args: [--markdown-linebreak-ext=md]
- id: end-of-file-fixer
exclude: '.travis.yml'
- id: check-toml
- id: check-added-large-files
- id: debug-statements

42
.travis.yml
View file

@ -1,10 +1,7 @@
{
"branches": {
"only": [
"auto",
"next",
"master",
"/^.*-maintenance$/"
"master"
]
},
"cache": "pip",
@ -25,40 +22,56 @@
"python": "3.6"
},
{
"env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ",
"env": "BUILD=test REQUIREMENTS=release",
"python": "3.5"
},
{
"env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ",
"env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=release ",
"python": "3.5"
},
{
"env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ",
"env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=release ",
"python": "3.5"
},
{
"env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ",
"env": "BUILD=test REQUIREMENTS=minimal",
"python": "3.5"
},
{
"env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ",
"env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=minimal ",
"python": "3.5"
},
{
"env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=minimal ",
"python": "3.5"
},
{
"env": "BUILD=test REQUIREMENTS=release",
"python": "3.6"
},
{
"env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ",
"env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=release ",
"python": "3.6"
},
{
"env": "BUILD=test DAV_SERVER=fastmail REQUIREMENTS=release ",
"env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=release ",
"python": "3.6"
},
{
"env": "BUILD=test-storage DAV_SERVER=fastmail REQUIREMENTS=release ",
"if": "NOT (type IN (pull_request))",
"python": "3.6"
},
{
"env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ",
"env": "BUILD=test REQUIREMENTS=minimal",
"python": "3.6"
},
{
"env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ",
"env": "BUILD=test-storage DAV_SERVER=radicale REQUIREMENTS=minimal ",
"python": "3.6"
},
{
"env": "BUILD=test-storage DAV_SERVER=xandikos REQUIREMENTS=minimal ",
"python": "3.6"
},
{
@ -70,5 +83,8 @@
"script": [
"make -e $BUILD"
],
"services": [
"docker"
],
"sudo": true
}

15
Makefile
View file

@ -46,14 +46,17 @@ export TESTSERVER_BASE := ./tests/storage/servers/
CODECOV_PATH = /tmp/codecov.sh
ifeq ($(CI), true)
test-storage:
curl -s https://codecov.io/bash > $(CODECOV_PATH)
$(PYTEST) tests/storage/
bash $(CODECOV_PATH) -c -F storage
test:
curl -s https://codecov.io/bash > $(CODECOV_PATH)
$(PYTEST) tests/unit/
bash $(CODECOV_PATH) -c -F unit
$(PYTEST) tests/system/
bash $(CODECOV_PATH) -c -F system
$(PYTEST) tests/storage/
bash $(CODECOV_PATH) -c -F storage
[ "$(ETESYNC_TESTS)" = "false" ] || make test-storage
else
test:
$(PYTEST)
@ -81,11 +84,15 @@ install-test: install-servers install-dev
fi
[ -z "$(TEST_EXTRA_PACKAGES)" ] || pip install $(TEST_EXTRA_PACKAGES)
install-test-storage: install-test
# This is just an alias
true
install-style: install-docs install-dev
pip install -U flake8 flake8-import-order flake8-bugbear
pip install pre-commit
style:
flake8
pre-commit run --all
! git grep -i syncroniz */*
! git grep -i 'text/icalendar' */*
sphinx-build -W -b html ./docs/ ./docs/_build/html/

17
docker-compose.yaml Normal file
View file

@ -0,0 +1,17 @@
version: '3'
services:
xandikos:
build: docker/xandikos/
ports:
- '8000:8000'
radicale:
build: docker/radicale/
ports:
- '8001:8001'
baikal:
build: docker/baikal/
ports:
- '8002:80'

26
docker/baikal/Dockerfile Normal file
View file

@ -0,0 +1,26 @@
# Based on https://github.com/ckulka/baikal-docker
# Sadly, we can't override the VOLUME it has set, and we want some static
# config.
FROM php:7.4-apache
ENV VERSION 0.7.0
ADD https://github.com/sabre-io/Baikal/releases/download/$VERSION/baikal-$VERSION.zip .
RUN apt-get update && apt-get install -y sqlite3 unzip
RUN unzip -q baikal-$VERSION.zip -d /var/www/
RUN chown -R www-data:www-data /var/www/baikal && \
docker-php-ext-install pdo pdo_mysql
COPY apache.conf /etc/apache2/sites-enabled/000-default.conf
COPY start.sh /opt/
RUN a2enmod rewrite
COPY baikal.yaml /var/www/baikal/config/baikal.yaml
COPY configure.sql /configure.sql
RUN touch /var/www/baikal/Specific/INSTALL_DISABLED
RUN cat /configure.sql | sqlite3 /var/www/baikal/Specific/db/db.sqlite
RUN chmod -R 777 /var/www/baikal/Specific/ /var/www/baikal/config/
CMD [ "sh", "/opt/start.sh" ]

25
docker/baikal/apache.conf Normal file
View file

@ -0,0 +1,25 @@
# Shameless copied from https://github.com/ckulka/baikal-docker/blob/master/files/apache.conf
<VirtualHost *:80>
# InjectedServerAlias dav.example.org dav.example.io
DocumentRoot /var/www/baikal/html
RewriteEngine On
RewriteRule /.well-known/carddav /dav.php [R,L]
RewriteRule /.well-known/caldav /dav.php [R,L]
<Directory "/var/www/baikal/html">
Options None
Options +FollowSymlinks
AllowOverride All
# Confiugration for apache-2.2:
Order allow,deny
Allow from all
# Confiugration for apache-2.4:
Require all granted
</Directory>
</VirtualHost>

18
docker/baikal/baikal.yaml Normal file
View file

@ -0,0 +1,18 @@
system:
configured_version: 0.7.0
timezone: Europe/Paris
card_enabled: true
cal_enabled: true
dav_auth_type: Basic
admin_passwordhash: 6a890c3aa185845a4bee1e1caed92e1faaf2dec6772291dca301cef6782e3bce
auth_realm: BaikalDAV
invite_from: noreply@localhost
database:
sqlite_file: /var/www/baikal/Specific/db/db.sqlite
mysql: false
mysql_host: ''
mysql_dbname: ''
mysql_username: ''
mysql_password: ''
encryption_key: bdf3bec969736e122e6d5f72c282c49e
configured_version: ''

139
docker/baikal/configure.sql Normal file
View file

@ -0,0 +1,139 @@
PRAGMA foreign_keys=OFF;
BEGIN TRANSACTION;
CREATE TABLE addressbooks (
id integer primary key asc NOT NULL,
principaluri text NOT NULL,
displayname text,
uri text NOT NULL,
description text,
synctoken integer DEFAULT 1 NOT NULL
);
INSERT INTO addressbooks VALUES(1,'principals/baikal','Default Address Book','default','Default Address Book for Baikal',1);
CREATE TABLE cards (
id integer primary key asc NOT NULL,
addressbookid integer NOT NULL,
carddata blob,
uri text NOT NULL,
lastmodified integer,
etag text,
size integer
);
CREATE TABLE addressbookchanges (
id integer primary key asc NOT NULL,
uri text,
synctoken integer NOT NULL,
addressbookid integer NOT NULL,
operation integer NOT NULL
);
CREATE TABLE calendarobjects (
id integer primary key asc NOT NULL,
calendardata blob NOT NULL,
uri text NOT NULL,
calendarid integer NOT NULL,
lastmodified integer NOT NULL,
etag text NOT NULL,
size integer NOT NULL,
componenttype text,
firstoccurence integer,
lastoccurence integer,
uid text
);
CREATE TABLE calendars (
id integer primary key asc NOT NULL,
synctoken integer DEFAULT 1 NOT NULL,
components text NOT NULL
);
INSERT INTO calendars VALUES(1,1,'VEVENT,VTODO');
CREATE TABLE calendarinstances (
id integer primary key asc NOT NULL,
calendarid integer,
principaluri text,
access integer,
displayname text,
uri text NOT NULL,
description text,
calendarorder integer,
calendarcolor text,
timezone text,
transparent bool,
share_href text,
share_displayname text,
share_invitestatus integer DEFAULT '2',
UNIQUE (principaluri, uri),
UNIQUE (calendarid, principaluri),
UNIQUE (calendarid, share_href)
);
INSERT INTO calendarinstances VALUES(1,1,'principals/baikal',NULL,'Default calendar','default','Default calendar',0,'','Europe/Paris',NULL,NULL,NULL,2);
CREATE TABLE calendarchanges (
id integer primary key asc NOT NULL,
uri text,
synctoken integer NOT NULL,
calendarid integer NOT NULL,
operation integer NOT NULL
);
CREATE TABLE calendarsubscriptions (
id integer primary key asc NOT NULL,
uri text NOT NULL,
principaluri text NOT NULL,
source text NOT NULL,
displayname text,
refreshrate text,
calendarorder integer,
calendarcolor text,
striptodos bool,
stripalarms bool,
stripattachments bool,
lastmodified int
);
CREATE TABLE schedulingobjects (
id integer primary key asc NOT NULL,
principaluri text NOT NULL,
calendardata blob,
uri text NOT NULL,
lastmodified integer,
etag text NOT NULL,
size integer NOT NULL
);
CREATE TABLE locks (
id integer primary key asc NOT NULL,
owner text,
timeout integer,
created integer,
token text,
scope integer,
depth integer,
uri text
);
CREATE TABLE principals (
id INTEGER PRIMARY KEY ASC NOT NULL,
uri TEXT NOT NULL,
email TEXT,
displayname TEXT,
UNIQUE(uri)
);
INSERT INTO principals VALUES(1,'principals/baikal','baikal@example.com','Baikal');
CREATE TABLE groupmembers (
id INTEGER PRIMARY KEY ASC NOT NULL,
principal_id INTEGER NOT NULL,
member_id INTEGER NOT NULL,
UNIQUE(principal_id, member_id)
);
CREATE TABLE propertystorage (
id integer primary key asc NOT NULL,
path text NOT NULL,
name text NOT NULL,
valuetype integer NOT NULL,
value string
);
CREATE TABLE users (
id integer primary key asc NOT NULL,
username TEXT NOT NULL,
digesta1 TEXT NOT NULL,
UNIQUE(username)
);
INSERT INTO users VALUES(1,'baikal','3b0845b235b7e985ce5905ab8df45e1a');
CREATE INDEX addressbookid_synctoken ON addressbookchanges (addressbookid, synctoken);
CREATE INDEX calendarid_synctoken ON calendarchanges (calendarid, synctoken);
CREATE INDEX principaluri_uri ON calendarsubscriptions (principaluri, uri);
CREATE UNIQUE INDEX path_property ON propertystorage (path, name);
COMMIT;

16
docker/baikal/start.sh Normal file
View file

@ -0,0 +1,16 @@
#!/bin/sh
# Shameless copied from https://raw.githubusercontent.com/ckulka/baikal-docker/master/files/start.sh
# Inject ServerName and ServerAlias if specified
APACHE_CONFIG="/etc/apache2/sites-available/000-default.conf"
if [ ! -z ${BAIKAL_SERVERNAME+x} ]
then
sed -i "s/# InjectedServerName .*/ServerName $BAIKAL_SERVERNAME/g" $APACHE_CONFIG
fi
if [ ! -z ${BAIKAL_SERVERALIAS+x} ]
then
sed -i "s/# InjectedServerAlias .*/ServerAlias $BAIKAL_SERVERALIAS/g" $APACHE_CONFIG
fi
apache2-foreground

5
docker/radicale/Dockerfile Normal file
View file

@ -0,0 +1,5 @@
FROM python:3.8
RUN pip install radicale
CMD radicale --storage-filesystem-folder /tmp/dav -H 0.0.0.0:8001 -D

13
docker/xandikos/Dockerfile Normal file
View file

@ -0,0 +1,13 @@
# Original file copyright 2017 Jelmer Vernooij
FROM ubuntu:bionic
RUN apt-get update && apt-get -y install xandikos locales
EXPOSE 8000
RUN locale-gen en_US.UTF-8
ENV PYTHONIOENCODING=utf-8
ENV LANG en_US.UTF-8
ENV LANGUAGE en_US:en
ENV LC_ALL en_US.UTF-8
CMD xandikos -d /tmp/dav -l 0.0.0.0 -p 8000 --autocreate

12
docs/contributing.rst
View file

@ -84,8 +84,8 @@ virtualenv_ and run this inside of it::
# - stylecheckers (flake8) and code formatters (autopep8)
make install-dev
# Install git commit hook for the stylechecker
make install-git-hooks
# Install git commit hook for some extra linting and checking
pre-commit install
# install test dependencies
make install-test
@ -100,9 +100,13 @@ The ``Makefile`` has a lot of options that allow you to control which tests are
run, and which servers are tested. Take a look at its code where they are all
initialized and documented.
For example, to test xandikos, run::
For example, to test xandikos, first run the server itself::
docker-compose build xandikos
docker-compose up -d xandikos
Then run the tests specifying this ``DAV_SERVER``, run::
make DAV_SERVER=xandikos install-test
make DAV_SERVER=xandikos test
If you have any questions, feel free to open issues about it.

11
scripts/make_travisconf.py
View file

@ -12,12 +12,14 @@ cfg['dist'] = 'trusty'
cfg['language'] = 'python'
cfg['cache'] = 'pip'
cfg['services'] = ['docker']
cfg['git'] = {
'submodules': False
}
cfg['branches'] = {
'only': ['auto', 'next', 'master', '/^.*-maintenance$/']
'only': ['master']
}
cfg['install'] = """
@ -43,13 +45,18 @@ for python, requirements in itertools.product(
):
dav_servers = ("radicale", "xandikos")
matrix.append({
'python': python,
'env': f"BUILD=test REQUIREMENTS={requirements}",
})
if python == latest_python and requirements == "release":
dav_servers += ("fastmail",)
for dav_server in dav_servers:
job = {
'python': python,
'env': ("BUILD=test "
'env': ("BUILD=test-storage "
"DAV_SERVER={dav_server} "
"REQUIREMENTS={requirements} "
.format(dav_server=dav_server,

3
setup.cfg
View file

@ -10,7 +10,8 @@ addopts = --tb=short
# E743: Ambiguous function definition
ignore = E731, E743
# E503: Line break occurred before a binary operator
extend-ignore = W503
extend-ignore = E203, W503
max-line-length = 88
select = C,E,F,W,B,B9
exclude = .eggs, tests/storage/servers/owncloud/, tests/storage/servers/nextcloud/, tests/storage/servers/baikal/, build/
application-package-names = tests,vdirsyncer

2
tests/storage/__init__.py
View file

@ -201,6 +201,8 @@ class StorageTests:
if getattr(self, 'dav_server', '') in \
('icloud', 'fastmail', 'davical'):
pytest.skip('Manual cleanup would be necessary.')
if getattr(self, 'dav_server', '') == "radicale":
pytest.skip("Radicale does not support collection creation")
args = get_storage_args(collection=None)
args['collection'] = 'test'

1
tests/storage/servers/baikal

@ -1 +0,0 @@
Subproject commit 6c8c379f1ee8bf4ab0ac54fc4eec3e4a6349c237

24
tests/storage/servers/baikal/__init__.py Normal file
View file

@ -0,0 +1,24 @@
import pytest
class ServerMixin:
@pytest.fixture
def get_storage_args(self, request, tmpdir, slow_create_collection):
def inner(collection="test"):
base_url = "http://127.0.0.1:8002/"
args = {
"url": base_url,
"username": "baikal",
"password": "baikal",
}
if self.storage_class.fileext == '.vcf':
args['url'] = base_url + "card.php/"
else:
args['url'] = base_url + "cal.php/"
if collection is not None:
args = slow_create_collection(self.storage_class, args, collection)
return args
return inner

4
tests/storage/servers/baikal/install.sh Normal file
View file

@ -0,0 +1,4 @@
#!/bin/sh
docker-compose build baikal
docker-compose up -d baikal

26
tests/storage/servers/fastmail/__init__.py
View file

@ -4,24 +4,28 @@ import pytest
class ServerMixin:
@pytest.fixture
def get_storage_args(self, slow_create_collection):
def inner(collection='test'):
def get_storage_args(self, item_type, slow_create_collection):
if item_type == "VTODO":
# Fastmail has non-standard support for TODOs
# See https://github.com/pimutils/vdirsyncer/issues/824
pytest.skip("Fastmail has non-standard VTODO support.")
def inner(collection="test"):
args = {
'username': os.environ['FASTMAIL_USERNAME'],
'password': os.environ['FASTMAIL_PASSWORD']
"username": os.environ["FASTMAIL_USERNAME"],
"password": os.environ["FASTMAIL_PASSWORD"],
}
if self.storage_class.fileext == '.ics':
args['url'] = 'https://caldav.fastmail.com/'
elif self.storage_class.fileext == '.vcf':
args['url'] = 'https://carddav.fastmail.com/'
if self.storage_class.fileext == ".ics":
args["url"] = "https://caldav.fastmail.com/"
elif self.storage_class.fileext == ".vcf":
args["url"] = "https://carddav.fastmail.com/"
else:
raise RuntimeError()
if collection is not None:
args = slow_create_collection(self.storage_class, args,
collection)
args = slow_create_collection(self.storage_class, args, collection)
return args
return inner

58
tests/storage/servers/radicale/__init__.py
View file

@ -1,57 +1,19 @@
import logging
import pytest
import radicale
import radicale.config
from pkg_resources import parse_version as ver
import wsgi_intercept
import wsgi_intercept.requests_intercept
logger = logging.getLogger(__name__)
class ServerMixin:
@pytest.fixture(autouse=True)
def setup(self, request, tmpdir):
if ver(radicale.VERSION) < ver('2.0.0-pre'):
raise RuntimeError('Testing against Radicale only works with '
'Radicale >= 2.0.0')
def get_app():
config = radicale.config.load(())
config.set('storage', 'filesystem_folder', str(tmpdir))
config.set('rights', 'type', 'owner_only')
app = radicale.Application(config, logger)
def is_authenticated(user, password):
return user == 'bob' and password == 'bob'
app.is_authenticated = is_authenticated
return app
wsgi_intercept.requests_intercept.install()
wsgi_intercept.add_wsgi_intercept('127.0.0.1', 80, get_app)
def teardown():
wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 80)
wsgi_intercept.requests_intercept.uninstall()
request.addfinalizer(teardown)
@pytest.fixture
def get_storage_args(self, get_item):
def inner(collection='test'):
url = 'http://127.0.0.1/'
rv = {'url': url, 'username': 'bob', 'password': 'bob'}
def get_storage_args(self, request, tmpdir, slow_create_collection):
def inner(collection="test"):
url = "http://127.0.0.1:8001/"
args = {
"url": url,
"username": "radicale",
"password": "radicale",
}
if collection is not None:
collection = collection + self.storage_class.fileext
rv = self.storage_class.create_collection(collection, **rv)
s = self.storage_class(**rv)
assert not list(s.list())
args = slow_create_collection(self.storage_class, args, collection)
return args
return rv
return inner

12
tests/storage/servers/radicale/install.sh
View file

@ -1,12 +1,4 @@
#!/bin/sh
set -e
if [ "$REQUIREMENTS" = "release" ] || [ "$REQUIREMENTS" = "minimal" ]; then
radicale_pkg="radicale==2.1.10"
elif [ "$REQUIREMENTS" = "devel" ]; then
radicale_pkg="git+https://github.com/Kozea/Radicale.git"
else
echo "Invalid requirements envvar"
false
fi
pip install wsgi_intercept $radicale_pkg
docker-compose build radicale
docker-compose up -d radicale

1
tests/storage/servers/skip/install.sh
View file

@ -1 +0,0 @@
#!/bin/sh

30
tests/storage/servers/xandikos/__init__.py
View file

@ -1,35 +1,15 @@
import pytest
from xandikos.web import XandikosApp, XandikosBackend, WellknownRedirector
import wsgi_intercept
import wsgi_intercept.requests_intercept
class ServerMixin:
@pytest.fixture
def get_storage_args(self, request, tmpdir, slow_create_collection):
tmpdir.mkdir('xandikos')
backend = XandikosBackend(path=str(tmpdir))
cup = '/user/'
backend.create_principal(cup, create_defaults=True)
app = XandikosApp(backend, cup)
app = WellknownRedirector(app, '/')
wsgi_intercept.requests_intercept.install()
wsgi_intercept.add_wsgi_intercept('127.0.0.1', 8080, lambda: app)
def teardown():
wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 8080)
wsgi_intercept.requests_intercept.uninstall()
request.addfinalizer(teardown)
def inner(collection='test'):
url = 'http://127.0.0.1:8080/'
args = {'url': url, 'collection': collection}
def inner(collection="test"):
url = "http://127.0.0.1:8000/"
args = {"url": url}
if collection is not None:
args = self.storage_class.create_collection(**args)
args = slow_create_collection(self.storage_class, args, collection)
return args
return inner

14
tests/storage/servers/xandikos/install.sh
View file

@ -1,14 +1,4 @@
#!/bin/sh
set -e
pip install wsgi_intercept
if [ "$REQUIREMENTS" = "release" ] || [ "$REQUIREMENTS" = "minimal" ]; then
# XXX: This is the last version to support Python 3.5
pip install -U "xandikos==0.0.11"
elif [ "$REQUIREMENTS" = "devel" ]; then
pip install -U git+https://github.com/jelmer/xandikos
else
echo "Invalid REQUIREMENTS value"
false
fi
docker-compose build xandikos
docker-compose up -d xandikos