From 2a2457e364fa92138bb40323da0e30abff9a6136 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 9 Feb 2018 20:50:48 +0100 Subject: [PATCH] CI refactor (#713) * Switch to CircleCI * add circleci badge --- .circleci/config.yml | 183 ++++++++++++++++++ .gitignore | 1 + .travis.yml | 138 ------------- Makefile | 35 ++-- README.rst | 4 +- .../docker-compose.yml => docker-compose.yml | 0 scripts/circleci-install.sh | 13 ++ scripts/make_travisconf.py | 84 -------- scripts/travis-install.sh | 13 -- tests/storage/servers/nextcloud/__init__.py | 28 +-- tests/storage/servers/nextcloud/install.sh | 1 - 11 files changed, 228 insertions(+), 272 deletions(-) create mode 100644 .circleci/config.yml delete mode 100644 .travis.yml rename tests/docker-compose.yml => docker-compose.yml (100%) create mode 100644 scripts/circleci-install.sh delete mode 100644 scripts/make_travisconf.py delete mode 100644 scripts/travis-install.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..a726062 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,183 @@ +version: 2 + +references: + basic_env: &basic_env + CI: true + restore_caches: &restore_caches + restore_cache: + keys: + - cache-{{ arch }}-{{ .Branch }} + + save_caches: &save_caches + save_cache: + key: cache-{{ arch }}-{{ .Branch }} + paths: + - "rust/target/" + - "~/.cargo/" + - "~/.cache/pip/" + - "~/.rustup/" + + basic_setup: &basic_setup + run: . scripts/circleci-install.sh + +jobs: + nextcloud: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + NEXTCLOUD_HOST: localhost:80 + DAV_SERVER: nextcloud + - image: nextcloud + environment: + SQLITE_DATABASE: nextcloud + NEXTCLOUD_ADMIN_USER: asdf + NEXTCLOUD_ADMIN_PASSWORD: asdf + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: wget -O - --retry-connrefused http://localhost:80/ + - run: make -e storage-test + + fastmail: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + DAV_SERVER: fastmail + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e storage-test + + style: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e style + + py34-minimal: + docker: + - image: circleci/python:3.4 + environment: + <<: *basic_env + REQUIREMENTS: minimal + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e test + + py34-release: + docker: + - image: circleci/python:3.4 + environment: + <<: *basic_env + REQUIREMENTS: release + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e test + + py34-devel: + docker: + - image: circleci/python:3.4 + environment: + <<: *basic_env + REQUIREMENTS: devel + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e test + + py36-minimal: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + REQUIREMENTS: minimal + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e test + + py36-release: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + REQUIREMENTS: release + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e test + + py36-devel: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + REQUIREMENTS: devel + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e test + + rust: + docker: + - image: circleci/python:3.6 + environment: + <<: *basic_env + REQUIREMENTS: release + steps: + - checkout + - *restore_caches + - *basic_setup + - *save_caches + + - run: make -e rust-test + +workflows: + version: 2 + test_all: + jobs: + - nextcloud + - fastmail + - style + - py34-minimal + - py34-release + - py34-devel + - py36-minimal + - py36-release + - py36-devel + - rust diff --git a/.gitignore b/.gitignore index 0008fb9..54ad956 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ docs/_build/ vdirsyncer/version.py vdirsyncer/_native* .hypothesis +codecov.sh diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6448382..0000000 --- a/.travis.yml +++ /dev/null @@ -1,138 +0,0 @@ -{ - "branches": { - "only": [ - "auto", - "master", - "/^.*-maintenance$/" - ] - }, - "cache": { - "directories": [ - "./rust/target/" - ], - "pip": true - }, - "dist": "trusty", - "git": { - "submodules": false - }, - "install": [ - ". scripts/travis-install.sh", - "pip install -U pip setuptools", - "pip install wheel", - "make -e install-dev", - "make -e install-$BUILD" - ], - "language": "python", - "matrix": { - "fast_finish": true, - "include": [ - { - "env": "BUILD=style", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=devel ", - "python": "3.4" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=devel ", - "python": "3.4" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ", - "python": "3.4" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ", - "python": "3.4" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ", - "python": "3.4" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ", - "python": "3.4" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=devel ", - "python": "3.5" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=devel ", - "python": "3.5" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ", - "python": "3.5" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ", - "python": "3.5" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ", - "python": "3.5" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ", - "python": "3.5" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=devel ", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=devel ", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=nextcloud REQUIREMENTS=release ", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=icloud REQUIREMENTS=release ", - "if": "NOT (type IN (pull_request))", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=fastmail REQUIREMENTS=release ", - "if": "NOT (type IN (pull_request))", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ", - "python": "3.6" - }, - { - "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ", - "python": "3.6" - }, - { - "env": "BUILD=test ETESYNC_TESTS=true REQUIREMENTS=latest", - "python": "3.6" - }, - { - "env": "BUILD=test", - "language": "generic", - "os": "osx" - } - ] - }, - "script": [ - "make -e $BUILD" - ], - "services": [ - "docker" - ], - "sudo": true -} \ No newline at end of file diff --git a/Makefile b/Makefile index ad244e7..9579a30 100644 --- a/Makefile +++ b/Makefile @@ -48,19 +48,30 @@ all: $(error Take a look at https://vdirsyncer.pimutils.org/en/stable/tutorial.html#installation) ifeq ($(CI), true) -test: - curl -s https://codecov.io/bash > $(CODECOV_PATH) - $(PYTEST) tests/unit/ - bash $(CODECOV_PATH) -c -F unit - $(PYTEST) tests/system/ - bash $(CODECOV_PATH) -c -F system - $(PYTEST) tests/storage/ - bash $(CODECOV_PATH) -c -F storage +codecov.sh: + curl -s https://codecov.io/bash > $@ else -test: - $(PYTEST) +codecov.sh: + echo > $@ endif +rust-test: + cd rust/ && cargo test --release + +test: unit-test system-test storage-test + +unit-test: codecov.sh + $(PYTEST) tests/unit/ + bash codecov.sh -c -F unit + +system-test: codecov.sh + $(PYTEST) tests/system/ + bash codecov.sh -c -F system + +storage-test: codecov.sh + $(PYTEST) tests/storage/ + bash codecov.sh -c -F storage + install-servers: set -ex; \ for server in $(DAV_SERVER); do \ @@ -88,10 +99,6 @@ style: ! git grep -i syncroniz */* ! git grep -i 'text/icalendar' */* sphinx-build -W -b html ./docs/ ./docs/_build/html/ - python3 scripts/make_travisconf.py | diff -b .travis.yml - - -travis-conf: - python3 scripts/make_travisconf.py > .travis.yml install-docs: pip install -Ur docs-requirements.txt diff --git a/README.rst b/README.rst index fb8c82d..a841ec5 100644 --- a/README.rst +++ b/README.rst @@ -20,8 +20,8 @@ It aims to be for calendars and contacts what `OfflineIMAP .. _programs: https://vdirsyncer.pimutils.org/en/latest/tutorials/ -.. image:: https://travis-ci.org/pimutils/vdirsyncer.svg?branch=master - :target: https://travis-ci.org/pimutils/vdirsyncer +.. image:: https://circleci.com/gh/pimutils/vdirsyncer.svg?style=svg + :target: https://circleci.com/gh/pimutils/vdirsyncer .. image:: https://codecov.io/github/pimutils/vdirsyncer/coverage.svg?branch=master :target: https://codecov.io/github/pimutils/vdirsyncer?branch=master diff --git a/tests/docker-compose.yml b/docker-compose.yml similarity index 100% rename from tests/docker-compose.yml rename to docker-compose.yml diff --git a/scripts/circleci-install.sh b/scripts/circleci-install.sh new file mode 100644 index 0000000..22db2b6 --- /dev/null +++ b/scripts/circleci-install.sh @@ -0,0 +1,13 @@ +make install-rust +echo "export PATH=$HOME/.cargo/bin/:$PATH" >> $BASH_ENV + +pip install --user virtualenv +~/.local/bin/virtualenv ~/env +echo ". ~/env/bin/activate" >> $BASH_ENV +. $BASH_ENV +pip install docker-compose + +make -e install-dev install-test +if python --version | grep -q 'Python 3.6'; then + make -e install-style +fi diff --git a/scripts/make_travisconf.py b/scripts/make_travisconf.py deleted file mode 100644 index a1a8576..0000000 --- a/scripts/make_travisconf.py +++ /dev/null @@ -1,84 +0,0 @@ -import itertools -import json -import sys - -python_versions = ("3.4", "3.5", "3.6") -latest_python = "3.6" - -cfg = {} - -cfg['sudo'] = True -cfg['services'] = ['docker'] -cfg['dist'] = 'trusty' -cfg['language'] = 'python' -cfg['cache'] = { - 'pip': True, - 'directories': [ - './rust/target/' - ] -} - -cfg['git'] = { - 'submodules': False -} - -cfg['branches'] = { - 'only': ['auto', 'master', '/^.*-maintenance$/'] -} - -cfg['install'] = """ -. scripts/travis-install.sh -pip install -U pip setuptools -pip install wheel -make -e install-dev -make -e install-$BUILD -""".strip().splitlines() - -cfg['script'] = ["make -e $BUILD"] - -matrix = [] -cfg['matrix'] = {'include': matrix, 'fast_finish': True} - -matrix.append({ - 'python': latest_python, - 'env': 'BUILD=style' -}) - - -for python, requirements in itertools.product(python_versions, - ("devel", "release", "minimal")): - dav_servers = ("radicale", "xandikos") - - if python == latest_python and requirements == "release": - dav_servers += ("nextcloud", "icloud", "fastmail") - - for dav_server in dav_servers: - job = { - 'python': python, - 'env': ("BUILD=test " - "DAV_SERVER={dav_server} " - "REQUIREMENTS={requirements} " - .format(dav_server=dav_server, - requirements=requirements)) - } - - build_prs = dav_server not in ("fastmail", "davical", "icloud") - if not build_prs: - job['if'] = 'NOT (type IN (pull_request))' - - matrix.append(job) - -matrix.append({ - 'python': latest_python, - 'env': ("BUILD=test " - "ETESYNC_TESTS=true " - "REQUIREMENTS=latest") -}) - -matrix.append({ - 'language': 'generic', - 'os': 'osx', - 'env': 'BUILD=test' -}) - -json.dump(cfg, sys.stdout, sort_keys=True, indent=2) diff --git a/scripts/travis-install.sh b/scripts/travis-install.sh deleted file mode 100644 index 8b7a3bc..0000000 --- a/scripts/travis-install.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh - -# The OS X VM doesn't have any Python support at all -# See https://github.com/travis-ci/travis-ci/issues/2312 -if [ "$TRAVIS_OS_NAME" = "osx" ]; then - brew update - brew install python3 - virtualenv -p python3 $HOME/osx-py3 - . $HOME/osx-py3/bin/activate -fi - -make install-rust -export PATH="$HOME/.cargo/bin/:$PATH" diff --git a/tests/storage/servers/nextcloud/__init__.py b/tests/storage/servers/nextcloud/__init__.py index a2b74e9..476461b 100644 --- a/tests/storage/servers/nextcloud/__init__.py +++ b/tests/storage/servers/nextcloud/__init__.py @@ -1,37 +1,25 @@ +import os import requests import pytest -def is_responsive(url): - try: - requests.get(url) - return True - except Exception: - return False +port = os.environ.get('NEXTCLOUD_HOST', None) or 'localhost:8080' +user = os.environ.get('NEXTCLOUD_USER', None) or 'asdf' +pwd = os.environ.get('NEXTCLOUD_PASS', None) or 'asdf' class ServerMixin(object): storage_class = None wsgi_teardown = None - @pytest.fixture(scope='session') - def nextcloud_server(self, docker_ip, docker_services): - url = 'http://{}:{}'.format(docker_ip, - docker_services.port_for('nextcloud', 80)) - docker_services.wait_until_responsive( - timeout=30.0, pause=0.1, - check=lambda: is_responsive(url) - ) - return url - @pytest.fixture - def get_storage_args(self, nextcloud_server, item_type, + def get_storage_args(self, item_type, slow_create_collection): def inner(collection='test'): args = { - 'username': 'asdf', - 'password': 'asdf', - 'url': nextcloud_server + '/remote.php/dav/' + 'username': user, + 'password': pwd, + 'url': 'http://{}/remote.php/dav/'.format(port) } if collection is not None: diff --git a/tests/storage/servers/nextcloud/install.sh b/tests/storage/servers/nextcloud/install.sh index 020bfc1..e69de29 100644 --- a/tests/storage/servers/nextcloud/install.sh +++ b/tests/storage/servers/nextcloud/install.sh @@ -1 +0,0 @@ -pip install pytest-docker