CI refactor (#713)

* Switch to CircleCI

* add circleci badge

.circleci/config.yml

@@ -0,0 +1,183 @@
+version: 2
+
+references:
+  basic_env: &basic_env
+    CI: true
+  restore_caches: &restore_caches
+    restore_cache:
+      keys:
+        - cache-{{ arch }}-{{ .Branch }}
+
+  save_caches: &save_caches
+    save_cache:
+      key: cache-{{ arch }}-{{ .Branch }}
+      paths:
+        - "rust/target/"
+        - "~/.cargo/"
+        - "~/.cache/pip/"
+        - "~/.rustup/"
+
+  basic_setup: &basic_setup
+    run: . scripts/circleci-install.sh
+
+jobs:
+  nextcloud:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          NEXTCLOUD_HOST: localhost:80
+          DAV_SERVER: nextcloud
+      - image: nextcloud
+        environment:
+          SQLITE_DATABASE: nextcloud
+          NEXTCLOUD_ADMIN_USER: asdf
+          NEXTCLOUD_ADMIN_PASSWORD: asdf
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: wget -O - --retry-connrefused http://localhost:80/
+      - run: make -e storage-test
+
+  fastmail:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          DAV_SERVER: fastmail
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e storage-test
+
+  style:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e style
+
+  py34-minimal:
+    docker:
+      - image: circleci/python:3.4
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: minimal
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e test
+
+  py34-release:
+    docker:
+      - image: circleci/python:3.4
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: release
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e test
+
+  py34-devel:
+    docker:
+      - image: circleci/python:3.4
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: devel
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e test
+
+  py36-minimal:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: minimal
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e test
+
+  py36-release:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: release
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e test
+
+  py36-devel:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: devel
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e test
+
+  rust:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          REQUIREMENTS: release
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - *save_caches
+
+      - run: make -e rust-test
+
+workflows:
+  version: 2
+  test_all:
+    jobs:
+      - nextcloud
+      - fastmail
+      - style
+      - py34-minimal
+      - py34-release
+      - py34-devel
+      - py36-minimal
+      - py36-release
+      - py36-devel
+      - rust

.gitignore

@@ -15,3 +15,4 @@ docs/_build/
 vdirsyncer/version.py
 vdirsyncer/_native*
 .hypothesis
+codecov.sh

.travis.yml

@@ -1,138 +0,0 @@
-{
-  "branches": {
-    "only": [
-      "auto",
-      "master",
-      "/^.*-maintenance$/"
-    ]
-  },
-  "cache": {
-    "directories": [
-      "./rust/target/"
-    ],
-    "pip": true
-  },
-  "dist": "trusty",
-  "git": {
-    "submodules": false
-  },
-  "install": [
-    ". scripts/travis-install.sh",
-    "pip install -U pip setuptools",
-    "pip install wheel",
-    "make -e install-dev",
-    "make -e install-$BUILD"
-  ],
-  "language": "python",
-  "matrix": {
-    "fast_finish": true,
-    "include": [
-      {
-        "env": "BUILD=style",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=devel ",
-        "python": "3.4"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=devel ",
-        "python": "3.4"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ",
-        "python": "3.4"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ",
-        "python": "3.4"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ",
-        "python": "3.4"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ",
-        "python": "3.4"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=devel ",
-        "python": "3.5"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=devel ",
-        "python": "3.5"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ",
-        "python": "3.5"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ",
-        "python": "3.5"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ",
-        "python": "3.5"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ",
-        "python": "3.5"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=devel ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=devel ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=release ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=release ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=nextcloud REQUIREMENTS=release ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=icloud REQUIREMENTS=release ",
-        "if": "NOT (type IN (pull_request))",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=fastmail REQUIREMENTS=release ",
-        "if": "NOT (type IN (pull_request))",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test DAV_SERVER=xandikos REQUIREMENTS=minimal ",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test ETESYNC_TESTS=true REQUIREMENTS=latest",
-        "python": "3.6"
-      },
-      {
-        "env": "BUILD=test",
-        "language": "generic",
-        "os": "osx"
-      }
-    ]
-  },
-  "script": [
-    "make -e $BUILD"
-  ],
-  "services": [
-    "docker"
-  ],
-  "sudo": true
-}

Makefile

@@ -48,19 +48,30 @@ all:
 	$(error Take a look at https://vdirsyncer.pimutils.org/en/stable/tutorial.html#installation)
 
 ifeq ($(CI), true)
-test:
+codecov.sh:
-	curl -s https://codecov.io/bash > $(CODECOV_PATH)
+	curl -s https://codecov.io/bash > $@
-	$(PYTEST) tests/unit/
-	bash $(CODECOV_PATH) -c -F unit
-	$(PYTEST) tests/system/
-	bash $(CODECOV_PATH) -c -F system
-	$(PYTEST) tests/storage/
-	bash $(CODECOV_PATH) -c -F storage
 else
-test:
+codecov.sh:
-	$(PYTEST)
+	echo > $@
 endif
 
+rust-test:
+	cd rust/ && cargo test --release
+
+test: unit-test system-test storage-test
+
+unit-test: codecov.sh
+	$(PYTEST) tests/unit/
+	bash codecov.sh -c -F unit
+
+system-test: codecov.sh
+	$(PYTEST) tests/system/
+	bash codecov.sh -c -F system
+
+storage-test: codecov.sh
+	$(PYTEST) tests/storage/
+	bash codecov.sh -c -F storage
+
 install-servers:
 	set -ex; \
 	for server in $(DAV_SERVER); do \
@@ -88,10 +99,6 @@ style:
 	! git grep -i syncroniz */*
 	! git grep -i 'text/icalendar' */*
 	sphinx-build -W -b html ./docs/ ./docs/_build/html/
-	python3 scripts/make_travisconf.py | diff -b .travis.yml -
-
-travis-conf:
-	python3 scripts/make_travisconf.py > .travis.yml
 
 install-docs:
 	pip install -Ur docs-requirements.txt

README.rst

@@ -20,8 +20,8 @@ It aims to be for calendars and contacts what `OfflineIMAP
 
 .. _programs: https://vdirsyncer.pimutils.org/en/latest/tutorials/
 
-.. image:: https://travis-ci.org/pimutils/vdirsyncer.svg?branch=master
+.. image:: https://circleci.com/gh/pimutils/vdirsyncer.svg?style=svg
-    :target: https://travis-ci.org/pimutils/vdirsyncer
+    :target: https://circleci.com/gh/pimutils/vdirsyncer
 
 .. image:: https://codecov.io/github/pimutils/vdirsyncer/coverage.svg?branch=master
     :target: https://codecov.io/github/pimutils/vdirsyncer?branch=master

scripts/circleci-install.sh

@@ -0,0 +1,13 @@
+make install-rust
+echo "export PATH=$HOME/.cargo/bin/:$PATH" >> $BASH_ENV
+
+pip install --user virtualenv
+~/.local/bin/virtualenv ~/env
+echo ". ~/env/bin/activate" >> $BASH_ENV
+. $BASH_ENV
+pip install docker-compose
+
+make -e install-dev install-test
+if python --version | grep -q 'Python 3.6'; then
+    make -e install-style
+fi

scripts/make_travisconf.py

@@ -1,84 +0,0 @@
-import itertools
-import json
-import sys
-
-python_versions = ("3.4", "3.5", "3.6")
-latest_python = "3.6"
-
-cfg = {}
-
-cfg['sudo'] = True
-cfg['services'] = ['docker']
-cfg['dist'] = 'trusty'
-cfg['language'] = 'python'
-cfg['cache'] = {
-    'pip': True,
-    'directories': [
-        './rust/target/'
-    ]
-}
-
-cfg['git'] = {
-    'submodules': False
-}
-
-cfg['branches'] = {
-    'only': ['auto', 'master', '/^.*-maintenance$/']
-}
-
-cfg['install'] = """
-. scripts/travis-install.sh
-pip install -U pip setuptools
-pip install wheel
-make -e install-dev
-make -e install-$BUILD
-""".strip().splitlines()
-
-cfg['script'] = ["make -e $BUILD"]
-
-matrix = []
-cfg['matrix'] = {'include': matrix, 'fast_finish': True}
-
-matrix.append({
-    'python': latest_python,
-    'env': 'BUILD=style'
-})
-
-
-for python, requirements in itertools.product(python_versions,
-                                              ("devel", "release", "minimal")):
-    dav_servers = ("radicale", "xandikos")
-
-    if python == latest_python and requirements == "release":
-        dav_servers += ("nextcloud", "icloud", "fastmail")
-
-    for dav_server in dav_servers:
-        job = {
-            'python': python,
-            'env': ("BUILD=test "
-                    "DAV_SERVER={dav_server} "
-                    "REQUIREMENTS={requirements} "
-                    .format(dav_server=dav_server,
-                            requirements=requirements))
-        }
-
-        build_prs = dav_server not in ("fastmail", "davical", "icloud")
-        if not build_prs:
-            job['if'] = 'NOT (type IN (pull_request))'
-
-        matrix.append(job)
-
-matrix.append({
-    'python': latest_python,
-    'env': ("BUILD=test "
-            "ETESYNC_TESTS=true "
-            "REQUIREMENTS=latest")
-})
-
-matrix.append({
-    'language': 'generic',
-    'os': 'osx',
-    'env': 'BUILD=test'
-})
-
-json.dump(cfg, sys.stdout, sort_keys=True, indent=2)

scripts/travis-install.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-
-# The OS X VM doesn't have any Python support at all
-# See https://github.com/travis-ci/travis-ci/issues/2312
-if [ "$TRAVIS_OS_NAME" = "osx" ]; then
-    brew update
-    brew install python3
-    virtualenv -p python3 $HOME/osx-py3
-    . $HOME/osx-py3/bin/activate
-fi
-
-make install-rust
-export PATH="$HOME/.cargo/bin/:$PATH"

tests/storage/servers/nextcloud/__init__.py

@@ -1,37 +1,25 @@
+import os
 import requests
 import pytest
 
 
-def is_responsive(url):
+port = os.environ.get('NEXTCLOUD_HOST', None) or 'localhost:8080'
-    try:
+user = os.environ.get('NEXTCLOUD_USER', None) or 'asdf'
-        requests.get(url)
+pwd = os.environ.get('NEXTCLOUD_PASS', None) or 'asdf'
-        return True
-    except Exception:
-        return False
 
 
 class ServerMixin(object):
     storage_class = None
     wsgi_teardown = None
 
-    @pytest.fixture(scope='session')
-    def nextcloud_server(self, docker_ip, docker_services):
-        url = 'http://{}:{}'.format(docker_ip,
-                                    docker_services.port_for('nextcloud', 80))
-        docker_services.wait_until_responsive(
-            timeout=30.0, pause=0.1,
-            check=lambda: is_responsive(url)
-        )
-        return url
-
     @pytest.fixture
-    def get_storage_args(self, nextcloud_server, item_type,
+    def get_storage_args(self, item_type,
                          slow_create_collection):
         def inner(collection='test'):
             args = {
-                'username': 'asdf',
+                'username': user,
-                'password': 'asdf',
+                'password': pwd,
-                'url': nextcloud_server + '/remote.php/dav/'
+                'url': 'http://{}/remote.php/dav/'.format(port)
             }
 
             if collection is not None:

tests/storage/servers/nextcloud/install.sh

@@ -1 +0,0 @@
-pip install pytest-docker