diff --git a/.travis.yml b/.travis.yml index c9e6235..3dc0e51 100644 --- a/.travis.yml +++ b/.travis.yml @@ -123,6 +123,10 @@ "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal BUILD_PRS=true ", "python": "pypy3" }, + { + "env": "BUILD=test ETESYNC_TESTS=true REQUIREMENTS=latest BUILD_PRS=true ", + "python": "3.6" + }, { "env": "BUILD=test BUILD_PRS=true", "language": "generic", diff --git a/Makefile b/Makefile index 3d4e9a3..0272cba 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,7 @@ export TESTSERVER_BASE := ./tests/storage/servers/ export CI := false export COVERAGE := $(CI) export DETERMINISTIC_TESTS := false +export ETESYNC_TESTS := false CODECOV_PATH = /tmp/codecov.sh @@ -18,6 +19,11 @@ ifeq ($(COVERAGE), true) PYTEST_ARGS += --cov-config .coveragerc --cov vdirsyncer endif +ifeq ($(ETESYNC_TESTS), true) + TEST_EXTRA_PACKAGES += git+https://github.com/etesync/journal-manager + TEST_EXTRA_PACKAGES += django djangorestframework wsgi_intercept +endif + ifeq ($(CI), true) test: curl -s https://codecov.io/bash > $(CODECOV_PATH) @@ -83,11 +89,9 @@ release: python setup.py sdist bdist_wheel upload install-dev: - set -xe && if [ "$(REMOTESTORAGE_SERVER)" != "skip" ]; then \ - pip install -e .[remotestorage]; \ - else \ - pip install -e .; \ - fi + pip install -e . + [ "$(REMOTESTORAGE_SERVER)" = "skip" ] || pip install -e .[remotestorage] + [ "$(ETESYNC_TESTS)" = "false" ] || pip install -e .[etesync] set -xe && if [ "$(REQUIREMENTS)" = "devel" ]; then \ pip install -U --force-reinstall \ git+https://github.com/mitsuhiko/click \ diff --git a/docs/config.rst b/docs/config.rst index 39cd63c..3cc3bfd 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -226,6 +226,24 @@ To use them, you need to install some optional dependencies with:: .. autostorage:: vdirsyncer.storage.remotestorage.RemoteStorageCalendars +EteSync ++++++++ + +`EteSync `_ is a new cloud provider for end to end +encrypted contacts and calendar storage. Vdirsyncer contains **experimental** +support for it. + +To use it, you need to install some optional dependencies:: + + pip install vdirsyncer[etesync] + +On first usage you will be prompted for the service password and the encryption +password. Neither are stored. + +.. autostorage:: vdirsyncer.storage.etesync.EtesyncContacts + +.. autostorage:: vdirsyncer.storage.etesync.EtesyncCalendars + Local +++++ diff --git a/scripts/make_travisconf.py b/scripts/make_travisconf.py index c7e4b66..0d2133d 100644 --- a/scripts/make_travisconf.py +++ b/scripts/make_travisconf.py @@ -71,6 +71,14 @@ for python, requirements in itertools.product(python_versions, build_prs=build_prs and "true" or "false")) }) +matrix.append({ + 'python': latest_python, + 'env': ("BUILD=test " + "ETESYNC_TESTS=true " + "REQUIREMENTS=latest " + "BUILD_PRS=true ") +}) + matrix.append({ 'language': 'generic', 'os': 'osx', diff --git a/setup.py b/setup.py index 6c89e4b..d0ff448 100644 --- a/setup.py +++ b/setup.py @@ -72,6 +72,7 @@ setup( extras_require={ 'remotestorage': ['requests-oauthlib'], 'google': ['requests-oauthlib'], + 'etesync': ['etesync'] }, # Build dependencies diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 574a371..697b4ce 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -226,6 +226,9 @@ class StorageTests(object): assert 'collection argument must not be given' in str(excinfo.value) def test_collection_arg(self, get_storage_args): + if self.storage_class.storage_name.startswith('etesync'): + pytest.skip('etesync uses UUIDs.') + if self.supports_collections: s = self.storage_class(**get_storage_args(collection='test2')) # Can't do stronger assertion because of radicale, which needs a @@ -270,6 +273,10 @@ class StorageTests(object): (_, etag3), = s.list() assert etag2 == etag3 + # etesync uses UUIDs for collection names + if self.storage_class.storage_name.startswith('etesync'): + return + assert collection in urlunquote(s.collection) if self.storage_class.storage_name.endswith('dav'): assert urlquote(uid, '/@:') in href diff --git a/tests/storage/etesync/__init__.py b/tests/storage/etesync/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/storage/etesync/etesync_server/db.sqlite3 b/tests/storage/etesync/etesync_server/db.sqlite3 new file mode 100644 index 0000000..a3d9983 Binary files /dev/null and b/tests/storage/etesync/etesync_server/db.sqlite3 differ diff --git a/tests/storage/etesync/etesync_server/etesync_server/__init__.py b/tests/storage/etesync/etesync_server/etesync_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/storage/etesync/etesync_server/etesync_server/settings.py b/tests/storage/etesync/etesync_server/etesync_server/settings.py new file mode 100644 index 0000000..4283f1b --- /dev/null +++ b/tests/storage/etesync/etesync_server/etesync_server/settings.py @@ -0,0 +1,124 @@ +""" +Django settings for etesync_server project. + +Generated by 'django-admin startproject' using Django 1.10.6. + +For more information on this file, see +https://docs.djangoproject.com/en/1.10/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/1.10/ref/settings/ +""" + +import os + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = 'd7r(p-9=$3a@bbt%*+$p@4)cej13nzd0gmnt8+m0bitb=-umj#' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'rest_framework', + 'rest_framework.authtoken', + 'journal.apps.JournalConfig', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'etesync_server.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'etesync_server.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/1.10/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.environ.get('ETESYNC_DB_PATH', + os.path.join(BASE_DIR, 'db.sqlite3')), + } +} + + +# Password validation +# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', # noqa + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', # noqa + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', # noqa + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', # noqa + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/1.10/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/1.10/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/tests/storage/etesync/etesync_server/etesync_server/urls.py b/tests/storage/etesync/etesync_server/etesync_server/urls.py new file mode 100644 index 0000000..ddcb807 --- /dev/null +++ b/tests/storage/etesync/etesync_server/etesync_server/urls.py @@ -0,0 +1,38 @@ +"""etesync_server URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/1.10/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.conf.urls import url, include + 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) +""" +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import admin + +from rest_framework import routers +from rest_framework.authtoken import views as token_views + +from journal import views + +router = routers.DefaultRouter() +router.register(r'journals', views.JournalViewSet) +router.register(r'journal/(?P[^/]+)', views.EntryViewSet) + +urlpatterns = [ + url(r'^api/v1/', include(router.urls)), + url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), # noqa + url(r'^api-token-auth/', token_views.obtain_auth_token), +] + +if settings.DEBUG: + urlpatterns += url(r'^reset/$', views.reset, name='reset_debug'), + +urlpatterns += url(r'^admin/', admin.site.urls), diff --git a/tests/storage/etesync/etesync_server/etesync_server/wsgi.py b/tests/storage/etesync/etesync_server/etesync_server/wsgi.py new file mode 100644 index 0000000..eeb40bc --- /dev/null +++ b/tests/storage/etesync/etesync_server/etesync_server/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for etesync_server project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings") + +application = get_wsgi_application() diff --git a/tests/storage/etesync/etesync_server/manage.py b/tests/storage/etesync/etesync_server/manage.py new file mode 100755 index 0000000..b100c54 --- /dev/null +++ b/tests/storage/etesync/etesync_server/manage.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os +import sys + +if __name__ == "__main__": + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings") + try: + from django.core.management import execute_from_command_line + except ImportError: + # The above import may fail for some other reason. Ensure that the + # issue is really that Django is missing to avoid masking other + # exceptions on Python 2. + try: + import django # noqa + except ImportError: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) + raise + execute_from_command_line(sys.argv) diff --git a/tests/storage/etesync/test@localhost/auth_token b/tests/storage/etesync/test@localhost/auth_token new file mode 100644 index 0000000..7eeeadb --- /dev/null +++ b/tests/storage/etesync/test@localhost/auth_token @@ -0,0 +1 @@ +63ae6eec45b592d5c511f79b7b0c312d2c5f7d6a diff --git a/tests/storage/etesync/test@localhost/key b/tests/storage/etesync/test@localhost/key new file mode 100644 index 0000000..0e3b27d Binary files /dev/null and b/tests/storage/etesync/test@localhost/key differ diff --git a/tests/storage/etesync/test_main.py b/tests/storage/etesync/test_main.py new file mode 100644 index 0000000..9add484 --- /dev/null +++ b/tests/storage/etesync/test_main.py @@ -0,0 +1,92 @@ +# -*- coding: utf-8 -*- + +import shutil +import os +import sys + + +import pytest +import requests + +from vdirsyncer.storage.etesync import EtesyncContacts, EtesyncCalendars + +from .. import StorageTests + + +pytestmark = pytest.mark.skipif(os.getenv('ETESYNC_TESTS', '') != 'true', + reason='etesync tests disabled') + + +@pytest.fixture(scope='session') +def etesync_app(tmpdir_factory): + sys.path.insert(0, os.path.join(os.path.dirname(__file__), + 'etesync_server')) + + db = tmpdir_factory.mktemp('etesync').join('etesync.sqlite') + shutil.copy( + os.path.join(os.path.dirname(__file__), 'etesync_server', + 'db.sqlite3'), + str(db) + ) + + os.environ['ETESYNC_DB_PATH'] = str(db) + from etesync_server.wsgi import application + return application + + +class EtesyncTests(StorageTests): + + supports_metadata = False + + @pytest.fixture + def get_storage_args(self, request, get_item, tmpdir, etesync_app): + import wsgi_intercept + import wsgi_intercept.requests_intercept + wsgi_intercept.requests_intercept.install() + wsgi_intercept.add_wsgi_intercept('127.0.0.1', 8000, + lambda: etesync_app) + + def teardown(): + wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 8000) + wsgi_intercept.requests_intercept.uninstall() + + request.addfinalizer(teardown) + + with open(os.path.join(os.path.dirname(__file__), + 'test@localhost/auth_token')) as f: + token = f.read().strip() + headers = {'Authorization': 'Token ' + token} + r = requests.post('http://127.0.0.1:8000/reset/', headers=headers, + allow_redirects=False) + assert r.status_code == 200 + + def inner(collection='test'): + rv = { + 'email': 'test@localhost', + 'db_path': str(tmpdir.join('etesync.db')), + 'secrets_dir': os.path.dirname(__file__), + 'server_url': 'http://127.0.0.1:8000/' + } + if collection is not None: + rv = self.storage_class.create_collection( + collection=collection, + **rv + ) + return rv + return inner + + +class TestContacts(EtesyncTests): + storage_class = EtesyncContacts + + @pytest.fixture(params=['VCARD']) + def item_type(self, request): + return request.param + + +class TestCalendars(EtesyncTests): + storage_class = EtesyncCalendars + + @pytest.fixture(params=['VEVENT']) + def item_type(self, request): + return request.param diff --git a/tests/storage/test_http_with_singlefile.py b/tests/storage/test_http_with_singlefile.py index b8bb8bd..9f86ebd 100644 --- a/tests/storage/test_http_with_singlefile.py +++ b/tests/storage/test_http_with_singlefile.py @@ -15,6 +15,7 @@ class CombinedStorage(Storage): '''A subclass of HttpStorage to make testing easier. It supports writes via SingleFileStorage.''' _repr_attributes = ('url', 'path') + storage_name = 'http_and_singlefile' def __init__(self, url, path, **kwargs): if kwargs.get('collection', None) is not None: diff --git a/vdirsyncer/cli/utils.py b/vdirsyncer/cli/utils.py index d53215d..30f1a83 100644 --- a/vdirsyncer/cli/utils.py +++ b/vdirsyncer/cli/utils.py @@ -39,7 +39,9 @@ class _StorageIndex(object): remotestorage_calendars=( 'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'), google_calendar='vdirsyncer.storage.google.GoogleCalendarStorage', - google_contacts='vdirsyncer.storage.google.GoogleContactsStorage' + google_contacts='vdirsyncer.storage.google.GoogleContactsStorage', + etesync_calendars='vdirsyncer.storage.etesync.EtesyncCalendars', + etesync_contacts='vdirsyncer.storage.etesync.EtesyncContacts' ) def __getitem__(self, name): diff --git a/vdirsyncer/storage/etesync.py b/vdirsyncer/storage/etesync.py new file mode 100644 index 0000000..85e1f01 --- /dev/null +++ b/vdirsyncer/storage/etesync.py @@ -0,0 +1,244 @@ +import contextlib +import functools +import logging +import os +import binascii + +import atomicwrites +import click + +try: + import etesync + import etesync.exceptions + from etesync import AddressBook, Contact, Calendar, Event + has_etesync = True +except ImportError: + has_etesync = False + AddressBook = Contact = Calendar = Event = None + +from .. import exceptions +from ..cli.utils import assert_permissions +from ..utils import checkdir +from ..vobject import Item + +from .base import Storage + + +logger = logging.getLogger(__name__) + + +def _writing_op(f): + @functools.wraps(f) + def inner(self, *args, **kwargs): + if not self._at_once: + self._sync_journal() + rv = f(self, *args, **kwargs) + if not self._at_once: + self._sync_journal() + return rv + return inner + + +class _Session: + def __init__(self, email, secrets_dir, server_url=None, db_path=None): + if not has_etesync: + raise exceptions.UserError('Dependencies for etesync are not ' + 'installed.') + server_url = server_url or etesync.API_URL + self.email = email + self.secrets_dir = os.path.join(secrets_dir, email + '/') + + self._auth_token_path = os.path.join(self.secrets_dir, 'auth_token') + self._key_path = os.path.join(self.secrets_dir, 'key') + + auth_token = self._get_auth_token() + if not auth_token: + password = click.prompt('Enter service password for {}' + .format(self.email), hide_input=True) + auth_token = etesync.Authenticator(server_url) \ + .get_auth_token(self.email, password) + self._set_auth_token(auth_token) + + self._db_path = db_path or os.path.join(self.secrets_dir, 'db.sqlite') + self.etesync = etesync.EteSync(email, auth_token, remote=server_url, + db_path=self._db_path) + + key = self._get_key() + if not key: + password = click.prompt('Enter key password', hide_input=True) + click.echo('Deriving key for {}'.format(self.email)) + self.etesync.derive_key(password) + self._set_key(self.etesync.cipher_key) + else: + self.etesync.cipher_key = key + + def _get_auth_token(self): + try: + with open(self._auth_token_path) as f: + return f.read().strip() or None + except (OSError, IOError): + pass + + def _set_auth_token(self, token): + checkdir(os.path.dirname(self._auth_token_path), create=True) + with atomicwrites.atomic_write(self._auth_token_path) as f: + f.write(token) + assert_permissions(self._auth_token_path, 0o600) + + def _get_key(self): + try: + with open(self._key_path, 'rb') as f: + return f.read() + except (OSError, IOError): + pass + + def _set_key(self, content): + checkdir(os.path.dirname(self._key_path), create=True) + with atomicwrites.atomic_write(self._key_path, mode='wb') as f: + f.write(content) + assert_permissions(self._key_path, 0o600) + + +class EtesyncStorage(Storage): + ''' + :param email: The email address of your account. + :param secrets_dir: A directory where vdirsyncer can store the encryption + key and authentication token. + :param server_url: Optional. URL to the root of your custom server. + :param db_path: Optional. Use a different path for the database. + ''' + + _collection_type = None + _item_type = None + _at_once = False + + def __init__(self, email, secrets_dir, server_url=None, db_path=None, + **kwargs): + if kwargs.get('collection', None) is None: + raise ValueError('Collection argument required') + + self._session = _Session(email, secrets_dir, server_url, db_path) + super(EtesyncStorage, self).__init__(**kwargs) + self._journal = self._session.etesync.get(self.collection) + + def _sync_journal(self): + self._session.etesync.sync_journal(self.collection) + + @classmethod + def discover(cls, email, secrets_dir, server_url=None, db_path=None, + **kwargs): + assert cls._collection_type + if kwargs.get('collection', None) is not None: + raise TypeError('collection argument must not be given.') + session = _Session(email, secrets_dir, server_url, db_path) + session.etesync.sync_journal_list() + for entry in session.etesync.list(): + if isinstance(entry.collection, cls._collection_type): + yield dict( + email=email, + secrets_dir=secrets_dir, + db_path=db_path, + collection=entry.uid, + **kwargs + ) + else: + logger.debug('Skipping collection: {!r}'.format(entry)) + + @classmethod + def create_collection(cls, collection, email, secrets_dir, server_url=None, + db_path=None, **kwargs): + session = _Session(email, secrets_dir, server_url, db_path) + content = {'displayName': collection} + c = cls._collection_type.create( + session.etesync, + binascii.hexlify(os.urandom(32)).decode(), + content + ) + c.save() + session.etesync.sync_journal_list() + return dict( + collection=c.journal.uid, + email=email, + secrets_dir=secrets_dir, + db_path=db_path, + server_url=server_url, + **kwargs + ) + + def list(self): + self._sync_journal() + for entry in self._journal.collection.list(): + item = Item(entry.content) + yield str(entry.uid), item.hash + + def get(self, href): + try: + item = Item(self._journal.collection.get(href).content) + except etesync.exceptions.DoesNotExist as e: + raise exceptions.NotFoundError(e) + return item, item.hash + + @_writing_op + def upload(self, item): + try: + entry = self._item_type.create(self._journal.collection, item.raw) + entry.save() + except etesync.exceptions.DoesNotExist as e: + raise exceptions.NotFoundError(e) + except etesync.exceptions.AlreadyExists as e: + raise exceptions.AlreadyExistingError(e) + return item.uid, item.hash + + @_writing_op + def update(self, href, item, etag): + try: + entry = self._journal.collection.get(href) + except etesync.exceptions.DoesNotExist as e: + raise exceptions.NotFoundError(e) + old_item = Item(entry.content) + if old_item.hash != etag: + raise exceptions.WrongEtagError(etag, old_item.hash) + entry.content = item.raw + entry.save() + return item.hash + + @_writing_op + def delete(self, href, etag): + try: + entry = self._journal.collection.get(href) + old_item = Item(entry.content) + if old_item.hash != etag: + raise exceptions.WrongEtagError(etag, old_item.hash) + entry.delete() + except etesync.exceptions.DoesNotExist as e: + raise exceptions.NotFoundError(e) + + @contextlib.contextmanager + def at_once(self): + self._sync_journal() + self._at_once = True + try: + yield self + self._sync_journal() + finally: + self._at_once = False + + +class EtesyncContacts(EtesyncStorage): + __doc__ = ''' + Contacts for EteSync. + ''' + EtesyncStorage.__doc__ + + _collection_type = AddressBook + _item_type = Contact + storage_name = 'etesync_contacts' + + +class EtesyncCalendars(EtesyncStorage): + __doc__ = ''' + Calendars for EteSync. + ''' + EtesyncStorage.__doc__ + + _collection_type = Calendar + _item_type = Event + storage_name = 'etesync_calendars' diff --git a/vdirsyncer/storage/memory.py b/vdirsyncer/storage/memory.py index 0d7b515..dd88916 100644 --- a/vdirsyncer/storage/memory.py +++ b/vdirsyncer/storage/memory.py @@ -13,6 +13,8 @@ def _random_string(): class MemoryStorage(Storage): + storage_name = 'memory' + ''' Saves data in RAM, only useful for testing. ''' diff --git a/vdirsyncer/vobject.py b/vdirsyncer/vobject.py index ff960f0..d97a057 100644 --- a/vdirsyncer/vobject.py +++ b/vdirsyncer/vobject.py @@ -40,7 +40,7 @@ class Item(object): VCARD''' def __init__(self, raw): - assert isinstance(raw, str) + assert isinstance(raw, str), type(raw) self._raw = raw def with_uid(self, new_uid):