Initial etesync support (#614)

* Initial etesync support

* Optimized get()

* Bugfixes

* bugfixes

* Add writing operations

* collection creation WIP

* Fix collection creation

* tests wip

* wip

* Final touch for test setup

* actually skip tests

* Disable metadata tests

* Avoid polluting working tree

* Avoid importing wsgi-intercept if not installed

* Fix collection tests

* Proper teardown

* Skip bad test

* Remove vtodo

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* wip

* Add docs

* Remove useless pkg

* Don't crash if etesync isn't imported

* Stylefix

* Fix more import errors

* More import errors

* Only run etesync on latest python

* Fix etesync check

* journal-manager migration

.travis.yml

@@ -123,6 +123,10 @@
         "env": "BUILD=test DAV_SERVER=radicale REQUIREMENTS=minimal BUILD_PRS=true ",
         "python": "pypy3"
       },
+      {
+        "env": "BUILD=test ETESYNC_TESTS=true REQUIREMENTS=latest BUILD_PRS=true ",
+        "python": "3.6"
+      },
       {
         "env": "BUILD=test BUILD_PRS=true",
         "language": "generic",

Makefile

@@ -8,6 +8,7 @@ export TESTSERVER_BASE := ./tests/storage/servers/
 export CI := false
 export COVERAGE := $(CI)
 export DETERMINISTIC_TESTS := false
+export ETESYNC_TESTS := false
 
 CODECOV_PATH = /tmp/codecov.sh
 
@@ -18,6 +19,11 @@ ifeq ($(COVERAGE), true)
 	PYTEST_ARGS += --cov-config .coveragerc --cov vdirsyncer
 endif
 
+ifeq ($(ETESYNC_TESTS), true)
+	TEST_EXTRA_PACKAGES += git+https://github.com/etesync/journal-manager
+	TEST_EXTRA_PACKAGES += django djangorestframework wsgi_intercept
+endif
+
 ifeq ($(CI), true)
 test:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
@@ -83,11 +89,9 @@ release:
 	python setup.py sdist bdist_wheel upload
 
 install-dev:
-	set -xe && if [ "$(REMOTESTORAGE_SERVER)" != "skip" ]; then \
-		pip install -e .[remotestorage]; \
-	else \
-		pip install -e .; \
-	fi
+	pip install -e .
+	[ "$(REMOTESTORAGE_SERVER)" = "skip" ] || pip install -e .[remotestorage]
+	[ "$(ETESYNC_TESTS)" = "false" ] || pip install -e .[etesync]
 	set -xe && if [ "$(REQUIREMENTS)" = "devel" ]; then \
 	    pip install -U --force-reinstall \
 			git+https://github.com/mitsuhiko/click \

docs/config.rst

@@ -226,6 +226,24 @@ To use them, you need to install some optional dependencies with::
 
 .. autostorage:: vdirsyncer.storage.remotestorage.RemoteStorageCalendars
 
+EteSync
++++++++
+
+`EteSync <https://www.etesync.com/>`_ is a new cloud provider for end to end
+encrypted contacts and calendar storage. Vdirsyncer contains **experimental**
+support for it.
+
+To use it, you need to install some optional dependencies::
+
+    pip install vdirsyncer[etesync]
+
+On first usage you will be prompted for the service password and the encryption
+password. Neither are stored.
+
+.. autostorage:: vdirsyncer.storage.etesync.EtesyncContacts
+
+.. autostorage:: vdirsyncer.storage.etesync.EtesyncCalendars
+
 Local
 +++++
 

scripts/make_travisconf.py

@@ -71,6 +71,14 @@ for python, requirements in itertools.product(python_versions,
                             build_prs=build_prs and "true" or "false"))
         })
 
+matrix.append({
+    'python': latest_python,
+    'env': ("BUILD=test "
+            "ETESYNC_TESTS=true "
+            "REQUIREMENTS=latest "
+            "BUILD_PRS=true ")
+})
+
 matrix.append({
     'language': 'generic',
     'os': 'osx',

setup.py

@@ -72,6 +72,7 @@ setup(
     extras_require={
         'remotestorage': ['requests-oauthlib'],
         'google': ['requests-oauthlib'],
+        'etesync': ['etesync']
     },
 
     # Build dependencies

tests/storage/__init__.py

@@ -226,6 +226,9 @@ class StorageTests(object):
         assert 'collection argument must not be given' in str(excinfo.value)
 
     def test_collection_arg(self, get_storage_args):
+        if self.storage_class.storage_name.startswith('etesync'):
+            pytest.skip('etesync uses UUIDs.')
+
         if self.supports_collections:
             s = self.storage_class(**get_storage_args(collection='test2'))
             # Can't do stronger assertion because of radicale, which needs a
@@ -270,6 +273,10 @@ class StorageTests(object):
         (_, etag3), = s.list()
         assert etag2 == etag3
 
+        # etesync uses UUIDs for collection names
+        if self.storage_class.storage_name.startswith('etesync'):
+            return
+
         assert collection in urlunquote(s.collection)
         if self.storage_class.storage_name.endswith('dav'):
             assert urlquote(uid, '/@:') in href

tests/storage/etesync/etesync_server/etesync_server/settings.py

@@ -0,0 +1,124 @@
+"""
+Django settings for etesync_server project.
+
+Generated by 'django-admin startproject' using Django 1.10.6.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.10/topics/settings/
+
+For the full list of settings and their values, see
+https://docs.djangoproject.com/en/1.10/ref/settings/
+"""
+
+import os
+
+# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
+BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+
+# Quick-start development settings - unsuitable for production
+# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
+
+# SECURITY WARNING: keep the secret key used in production secret!
+SECRET_KEY = 'd7r(p-9=$3a@bbt%*+$p@4)cej13nzd0gmnt8+m0bitb=-umj#'
+
+# SECURITY WARNING: don't run with debug turned on in production!
+DEBUG = True
+
+ALLOWED_HOSTS = []
+
+
+# Application definition
+
+INSTALLED_APPS = [
+    'django.contrib.admin',
+    'django.contrib.auth',
+    'django.contrib.contenttypes',
+    'django.contrib.sessions',
+    'django.contrib.messages',
+    'django.contrib.staticfiles',
+    'rest_framework',
+    'rest_framework.authtoken',
+    'journal.apps.JournalConfig',
+]
+
+MIDDLEWARE = [
+    'django.middleware.security.SecurityMiddleware',
+    'django.contrib.sessions.middleware.SessionMiddleware',
+    'django.middleware.common.CommonMiddleware',
+    'django.middleware.csrf.CsrfViewMiddleware',
+    'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'django.contrib.messages.middleware.MessageMiddleware',
+    'django.middleware.clickjacking.XFrameOptionsMiddleware',
+]
+
+ROOT_URLCONF = 'etesync_server.urls'
+
+TEMPLATES = [
+    {
+        'BACKEND': 'django.template.backends.django.DjangoTemplates',
+        'DIRS': [],
+        'APP_DIRS': True,
+        'OPTIONS': {
+            'context_processors': [
+                'django.template.context_processors.debug',
+                'django.template.context_processors.request',
+                'django.contrib.auth.context_processors.auth',
+                'django.contrib.messages.context_processors.messages',
+            ],
+        },
+    },
+]
+
+WSGI_APPLICATION = 'etesync_server.wsgi.application'
+
+
+# Database
+# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.sqlite3',
+        'NAME': os.environ.get('ETESYNC_DB_PATH',
+                               os.path.join(BASE_DIR, 'db.sqlite3')),
+    }
+}
+
+
+# Password validation
+# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
+
+AUTH_PASSWORD_VALIDATORS = [
+    {
+        'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator',  # noqa
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator',  # noqa
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator',  # noqa
+    },
+    {
+        'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator',  # noqa
+    },
+]
+
+
+# Internationalization
+# https://docs.djangoproject.com/en/1.10/topics/i18n/
+
+LANGUAGE_CODE = 'en-us'
+
+TIME_ZONE = 'UTC'
+
+USE_I18N = True
+
+USE_L10N = True
+
+USE_TZ = True
+
+
+# Static files (CSS, JavaScript, Images)
+# https://docs.djangoproject.com/en/1.10/howto/static-files/
+
+STATIC_URL = '/static/'

tests/storage/etesync/etesync_server/etesync_server/urls.py

@@ -0,0 +1,38 @@
+"""etesync_server URL Configuration
+
+The `urlpatterns` list routes URLs to views. For more information please see:
+    https://docs.djangoproject.com/en/1.10/topics/http/urls/
+Examples:
+Function views
+    1. Add an import:  from my_app import views
+    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
+Class-based views
+    1. Add an import:  from other_app.views import Home
+    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
+Including another URLconf
+    1. Import the include() function: from django.conf.urls import url, include
+    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
+"""
+from django.conf import settings
+from django.conf.urls import include, url
+from django.contrib import admin
+
+from rest_framework import routers
+from rest_framework.authtoken import views as token_views
+
+from journal import views
+
+router = routers.DefaultRouter()
+router.register(r'journals', views.JournalViewSet)
+router.register(r'journal/(?P<journal>[^/]+)', views.EntryViewSet)
+
+urlpatterns = [
+    url(r'^api/v1/', include(router.urls)),
+    url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')),  # noqa
+    url(r'^api-token-auth/', token_views.obtain_auth_token),
+]
+
+if settings.DEBUG:
+    urlpatterns += url(r'^reset/$', views.reset, name='reset_debug'),
+
+urlpatterns += url(r'^admin/', admin.site.urls),

tests/storage/etesync/etesync_server/etesync_server/wsgi.py

@@ -0,0 +1,16 @@
+"""
+WSGI config for etesync_server project.
+
+It exposes the WSGI callable as a module-level variable named ``application``.
+
+For more information on this file, see
+https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/
+"""
+
+import os
+
+from django.core.wsgi import get_wsgi_application
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings")
+
+application = get_wsgi_application()

tests/storage/etesync/etesync_server/manage.py

@@ -0,0 +1,22 @@
+#!/usr/bin/env python
+import os
+import sys
+
+if __name__ == "__main__":
+    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings")
+    try:
+        from django.core.management import execute_from_command_line
+    except ImportError:
+        # The above import may fail for some other reason. Ensure that the
+        # issue is really that Django is missing to avoid masking other
+        # exceptions on Python 2.
+        try:
+            import django  # noqa
+        except ImportError:
+            raise ImportError(
+                "Couldn't import Django. Are you sure it's installed and "
+                "available on your PYTHONPATH environment variable? Did you "
+                "forget to activate a virtual environment?"
+            )
+        raise
+    execute_from_command_line(sys.argv)

tests/storage/etesync/test@localhost/auth_token

@@ -0,0 +1 @@
+63ae6eec45b592d5c511f79b7b0c312d2c5f7d6a

tests/storage/etesync/test_main.py

@@ -0,0 +1,92 @@
+# -*- coding: utf-8 -*-
+
+import shutil
+import os
+import sys
+
+
+import pytest
+import requests
+
+from vdirsyncer.storage.etesync import EtesyncContacts, EtesyncCalendars
+
+from .. import StorageTests
+
+
+pytestmark = pytest.mark.skipif(os.getenv('ETESYNC_TESTS', '') != 'true',
+                                reason='etesync tests disabled')
+
+
+@pytest.fixture(scope='session')
+def etesync_app(tmpdir_factory):
+    sys.path.insert(0, os.path.join(os.path.dirname(__file__),
+                                    'etesync_server'))
+
+    db = tmpdir_factory.mktemp('etesync').join('etesync.sqlite')
+    shutil.copy(
+        os.path.join(os.path.dirname(__file__), 'etesync_server',
+                     'db.sqlite3'),
+        str(db)
+    )
+
+    os.environ['ETESYNC_DB_PATH'] = str(db)
+    from etesync_server.wsgi import application
+    return application
+
+
+class EtesyncTests(StorageTests):
+
+    supports_metadata = False
+
+    @pytest.fixture
+    def get_storage_args(self, request, get_item, tmpdir, etesync_app):
+        import wsgi_intercept
+        import wsgi_intercept.requests_intercept
+        wsgi_intercept.requests_intercept.install()
+        wsgi_intercept.add_wsgi_intercept('127.0.0.1', 8000,
+                                          lambda: etesync_app)
+
+        def teardown():
+            wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 8000)
+            wsgi_intercept.requests_intercept.uninstall()
+
+        request.addfinalizer(teardown)
+
+        with open(os.path.join(os.path.dirname(__file__),
+                               'test@localhost/auth_token')) as f:
+            token = f.read().strip()
+        headers = {'Authorization': 'Token ' + token}
+        r = requests.post('http://127.0.0.1:8000/reset/', headers=headers,
+                          allow_redirects=False)
+        assert r.status_code == 200
+
+        def inner(collection='test'):
+            rv = {
+                'email': 'test@localhost',
+                'db_path': str(tmpdir.join('etesync.db')),
+                'secrets_dir': os.path.dirname(__file__),
+                'server_url': 'http://127.0.0.1:8000/'
+            }
+            if collection is not None:
+                rv = self.storage_class.create_collection(
+                    collection=collection,
+                    **rv
+                )
+            return rv
+        return inner
+
+
+class TestContacts(EtesyncTests):
+    storage_class = EtesyncContacts
+
+    @pytest.fixture(params=['VCARD'])
+    def item_type(self, request):
+        return request.param
+
+
+class TestCalendars(EtesyncTests):
+    storage_class = EtesyncCalendars
+
+    @pytest.fixture(params=['VEVENT'])
+    def item_type(self, request):
+        return request.param

tests/storage/test_http_with_singlefile.py

@@ -15,6 +15,7 @@ class CombinedStorage(Storage):
     '''A subclass of HttpStorage to make testing easier. It supports writes via
     SingleFileStorage.'''
     _repr_attributes = ('url', 'path')
+    storage_name = 'http_and_singlefile'
 
     def __init__(self, url, path, **kwargs):
         if kwargs.get('collection', None) is not None:

vdirsyncer/cli/utils.py

@@ -39,7 +39,9 @@ class _StorageIndex(object):
             remotestorage_calendars=(
                 'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'),
             google_calendar='vdirsyncer.storage.google.GoogleCalendarStorage',
-            google_contacts='vdirsyncer.storage.google.GoogleContactsStorage'
+            google_contacts='vdirsyncer.storage.google.GoogleContactsStorage',
+            etesync_calendars='vdirsyncer.storage.etesync.EtesyncCalendars',
+            etesync_contacts='vdirsyncer.storage.etesync.EtesyncContacts'
         )
 
     def __getitem__(self, name):

vdirsyncer/storage/etesync.py

@@ -0,0 +1,244 @@
+import contextlib
+import functools
+import logging
+import os
+import binascii
+
+import atomicwrites
+import click
+
+try:
+    import etesync
+    import etesync.exceptions
+    from etesync import AddressBook, Contact, Calendar, Event
+    has_etesync = True
+except ImportError:
+    has_etesync = False
+    AddressBook = Contact = Calendar = Event = None
+
+from .. import exceptions
+from ..cli.utils import assert_permissions
+from ..utils import checkdir
+from ..vobject import Item
+
+from .base import Storage
+
+
+logger = logging.getLogger(__name__)
+
+
+def _writing_op(f):
+    @functools.wraps(f)
+    def inner(self, *args, **kwargs):
+        if not self._at_once:
+            self._sync_journal()
+        rv = f(self, *args, **kwargs)
+        if not self._at_once:
+            self._sync_journal()
+        return rv
+    return inner
+
+
+class _Session:
+    def __init__(self, email, secrets_dir, server_url=None, db_path=None):
+        if not has_etesync:
+            raise exceptions.UserError('Dependencies for etesync are not '
+                                       'installed.')
+        server_url = server_url or etesync.API_URL
+        self.email = email
+        self.secrets_dir = os.path.join(secrets_dir, email + '/')
+
+        self._auth_token_path = os.path.join(self.secrets_dir, 'auth_token')
+        self._key_path = os.path.join(self.secrets_dir, 'key')
+
+        auth_token = self._get_auth_token()
+        if not auth_token:
+            password = click.prompt('Enter service password for {}'
+                                    .format(self.email), hide_input=True)
+            auth_token = etesync.Authenticator(server_url) \
+                .get_auth_token(self.email, password)
+            self._set_auth_token(auth_token)
+
+        self._db_path = db_path or os.path.join(self.secrets_dir, 'db.sqlite')
+        self.etesync = etesync.EteSync(email, auth_token, remote=server_url,
+                                       db_path=self._db_path)
+
+        key = self._get_key()
+        if not key:
+            password = click.prompt('Enter key password', hide_input=True)
+            click.echo('Deriving key for {}'.format(self.email))
+            self.etesync.derive_key(password)
+            self._set_key(self.etesync.cipher_key)
+        else:
+            self.etesync.cipher_key = key
+
+    def _get_auth_token(self):
+        try:
+            with open(self._auth_token_path) as f:
+                return f.read().strip() or None
+        except (OSError, IOError):
+            pass
+
+    def _set_auth_token(self, token):
+        checkdir(os.path.dirname(self._auth_token_path), create=True)
+        with atomicwrites.atomic_write(self._auth_token_path) as f:
+            f.write(token)
+        assert_permissions(self._auth_token_path, 0o600)
+
+    def _get_key(self):
+        try:
+            with open(self._key_path, 'rb') as f:
+                return f.read()
+        except (OSError, IOError):
+            pass
+
+    def _set_key(self, content):
+        checkdir(os.path.dirname(self._key_path), create=True)
+        with atomicwrites.atomic_write(self._key_path, mode='wb') as f:
+            f.write(content)
+        assert_permissions(self._key_path, 0o600)
+
+
+class EtesyncStorage(Storage):
+    '''
+    :param email: The email address of your account.
+    :param secrets_dir: A directory where vdirsyncer can store the encryption
+        key and authentication token.
+    :param server_url: Optional. URL to the root of your custom server.
+    :param db_path: Optional. Use a different path for the database.
+    '''
+
+    _collection_type = None
+    _item_type = None
+    _at_once = False
+
+    def __init__(self, email, secrets_dir, server_url=None, db_path=None,
+                 **kwargs):
+        if kwargs.get('collection', None) is None:
+            raise ValueError('Collection argument required')
+
+        self._session = _Session(email, secrets_dir, server_url, db_path)
+        super(EtesyncStorage, self).__init__(**kwargs)
+        self._journal = self._session.etesync.get(self.collection)
+
+    def _sync_journal(self):
+        self._session.etesync.sync_journal(self.collection)
+
+    @classmethod
+    def discover(cls, email, secrets_dir, server_url=None, db_path=None,
+                 **kwargs):
+        assert cls._collection_type
+        if kwargs.get('collection', None) is not None:
+            raise TypeError('collection argument must not be given.')
+        session = _Session(email, secrets_dir, server_url, db_path)
+        session.etesync.sync_journal_list()
+        for entry in session.etesync.list():
+            if isinstance(entry.collection, cls._collection_type):
+                yield dict(
+                    email=email,
+                    secrets_dir=secrets_dir,
+                    db_path=db_path,
+                    collection=entry.uid,
+                    **kwargs
+                )
+            else:
+                logger.debug('Skipping collection: {!r}'.format(entry))
+
+    @classmethod
+    def create_collection(cls, collection, email, secrets_dir, server_url=None,
+                          db_path=None, **kwargs):
+        session = _Session(email, secrets_dir, server_url, db_path)
+        content = {'displayName': collection}
+        c = cls._collection_type.create(
+            session.etesync,
+            binascii.hexlify(os.urandom(32)).decode(),
+            content
+        )
+        c.save()
+        session.etesync.sync_journal_list()
+        return dict(
+            collection=c.journal.uid,
+            email=email,
+            secrets_dir=secrets_dir,
+            db_path=db_path,
+            server_url=server_url,
+            **kwargs
+        )
+
+    def list(self):
+        self._sync_journal()
+        for entry in self._journal.collection.list():
+            item = Item(entry.content)
+            yield str(entry.uid), item.hash
+
+    def get(self, href):
+        try:
+            item = Item(self._journal.collection.get(href).content)
+        except etesync.exceptions.DoesNotExist as e:
+            raise exceptions.NotFoundError(e)
+        return item, item.hash
+
+    @_writing_op
+    def upload(self, item):
+        try:
+            entry = self._item_type.create(self._journal.collection, item.raw)
+            entry.save()
+        except etesync.exceptions.DoesNotExist as e:
+            raise exceptions.NotFoundError(e)
+        except etesync.exceptions.AlreadyExists as e:
+            raise exceptions.AlreadyExistingError(e)
+        return item.uid, item.hash
+
+    @_writing_op
+    def update(self, href, item, etag):
+        try:
+            entry = self._journal.collection.get(href)
+        except etesync.exceptions.DoesNotExist as e:
+            raise exceptions.NotFoundError(e)
+        old_item = Item(entry.content)
+        if old_item.hash != etag:
+            raise exceptions.WrongEtagError(etag, old_item.hash)
+        entry.content = item.raw
+        entry.save()
+        return item.hash
+
+    @_writing_op
+    def delete(self, href, etag):
+        try:
+            entry = self._journal.collection.get(href)
+            old_item = Item(entry.content)
+            if old_item.hash != etag:
+                raise exceptions.WrongEtagError(etag, old_item.hash)
+            entry.delete()
+        except etesync.exceptions.DoesNotExist as e:
+            raise exceptions.NotFoundError(e)
+
+    @contextlib.contextmanager
+    def at_once(self):
+        self._sync_journal()
+        self._at_once = True
+        try:
+            yield self
+            self._sync_journal()
+        finally:
+            self._at_once = False
+
+
+class EtesyncContacts(EtesyncStorage):
+    __doc__ = '''
+    Contacts for EteSync.
+    ''' + EtesyncStorage.__doc__
+
+    _collection_type = AddressBook
+    _item_type = Contact
+    storage_name = 'etesync_contacts'
+
+
+class EtesyncCalendars(EtesyncStorage):
+    __doc__ = '''
+    Calendars for EteSync.
+    ''' + EtesyncStorage.__doc__
+
+    _collection_type = Calendar
+    _item_type = Event
+    storage_name = 'etesync_calendars'

vdirsyncer/storage/memory.py

@@ -13,6 +13,8 @@ def _random_string():
 
 class MemoryStorage(Storage):
 
+    storage_name = 'memory'
+
     '''
     Saves data in RAM, only useful for testing.
     '''

vdirsyncer/vobject.py

@@ -40,7 +40,7 @@ class Item(object):
     VCARD'''
 
     def __init__(self, raw):
-        assert isinstance(raw, str)
+        assert isinstance(raw, str), type(raw)
         self._raw = raw
 
     def with_uid(self, new_uid):