Drop etesync

The current [experimental] implementation is stale, and hasn't been
maintained for a long time. Regrettably, not even its tests still work,
and there nobody interested in maintaining it.

If anyone is interested in re-implementing this in a third-party
package, I can consider adding support for pluggable storages.

CHANGELOG.rst

@@ -29,6 +29,13 @@ Version 0.19.0
   you're validating certificate fingerprints, use `sha256` instead.
 - The ``google`` storage types no longer require ``requests-oauthlib``, but
   require ``python-aiohttp-oauthlib`` instead.
+- Vdirsyncer no longer includes experimental support for `EteSync
+  <https://www.etesync.com/>`_. The existing integration had not been supported
+  for a long time and no longer worked. Support for external storages may be
+  added if anyone is interested in maintaining an EteSync plugin. EteSync
+  users should consider using `etesync-dav`_.
+
+.. _etesync-dav: https://github.com/etesync/etesync-dav
 
 Version 0.18.0
 ==============

MANIFEST.in

@@ -2,7 +2,6 @@
 prune docker
 prune scripts
 prune tests/storage/servers
-prune tests/storage/etesync
 recursive-include tests/storage/servers/radicale *
 recursive-include tests/storage/servers/skip *
 

Makefile

@@ -12,9 +12,6 @@ export REQUIREMENTS := release
 # Set this to true if you run vdirsyncer's test as part of e.g. packaging.
 export DETERMINISTIC_TESTS := false
 
-# Run the etesync testsuite.
-export ETESYNC_TESTS := false
-
 # Assume to run in CI. Don't use this outside of a virtual machine. It will
 # heavily "pollute" your system, such as attempting to install a new Python
 # systemwide.
@@ -30,11 +27,6 @@ PYTEST_ARGS =
 
 TEST_EXTRA_PACKAGES =
 
-ifeq ($(ETESYNC_TESTS), true)
-	TEST_EXTRA_PACKAGES += git+https://github.com/etesync/journal-manager@v0.5.2
-	TEST_EXTRA_PACKAGES += django djangorestframework==3.8.2 wsgi_intercept drf-nested-routers
-endif
-
 PYTEST = py.test $(PYTEST_ARGS)
 CODECOV_PATH = /tmp/codecov.sh
 
@@ -45,7 +37,6 @@ ci-test:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
 	$(PYTEST) --cov vdirsyncer --cov-append tests/unit/ tests/system/
 	bash $(CODECOV_PATH) -c
-	[ "$(ETESYNC_TESTS)" = "false" ] || make test-storage
 
 ci-test-storage:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
@@ -83,7 +74,6 @@ install-dev:
 	pip install -e .
 	pip install -Ur test-requirements.txt $(TEST_EXTRA_PACKAGES)
 	pip install pre-commit
-	[ "$(ETESYNC_TESTS)" = "false" ] || pip install -Ue .[etesync]
 	set -xe && if [ "$(REQUIREMENTS)" = "minimal" ]; then \
 		pip install -U --force-reinstall $$(python setup.py --quiet minimal_requirements); \
 	fi

docs/config.rst

@@ -349,56 +349,6 @@ or write anything to it.
    :param client_id/client_secret: OAuth credentials, obtained from the Google
                                    API Manager.
 
-EteSync
-+++++++
-
-`EteSync <https://www.etesync.com/>`_ is a new cloud provider for end to end
-encrypted contacts and calendar storage. Vdirsyncer contains **experimental**
-support for it.
-
-To use it, you need to install some optional dependencies::
-
-    pip install vdirsyncer[etesync]
-
-On first usage you will be prompted for the service password and the encryption
-password. Neither are stored.
-
-.. storage:: etesync_contacts
-
-    Contacts for etesync.
-
-    ::
-
-        [storage example_for_etesync_contacts]
-        email = ...
-        secrets_dir = ...
-        #server_path = ...
-        #db_path = ...
-
-   :param email: The email address of your account.
-   :param secrets_dir: A directory where vdirsyncer can store the encryption
-                       key and authentication token.
-   :param server_url: Optional. URL to the root of your custom server.
-   :param db_path: Optional. Use a different path for the database.
-
-.. storage:: etesync_calendars
-
-    Calendars for etesync.
-
-    ::
-
-        [storage example_for_etesync_calendars]
-        email = ...
-        secrets_dir = ...
-        #server_path = ...
-        #db_path = ...
-
-    :param email: The email address of your account.
-    :param secrets_dir: A directory where vdirsyncer can store the encryption
-                        key and authentication token.
-    :param server_url: Optional. URL to the root of your custom server.
-    :param db_path: Optional. Use a different path for the database.
-
 Local
 +++++
 

setup.py

@@ -56,7 +56,6 @@ setup(
     # Optional dependencies
     extras_require={
         "google": ["aiohttp-oauthlib"],
-        "etesync": ["etesync==0.5.2", "django<2.0"],
     },
     # Build dependencies
     setup_requires=["setuptools_scm != 1.12.0"],

tests/storage/__init__.py

@@ -254,9 +254,6 @@ class StorageTests:
 
     @pytest.mark.asyncio
     async def test_collection_arg(self, get_storage_args):
-        if self.storage_class.storage_name.startswith("etesync"):
-            pytest.skip("etesync uses UUIDs.")
-
         if self.supports_collections:
             s = self.storage_class(**await get_storage_args(collection="test2"))
             # Can't do stronger assertion because of radicale, which needs a
@@ -302,10 +299,6 @@ class StorageTests:
         ((_, etag3),) = await aiostream.stream.list(s.list())
         assert etag2 == etag3
 
-        # etesync uses UUIDs for collection names
-        if self.storage_class.storage_name.startswith("etesync"):
-            return
-
         assert collection in urlunquote(s.collection)
         if self.storage_class.storage_name.endswith("dav"):
             assert urlquote(uid, "/@:") in href

tests/storage/etesync/etesync_server/etesync_server/settings.py

@@ -1,122 +0,0 @@
-"""
-Django settings for etesync_server project.
-
-Generated by 'django-admin startproject' using Django 1.10.6.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/1.10/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/1.10/ref/settings/
-"""
-import os
-
-# Build paths inside the project like this: os.path.join(BASE_DIR, ...)
-BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
-
-# Quick-start development settings - unsuitable for production
-# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/
-
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = "d7r(p-9=$3a@bbt%*+$p@4)cej13nzd0gmnt8+m0bitb=-umj#"
-
-# SECURITY WARNING: don't run with debug turned on in production!
-DEBUG = True
-
-ALLOWED_HOSTS = []
-
-
-# Application definition
-
-INSTALLED_APPS = [
-    "django.contrib.admin",
-    "django.contrib.auth",
-    "django.contrib.contenttypes",
-    "django.contrib.sessions",
-    "django.contrib.messages",
-    "django.contrib.staticfiles",
-    "rest_framework",
-    "rest_framework.authtoken",
-    "journal.apps.JournalConfig",
-]
-
-MIDDLEWARE = [
-    "django.middleware.security.SecurityMiddleware",
-    "django.contrib.sessions.middleware.SessionMiddleware",
-    "django.middleware.common.CommonMiddleware",
-    "django.middleware.csrf.CsrfViewMiddleware",
-    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    "django.contrib.messages.middleware.MessageMiddleware",
-    "django.middleware.clickjacking.XFrameOptionsMiddleware",
-]
-
-ROOT_URLCONF = "etesync_server.urls"
-
-TEMPLATES = [
-    {
-        "BACKEND": "django.template.backends.django.DjangoTemplates",
-        "DIRS": [],
-        "APP_DIRS": True,
-        "OPTIONS": {
-            "context_processors": [
-                "django.template.context_processors.debug",
-                "django.template.context_processors.request",
-                "django.contrib.auth.context_processors.auth",
-                "django.contrib.messages.context_processors.messages",
-            ],
-        },
-    },
-]
-
-WSGI_APPLICATION = "etesync_server.wsgi.application"
-
-
-# Database
-# https://docs.djangoproject.com/en/1.10/ref/settings/#databases
-
-DATABASES = {
-    "default": {
-        "ENGINE": "django.db.backends.sqlite3",
-        "NAME": os.environ.get("ETESYNC_DB_PATH", os.path.join(BASE_DIR, "db.sqlite3")),
-    }
-}
-
-
-# Password validation
-# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators
-
-AUTH_PASSWORD_VALIDATORS = [
-    {
-        "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator",  # noqa
-    },
-    {
-        "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator",  # noqa
-    },
-    {
-        "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator",  # noqa
-    },
-    {
-        "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator",  # noqa
-    },
-]
-
-
-# Internationalization
-# https://docs.djangoproject.com/en/1.10/topics/i18n/
-
-LANGUAGE_CODE = "en-us"
-
-TIME_ZONE = "UTC"
-
-USE_I18N = True
-
-USE_L10N = True
-
-USE_TZ = True
-
-
-# Static files (CSS, JavaScript, Images)
-# https://docs.djangoproject.com/en/1.10/howto/static-files/
-
-STATIC_URL = "/static/"

tests/storage/etesync/etesync_server/etesync_server/urls.py

@@ -1,37 +0,0 @@
-"""etesync_server URL Configuration
-
-The `urlpatterns` list routes URLs to views. For more information please see:
-    https://docs.djangoproject.com/en/1.10/topics/http/urls/
-Examples:
-Function views
-    1. Add an import:  from my_app import views
-    2. Add a URL to urlpatterns:  url(r'^$', views.home, name='home')
-Class-based views
-    1. Add an import:  from other_app.views import Home
-    2. Add a URL to urlpatterns:  url(r'^$', Home.as_view(), name='home')
-Including another URLconf
-    1. Import the include() function: from django.conf.urls import url, include
-    2. Add a URL to urlpatterns:  url(r'^blog/', include('blog.urls'))
-"""
-from django.conf.urls import include
-from django.conf.urls import url
-from journal import views
-from rest_framework_nested import routers
-
-router = routers.DefaultRouter()
-router.register(r"journals", views.JournalViewSet)
-router.register(r"journal/(?P<journal_uid>[^/]+)", views.EntryViewSet)
-router.register(r"user", views.UserInfoViewSet)
-
-journals_router = routers.NestedSimpleRouter(router, r"journals", lookup="journal")
-journals_router.register(r"members", views.MembersViewSet, base_name="journal-members")
-journals_router.register(r"entries", views.EntryViewSet, base_name="journal-entries")
-
-
-urlpatterns = [
-    url(r"^api/v1/", include(router.urls)),
-    url(r"^api/v1/", include(journals_router.urls)),
-]
-
-# Adding this just for testing, this shouldn't be here normally
-urlpatterns += (url(r"^reset/$", views.reset, name="reset_debug"),)

tests/storage/etesync/etesync_server/etesync_server/wsgi.py

@@ -1,15 +0,0 @@
-"""
-WSGI config for etesync_server project.
-
-It exposes the WSGI callable as a module-level variable named ``application``.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/
-"""
-import os
-
-from django.core.wsgi import get_wsgi_application
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings")
-
-application = get_wsgi_application()

tests/storage/etesync/etesync_server/manage.py

@@ -1,22 +0,0 @@
-#!/usr/bin/env python
-import os
-import sys
-
-if __name__ == "__main__":
-    os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings")
-    try:
-        from django.core.management import execute_from_command_line
-    except ImportError:
-        # The above import may fail for some other reason. Ensure that the
-        # issue is really that Django is missing to avoid masking other
-        # exceptions on Python 2.
-        try:
-            import django  # noqa
-        except ImportError:
-            raise ImportError(
-                "Couldn't import Django. Are you sure it's installed and "
-                "available on your PYTHONPATH environment variable? Did you "
-                "forget to activate a virtual environment?"
-            )
-        raise
-    execute_from_command_line(sys.argv)

tests/storage/etesync/test@localhost/auth_token

@@ -1 +0,0 @@
-63ae6eec45b592d5c511f79b7b0c312d2c5f7d6a

tests/storage/etesync/test_main.py

@@ -1,88 +0,0 @@
-import os
-import shutil
-import sys
-
-import pytest
-import requests
-
-from vdirsyncer.storage.etesync import EtesyncCalendars
-from vdirsyncer.storage.etesync import EtesyncContacts
-
-from .. import StorageTests
-
-pytestmark = pytest.mark.skipif(
-    os.getenv("ETESYNC_TESTS", "") != "true", reason="etesync tests disabled"
-)
-
-
-@pytest.fixture(scope="session")
-def etesync_app(tmpdir_factory):
-    sys.path.insert(0, os.path.join(os.path.dirname(__file__), "etesync_server"))
-
-    db = tmpdir_factory.mktemp("etesync").join("etesync.sqlite")
-    shutil.copy(
-        os.path.join(os.path.dirname(__file__), "etesync_server", "db.sqlite3"), str(db)
-    )
-
-    os.environ["ETESYNC_DB_PATH"] = str(db)
-    from etesync_server.wsgi import application
-
-    return application
-
-
-class EtesyncTests(StorageTests):
-
-    supports_metadata = False
-
-    @pytest.fixture
-    def get_storage_args(self, request, get_item, tmpdir, etesync_app):
-        import wsgi_intercept
-        import wsgi_intercept.requests_intercept
-
-        wsgi_intercept.requests_intercept.install()
-        wsgi_intercept.add_wsgi_intercept("127.0.0.1", 8000, lambda: etesync_app)
-
-        def teardown():
-            wsgi_intercept.remove_wsgi_intercept("127.0.0.1", 8000)
-            wsgi_intercept.requests_intercept.uninstall()
-
-        request.addfinalizer(teardown)
-
-        with open(
-            os.path.join(os.path.dirname(__file__), "test@localhost/auth_token")
-        ) as f:
-            token = f.read().strip()
-        headers = {"Authorization": "Token " + token}
-        r = requests.post(
-            "http://127.0.0.1:8000/reset/", headers=headers, allow_redirects=False
-        )
-        assert r.status_code == 200
-
-        async def inner(collection="test"):
-            rv = {
-                "email": "test@localhost",
-                "db_path": str(tmpdir.join("etesync.db")),
-                "secrets_dir": os.path.dirname(__file__),
-                "server_url": "http://127.0.0.1:8000/",
-            }
-            if collection is not None:
-                rv = self.storage_class.create_collection(collection=collection, **rv)
-            return rv
-
-        return inner
-
-
-class TestContacts(EtesyncTests):
-    storage_class = EtesyncContacts
-
-    @pytest.fixture(params=["VCARD"])
-    def item_type(self, request):
-        return request.param
-
-
-class TestCalendars(EtesyncTests):
-    storage_class = EtesyncCalendars
-
-    @pytest.fixture(params=["VEVENT"])
-    def item_type(self, request):
-        return request.param

vdirsyncer/cli/utils.py

@@ -35,8 +35,6 @@ class _StorageIndex:
             singlefile="vdirsyncer.storage.singlefile.SingleFileStorage",
             google_calendar="vdirsyncer.storage.google.GoogleCalendarStorage",
             google_contacts="vdirsyncer.storage.google.GoogleContactsStorage",
-            etesync_calendars="vdirsyncer.storage.etesync.EtesyncCalendars",
-            etesync_contacts="vdirsyncer.storage.etesync.EtesyncContacts",
         )
 
     def __getitem__(self, name):

vdirsyncer/storage/etesync.py

@@ -1,237 +0,0 @@
-import binascii
-import contextlib
-import functools
-import logging
-import os
-
-import atomicwrites
-import click
-
-try:
-    import etesync
-    import etesync.exceptions
-    from etesync import AddressBook
-    from etesync import Calendar
-    from etesync import Contact
-    from etesync import Event
-
-    has_etesync = True
-except ImportError:
-    has_etesync = False
-    AddressBook = Contact = Calendar = Event = None
-
-from .. import exceptions
-from ..cli.utils import assert_permissions
-from ..utils import checkdir
-from ..vobject import Item
-from .base import Storage
-
-logger = logging.getLogger(__name__)
-
-
-def _writing_op(f):
-    @functools.wraps(f)
-    async def inner(self, *args, **kwargs):
-        if not self._at_once:
-            self._sync_journal()
-        rv = f(self, *args, **kwargs)
-        if not self._at_once:
-            self._sync_journal()
-        return await rv
-
-    return inner
-
-
-class _Session:
-    def __init__(self, email, secrets_dir, server_url=None, db_path=None):
-        if not has_etesync:
-            raise exceptions.UserError("Dependencies for etesync are not " "installed.")
-        server_url = server_url or etesync.API_URL
-        self.email = email
-        self.secrets_dir = os.path.join(secrets_dir, email + "/")
-
-        self._auth_token_path = os.path.join(self.secrets_dir, "auth_token")
-        self._key_path = os.path.join(self.secrets_dir, "key")
-
-        auth_token = self._get_auth_token()
-        if not auth_token:
-            password = click.prompt(
-                f"Enter service password for {self.email}", hide_input=True
-            )
-            auth_token = etesync.Authenticator(server_url).get_auth_token(
-                self.email, password
-            )
-            self._set_auth_token(auth_token)
-
-        self._db_path = db_path or os.path.join(self.secrets_dir, "db.sqlite")
-        self.etesync = etesync.EteSync(
-            email, auth_token, remote=server_url, db_path=self._db_path
-        )
-
-        key = self._get_key()
-        if not key:
-            password = click.prompt("Enter key password", hide_input=True)
-            click.echo(f"Deriving key for {self.email}")
-            self.etesync.derive_key(password)
-            self._set_key(self.etesync.cipher_key)
-        else:
-            self.etesync.cipher_key = key
-
-    def _get_auth_token(self):
-        try:
-            with open(self._auth_token_path) as f:
-                return f.read().strip() or None
-        except OSError:
-            pass
-
-    def _set_auth_token(self, token):
-        checkdir(os.path.dirname(self._auth_token_path), create=True)
-        with atomicwrites.atomic_write(self._auth_token_path) as f:
-            f.write(token)
-        assert_permissions(self._auth_token_path, 0o600)
-
-    def _get_key(self):
-        try:
-            with open(self._key_path, "rb") as f:
-                return f.read()
-        except OSError:
-            pass
-
-    def _set_key(self, content):
-        checkdir(os.path.dirname(self._key_path), create=True)
-        with atomicwrites.atomic_write(self._key_path, mode="wb") as f:
-            f.write(content)
-        assert_permissions(self._key_path, 0o600)
-
-
-class EtesyncStorage(Storage):
-    _collection_type = None
-    _item_type = None
-    _at_once = False
-
-    def __init__(self, email, secrets_dir, server_url=None, db_path=None, **kwargs):
-        if kwargs.get("collection", None) is None:
-            raise ValueError("Collection argument required")
-
-        self._session = _Session(email, secrets_dir, server_url, db_path)
-        super().__init__(**kwargs)
-        self._journal = self._session.etesync.get(self.collection)
-
-    def _sync_journal(self):
-        self._session.etesync.sync_journal(self.collection)
-
-    @classmethod
-    async def discover(
-        cls,
-        email,
-        secrets_dir,
-        server_url=None,
-        db_path=None,
-        **kwargs,
-    ):
-        if kwargs.get("collection", None) is not None:
-            raise TypeError("collection argument must not be given.")
-        session = _Session(email, secrets_dir, server_url, db_path)
-        assert cls._collection_type
-        session.etesync.sync_journal_list()
-        for entry in session.etesync.list():
-            if isinstance(entry.collection, cls._collection_type):
-                yield dict(
-                    email=email,
-                    secrets_dir=secrets_dir,
-                    db_path=db_path,
-                    collection=entry.uid,
-                    **kwargs,
-                )
-            else:
-                logger.debug(f"Skipping collection: {entry!r}")
-
-    @classmethod
-    async def create_collection(
-        cls, collection, email, secrets_dir, server_url=None, db_path=None, **kwargs
-    ):
-        session = _Session(email, secrets_dir, server_url, db_path)
-        content = {"displayName": collection}
-        c = cls._collection_type.create(
-            session.etesync, binascii.hexlify(os.urandom(32)).decode(), content
-        )
-        c.save()
-        session.etesync.sync_journal_list()
-        return dict(
-            collection=c.journal.uid,
-            email=email,
-            secrets_dir=secrets_dir,
-            db_path=db_path,
-            server_url=server_url,
-            **kwargs,
-        )
-
-    async def list(self):
-        self._sync_journal()
-        for entry in self._journal.collection.list():
-            item = Item(entry.content)
-            yield str(entry.uid), item.hash
-
-    async def get(self, href):
-        try:
-            item = Item(self._journal.collection.get(href).content)
-        except etesync.exceptions.DoesNotExist as e:
-            raise exceptions.NotFoundError(e)
-        return item, item.hash
-
-    @_writing_op
-    async def upload(self, item):
-        try:
-            entry = self._item_type.create(self._journal.collection, item.raw)
-            entry.save()
-        except etesync.exceptions.DoesNotExist as e:
-            raise exceptions.NotFoundError(e)
-        except etesync.exceptions.AlreadyExists as e:
-            raise exceptions.AlreadyExistingError(e)
-        return item.uid, item.hash
-
-    @_writing_op
-    async def update(self, href, item, etag):
-        try:
-            entry = self._journal.collection.get(href)
-        except etesync.exceptions.DoesNotExist as e:
-            raise exceptions.NotFoundError(e)
-        old_item = Item(entry.content)
-        if old_item.hash != etag:
-            raise exceptions.WrongEtagError(etag, old_item.hash)
-        entry.content = item.raw
-        entry.save()
-        return item.hash
-
-    @_writing_op
-    async def delete(self, href, etag):
-        try:
-            entry = self._journal.collection.get(href)
-            old_item = Item(entry.content)
-            if old_item.hash != etag:
-                raise exceptions.WrongEtagError(etag, old_item.hash)
-            entry.delete()
-        except etesync.exceptions.DoesNotExist as e:
-            raise exceptions.NotFoundError(e)
-
-    @contextlib.asynccontextmanager
-    async def at_once(self):
-        self._sync_journal()
-        self._at_once = True
-        try:
-            yield self
-            self._sync_journal()
-        finally:
-            self._at_once = False
-
-
-class EtesyncContacts(EtesyncStorage):
-    _collection_type = AddressBook
-    _item_type = Contact
-    storage_name = "etesync_contacts"
-
-
-class EtesyncCalendars(EtesyncStorage):
-    _collection_type = Calendar
-    _item_type = Event
-    storage_name = "etesync_calendars"