diff --git a/CHANGELOG.rst b/CHANGELOG.rst index ddb1018..8202b36 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -29,6 +29,13 @@ Version 0.19.0 you're validating certificate fingerprints, use `sha256` instead. - The ``google`` storage types no longer require ``requests-oauthlib``, but require ``python-aiohttp-oauthlib`` instead. +- Vdirsyncer no longer includes experimental support for `EteSync + `_. The existing integration had not been supported + for a long time and no longer worked. Support for external storages may be + added if anyone is interested in maintaining an EteSync plugin. EteSync + users should consider using `etesync-dav`_. + +.. _etesync-dav: https://github.com/etesync/etesync-dav Version 0.18.0 ============== diff --git a/MANIFEST.in b/MANIFEST.in index 9260ca0..64e6349 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,7 +2,6 @@ prune docker prune scripts prune tests/storage/servers -prune tests/storage/etesync recursive-include tests/storage/servers/radicale * recursive-include tests/storage/servers/skip * diff --git a/Makefile b/Makefile index 1fc70c9..9e18e82 100644 --- a/Makefile +++ b/Makefile @@ -12,9 +12,6 @@ export REQUIREMENTS := release # Set this to true if you run vdirsyncer's test as part of e.g. packaging. export DETERMINISTIC_TESTS := false -# Run the etesync testsuite. -export ETESYNC_TESTS := false - # Assume to run in CI. Don't use this outside of a virtual machine. It will # heavily "pollute" your system, such as attempting to install a new Python # systemwide. @@ -30,11 +27,6 @@ PYTEST_ARGS = TEST_EXTRA_PACKAGES = -ifeq ($(ETESYNC_TESTS), true) - TEST_EXTRA_PACKAGES += git+https://github.com/etesync/journal-manager@v0.5.2 - TEST_EXTRA_PACKAGES += django djangorestframework==3.8.2 wsgi_intercept drf-nested-routers -endif - PYTEST = py.test $(PYTEST_ARGS) CODECOV_PATH = /tmp/codecov.sh @@ -45,7 +37,6 @@ ci-test: curl -s https://codecov.io/bash > $(CODECOV_PATH) $(PYTEST) --cov vdirsyncer --cov-append tests/unit/ tests/system/ bash $(CODECOV_PATH) -c - [ "$(ETESYNC_TESTS)" = "false" ] || make test-storage ci-test-storage: curl -s https://codecov.io/bash > $(CODECOV_PATH) @@ -83,7 +74,6 @@ install-dev: pip install -e . pip install -Ur test-requirements.txt $(TEST_EXTRA_PACKAGES) pip install pre-commit - [ "$(ETESYNC_TESTS)" = "false" ] || pip install -Ue .[etesync] set -xe && if [ "$(REQUIREMENTS)" = "minimal" ]; then \ pip install -U --force-reinstall $$(python setup.py --quiet minimal_requirements); \ fi diff --git a/docs/config.rst b/docs/config.rst index f6a6556..3b85ecf 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -349,56 +349,6 @@ or write anything to it. :param client_id/client_secret: OAuth credentials, obtained from the Google API Manager. -EteSync -+++++++ - -`EteSync `_ is a new cloud provider for end to end -encrypted contacts and calendar storage. Vdirsyncer contains **experimental** -support for it. - -To use it, you need to install some optional dependencies:: - - pip install vdirsyncer[etesync] - -On first usage you will be prompted for the service password and the encryption -password. Neither are stored. - -.. storage:: etesync_contacts - - Contacts for etesync. - - :: - - [storage example_for_etesync_contacts] - email = ... - secrets_dir = ... - #server_path = ... - #db_path = ... - - :param email: The email address of your account. - :param secrets_dir: A directory where vdirsyncer can store the encryption - key and authentication token. - :param server_url: Optional. URL to the root of your custom server. - :param db_path: Optional. Use a different path for the database. - -.. storage:: etesync_calendars - - Calendars for etesync. - - :: - - [storage example_for_etesync_calendars] - email = ... - secrets_dir = ... - #server_path = ... - #db_path = ... - - :param email: The email address of your account. - :param secrets_dir: A directory where vdirsyncer can store the encryption - key and authentication token. - :param server_url: Optional. URL to the root of your custom server. - :param db_path: Optional. Use a different path for the database. - Local +++++ diff --git a/setup.py b/setup.py index 99f8de7..e523e59 100644 --- a/setup.py +++ b/setup.py @@ -56,7 +56,6 @@ setup( # Optional dependencies extras_require={ "google": ["aiohttp-oauthlib"], - "etesync": ["etesync==0.5.2", "django<2.0"], }, # Build dependencies setup_requires=["setuptools_scm != 1.12.0"], diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 699c75e..029261e 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -254,9 +254,6 @@ class StorageTests: @pytest.mark.asyncio async def test_collection_arg(self, get_storage_args): - if self.storage_class.storage_name.startswith("etesync"): - pytest.skip("etesync uses UUIDs.") - if self.supports_collections: s = self.storage_class(**await get_storage_args(collection="test2")) # Can't do stronger assertion because of radicale, which needs a @@ -302,10 +299,6 @@ class StorageTests: ((_, etag3),) = await aiostream.stream.list(s.list()) assert etag2 == etag3 - # etesync uses UUIDs for collection names - if self.storage_class.storage_name.startswith("etesync"): - return - assert collection in urlunquote(s.collection) if self.storage_class.storage_name.endswith("dav"): assert urlquote(uid, "/@:") in href diff --git a/tests/storage/etesync/__init__.py b/tests/storage/etesync/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/storage/etesync/etesync_server/db.sqlite3 b/tests/storage/etesync/etesync_server/db.sqlite3 deleted file mode 100644 index 78126e7..0000000 Binary files a/tests/storage/etesync/etesync_server/db.sqlite3 and /dev/null differ diff --git a/tests/storage/etesync/etesync_server/etesync_server/__init__.py b/tests/storage/etesync/etesync_server/etesync_server/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/storage/etesync/etesync_server/etesync_server/settings.py b/tests/storage/etesync/etesync_server/etesync_server/settings.py deleted file mode 100644 index 6a28ec7..0000000 --- a/tests/storage/etesync/etesync_server/etesync_server/settings.py +++ /dev/null @@ -1,122 +0,0 @@ -""" -Django settings for etesync_server project. - -Generated by 'django-admin startproject' using Django 1.10.6. - -For more information on this file, see -https://docs.djangoproject.com/en/1.10/topics/settings/ - -For the full list of settings and their values, see -https://docs.djangoproject.com/en/1.10/ref/settings/ -""" -import os - -# Build paths inside the project like this: os.path.join(BASE_DIR, ...) -BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) - - -# Quick-start development settings - unsuitable for production -# See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ - -# SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = "d7r(p-9=$3a@bbt%*+$p@4)cej13nzd0gmnt8+m0bitb=-umj#" - -# SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True - -ALLOWED_HOSTS = [] - - -# Application definition - -INSTALLED_APPS = [ - "django.contrib.admin", - "django.contrib.auth", - "django.contrib.contenttypes", - "django.contrib.sessions", - "django.contrib.messages", - "django.contrib.staticfiles", - "rest_framework", - "rest_framework.authtoken", - "journal.apps.JournalConfig", -] - -MIDDLEWARE = [ - "django.middleware.security.SecurityMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.common.CommonMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "django.middleware.clickjacking.XFrameOptionsMiddleware", -] - -ROOT_URLCONF = "etesync_server.urls" - -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [], - "APP_DIRS": True, - "OPTIONS": { - "context_processors": [ - "django.template.context_processors.debug", - "django.template.context_processors.request", - "django.contrib.auth.context_processors.auth", - "django.contrib.messages.context_processors.messages", - ], - }, - }, -] - -WSGI_APPLICATION = "etesync_server.wsgi.application" - - -# Database -# https://docs.djangoproject.com/en/1.10/ref/settings/#databases - -DATABASES = { - "default": { - "ENGINE": "django.db.backends.sqlite3", - "NAME": os.environ.get("ETESYNC_DB_PATH", os.path.join(BASE_DIR, "db.sqlite3")), - } -} - - -# Password validation -# https://docs.djangoproject.com/en/1.10/ref/settings/#auth-password-validators - -AUTH_PASSWORD_VALIDATORS = [ - { - "NAME": "django.contrib.auth.password_validation.UserAttributeSimilarityValidator", # noqa - }, - { - "NAME": "django.contrib.auth.password_validation.MinimumLengthValidator", # noqa - }, - { - "NAME": "django.contrib.auth.password_validation.CommonPasswordValidator", # noqa - }, - { - "NAME": "django.contrib.auth.password_validation.NumericPasswordValidator", # noqa - }, -] - - -# Internationalization -# https://docs.djangoproject.com/en/1.10/topics/i18n/ - -LANGUAGE_CODE = "en-us" - -TIME_ZONE = "UTC" - -USE_I18N = True - -USE_L10N = True - -USE_TZ = True - - -# Static files (CSS, JavaScript, Images) -# https://docs.djangoproject.com/en/1.10/howto/static-files/ - -STATIC_URL = "/static/" diff --git a/tests/storage/etesync/etesync_server/etesync_server/urls.py b/tests/storage/etesync/etesync_server/etesync_server/urls.py deleted file mode 100644 index 7f3bd32..0000000 --- a/tests/storage/etesync/etesync_server/etesync_server/urls.py +++ /dev/null @@ -1,37 +0,0 @@ -"""etesync_server URL Configuration - -The `urlpatterns` list routes URLs to views. For more information please see: - https://docs.djangoproject.com/en/1.10/topics/http/urls/ -Examples: -Function views - 1. Add an import: from my_app import views - 2. Add a URL to urlpatterns: url(r'^$', views.home, name='home') -Class-based views - 1. Add an import: from other_app.views import Home - 2. Add a URL to urlpatterns: url(r'^$', Home.as_view(), name='home') -Including another URLconf - 1. Import the include() function: from django.conf.urls import url, include - 2. Add a URL to urlpatterns: url(r'^blog/', include('blog.urls')) -""" -from django.conf.urls import include -from django.conf.urls import url -from journal import views -from rest_framework_nested import routers - -router = routers.DefaultRouter() -router.register(r"journals", views.JournalViewSet) -router.register(r"journal/(?P[^/]+)", views.EntryViewSet) -router.register(r"user", views.UserInfoViewSet) - -journals_router = routers.NestedSimpleRouter(router, r"journals", lookup="journal") -journals_router.register(r"members", views.MembersViewSet, base_name="journal-members") -journals_router.register(r"entries", views.EntryViewSet, base_name="journal-entries") - - -urlpatterns = [ - url(r"^api/v1/", include(router.urls)), - url(r"^api/v1/", include(journals_router.urls)), -] - -# Adding this just for testing, this shouldn't be here normally -urlpatterns += (url(r"^reset/$", views.reset, name="reset_debug"),) diff --git a/tests/storage/etesync/etesync_server/etesync_server/wsgi.py b/tests/storage/etesync/etesync_server/etesync_server/wsgi.py deleted file mode 100644 index 147e71e..0000000 --- a/tests/storage/etesync/etesync_server/etesync_server/wsgi.py +++ /dev/null @@ -1,15 +0,0 @@ -""" -WSGI config for etesync_server project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/1.10/howto/deployment/wsgi/ -""" -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings") - -application = get_wsgi_application() diff --git a/tests/storage/etesync/etesync_server/manage.py b/tests/storage/etesync/etesync_server/manage.py deleted file mode 100755 index b100c54..0000000 --- a/tests/storage/etesync/etesync_server/manage.py +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env python -import os -import sys - -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings") - try: - from django.core.management import execute_from_command_line - except ImportError: - # The above import may fail for some other reason. Ensure that the - # issue is really that Django is missing to avoid masking other - # exceptions on Python 2. - try: - import django # noqa - except ImportError: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) - raise - execute_from_command_line(sys.argv) diff --git a/tests/storage/etesync/test@localhost/auth_token b/tests/storage/etesync/test@localhost/auth_token deleted file mode 100644 index 7eeeadb..0000000 --- a/tests/storage/etesync/test@localhost/auth_token +++ /dev/null @@ -1 +0,0 @@ -63ae6eec45b592d5c511f79b7b0c312d2c5f7d6a diff --git a/tests/storage/etesync/test@localhost/key b/tests/storage/etesync/test@localhost/key deleted file mode 100644 index 0e3b27d..0000000 Binary files a/tests/storage/etesync/test@localhost/key and /dev/null differ diff --git a/tests/storage/etesync/test_main.py b/tests/storage/etesync/test_main.py deleted file mode 100644 index 85ab27e..0000000 --- a/tests/storage/etesync/test_main.py +++ /dev/null @@ -1,88 +0,0 @@ -import os -import shutil -import sys - -import pytest -import requests - -from vdirsyncer.storage.etesync import EtesyncCalendars -from vdirsyncer.storage.etesync import EtesyncContacts - -from .. import StorageTests - -pytestmark = pytest.mark.skipif( - os.getenv("ETESYNC_TESTS", "") != "true", reason="etesync tests disabled" -) - - -@pytest.fixture(scope="session") -def etesync_app(tmpdir_factory): - sys.path.insert(0, os.path.join(os.path.dirname(__file__), "etesync_server")) - - db = tmpdir_factory.mktemp("etesync").join("etesync.sqlite") - shutil.copy( - os.path.join(os.path.dirname(__file__), "etesync_server", "db.sqlite3"), str(db) - ) - - os.environ["ETESYNC_DB_PATH"] = str(db) - from etesync_server.wsgi import application - - return application - - -class EtesyncTests(StorageTests): - - supports_metadata = False - - @pytest.fixture - def get_storage_args(self, request, get_item, tmpdir, etesync_app): - import wsgi_intercept - import wsgi_intercept.requests_intercept - - wsgi_intercept.requests_intercept.install() - wsgi_intercept.add_wsgi_intercept("127.0.0.1", 8000, lambda: etesync_app) - - def teardown(): - wsgi_intercept.remove_wsgi_intercept("127.0.0.1", 8000) - wsgi_intercept.requests_intercept.uninstall() - - request.addfinalizer(teardown) - - with open( - os.path.join(os.path.dirname(__file__), "test@localhost/auth_token") - ) as f: - token = f.read().strip() - headers = {"Authorization": "Token " + token} - r = requests.post( - "http://127.0.0.1:8000/reset/", headers=headers, allow_redirects=False - ) - assert r.status_code == 200 - - async def inner(collection="test"): - rv = { - "email": "test@localhost", - "db_path": str(tmpdir.join("etesync.db")), - "secrets_dir": os.path.dirname(__file__), - "server_url": "http://127.0.0.1:8000/", - } - if collection is not None: - rv = self.storage_class.create_collection(collection=collection, **rv) - return rv - - return inner - - -class TestContacts(EtesyncTests): - storage_class = EtesyncContacts - - @pytest.fixture(params=["VCARD"]) - def item_type(self, request): - return request.param - - -class TestCalendars(EtesyncTests): - storage_class = EtesyncCalendars - - @pytest.fixture(params=["VEVENT"]) - def item_type(self, request): - return request.param diff --git a/vdirsyncer/cli/utils.py b/vdirsyncer/cli/utils.py index a25fff7..acd6dea 100644 --- a/vdirsyncer/cli/utils.py +++ b/vdirsyncer/cli/utils.py @@ -35,8 +35,6 @@ class _StorageIndex: singlefile="vdirsyncer.storage.singlefile.SingleFileStorage", google_calendar="vdirsyncer.storage.google.GoogleCalendarStorage", google_contacts="vdirsyncer.storage.google.GoogleContactsStorage", - etesync_calendars="vdirsyncer.storage.etesync.EtesyncCalendars", - etesync_contacts="vdirsyncer.storage.etesync.EtesyncContacts", ) def __getitem__(self, name): diff --git a/vdirsyncer/storage/etesync.py b/vdirsyncer/storage/etesync.py deleted file mode 100644 index f3d9aba..0000000 --- a/vdirsyncer/storage/etesync.py +++ /dev/null @@ -1,237 +0,0 @@ -import binascii -import contextlib -import functools -import logging -import os - -import atomicwrites -import click - -try: - import etesync - import etesync.exceptions - from etesync import AddressBook - from etesync import Calendar - from etesync import Contact - from etesync import Event - - has_etesync = True -except ImportError: - has_etesync = False - AddressBook = Contact = Calendar = Event = None - -from .. import exceptions -from ..cli.utils import assert_permissions -from ..utils import checkdir -from ..vobject import Item -from .base import Storage - -logger = logging.getLogger(__name__) - - -def _writing_op(f): - @functools.wraps(f) - async def inner(self, *args, **kwargs): - if not self._at_once: - self._sync_journal() - rv = f(self, *args, **kwargs) - if not self._at_once: - self._sync_journal() - return await rv - - return inner - - -class _Session: - def __init__(self, email, secrets_dir, server_url=None, db_path=None): - if not has_etesync: - raise exceptions.UserError("Dependencies for etesync are not " "installed.") - server_url = server_url or etesync.API_URL - self.email = email - self.secrets_dir = os.path.join(secrets_dir, email + "/") - - self._auth_token_path = os.path.join(self.secrets_dir, "auth_token") - self._key_path = os.path.join(self.secrets_dir, "key") - - auth_token = self._get_auth_token() - if not auth_token: - password = click.prompt( - f"Enter service password for {self.email}", hide_input=True - ) - auth_token = etesync.Authenticator(server_url).get_auth_token( - self.email, password - ) - self._set_auth_token(auth_token) - - self._db_path = db_path or os.path.join(self.secrets_dir, "db.sqlite") - self.etesync = etesync.EteSync( - email, auth_token, remote=server_url, db_path=self._db_path - ) - - key = self._get_key() - if not key: - password = click.prompt("Enter key password", hide_input=True) - click.echo(f"Deriving key for {self.email}") - self.etesync.derive_key(password) - self._set_key(self.etesync.cipher_key) - else: - self.etesync.cipher_key = key - - def _get_auth_token(self): - try: - with open(self._auth_token_path) as f: - return f.read().strip() or None - except OSError: - pass - - def _set_auth_token(self, token): - checkdir(os.path.dirname(self._auth_token_path), create=True) - with atomicwrites.atomic_write(self._auth_token_path) as f: - f.write(token) - assert_permissions(self._auth_token_path, 0o600) - - def _get_key(self): - try: - with open(self._key_path, "rb") as f: - return f.read() - except OSError: - pass - - def _set_key(self, content): - checkdir(os.path.dirname(self._key_path), create=True) - with atomicwrites.atomic_write(self._key_path, mode="wb") as f: - f.write(content) - assert_permissions(self._key_path, 0o600) - - -class EtesyncStorage(Storage): - _collection_type = None - _item_type = None - _at_once = False - - def __init__(self, email, secrets_dir, server_url=None, db_path=None, **kwargs): - if kwargs.get("collection", None) is None: - raise ValueError("Collection argument required") - - self._session = _Session(email, secrets_dir, server_url, db_path) - super().__init__(**kwargs) - self._journal = self._session.etesync.get(self.collection) - - def _sync_journal(self): - self._session.etesync.sync_journal(self.collection) - - @classmethod - async def discover( - cls, - email, - secrets_dir, - server_url=None, - db_path=None, - **kwargs, - ): - if kwargs.get("collection", None) is not None: - raise TypeError("collection argument must not be given.") - session = _Session(email, secrets_dir, server_url, db_path) - assert cls._collection_type - session.etesync.sync_journal_list() - for entry in session.etesync.list(): - if isinstance(entry.collection, cls._collection_type): - yield dict( - email=email, - secrets_dir=secrets_dir, - db_path=db_path, - collection=entry.uid, - **kwargs, - ) - else: - logger.debug(f"Skipping collection: {entry!r}") - - @classmethod - async def create_collection( - cls, collection, email, secrets_dir, server_url=None, db_path=None, **kwargs - ): - session = _Session(email, secrets_dir, server_url, db_path) - content = {"displayName": collection} - c = cls._collection_type.create( - session.etesync, binascii.hexlify(os.urandom(32)).decode(), content - ) - c.save() - session.etesync.sync_journal_list() - return dict( - collection=c.journal.uid, - email=email, - secrets_dir=secrets_dir, - db_path=db_path, - server_url=server_url, - **kwargs, - ) - - async def list(self): - self._sync_journal() - for entry in self._journal.collection.list(): - item = Item(entry.content) - yield str(entry.uid), item.hash - - async def get(self, href): - try: - item = Item(self._journal.collection.get(href).content) - except etesync.exceptions.DoesNotExist as e: - raise exceptions.NotFoundError(e) - return item, item.hash - - @_writing_op - async def upload(self, item): - try: - entry = self._item_type.create(self._journal.collection, item.raw) - entry.save() - except etesync.exceptions.DoesNotExist as e: - raise exceptions.NotFoundError(e) - except etesync.exceptions.AlreadyExists as e: - raise exceptions.AlreadyExistingError(e) - return item.uid, item.hash - - @_writing_op - async def update(self, href, item, etag): - try: - entry = self._journal.collection.get(href) - except etesync.exceptions.DoesNotExist as e: - raise exceptions.NotFoundError(e) - old_item = Item(entry.content) - if old_item.hash != etag: - raise exceptions.WrongEtagError(etag, old_item.hash) - entry.content = item.raw - entry.save() - return item.hash - - @_writing_op - async def delete(self, href, etag): - try: - entry = self._journal.collection.get(href) - old_item = Item(entry.content) - if old_item.hash != etag: - raise exceptions.WrongEtagError(etag, old_item.hash) - entry.delete() - except etesync.exceptions.DoesNotExist as e: - raise exceptions.NotFoundError(e) - - @contextlib.asynccontextmanager - async def at_once(self): - self._sync_journal() - self._at_once = True - try: - yield self - self._sync_journal() - finally: - self._at_once = False - - -class EtesyncContacts(EtesyncStorage): - _collection_type = AddressBook - _item_type = Contact - storage_name = "etesync_contacts" - - -class EtesyncCalendars(EtesyncStorage): - _collection_type = Calendar - _item_type = Event - storage_name = "etesync_calendars"