diff --git a/.travis.yml b/.travis.yml index 96f5e70..8f88823 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,22 +10,34 @@ env: - BUILD=test # Default build, see Makefile - - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem REQUIREMENTS=release + - BUILD=style + # flake8 with plugins + + # REMOTESTORAGE TESTS + + # - BUILD=test REMOTESTORAGE_SERVER=restore + # Testing against reStore + # https://github.com/jcoglan/restore/issues/38 + # https://github.com/jcoglan/restore/issues/37 + + # DAV TESTS + + - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem + # Radicale-release with filesystem storage + + - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem PKGS='lxml==3.0 requests==2.4.1 requests_toolbelt==0.4.0 click==5.0' # Minimal requirements - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem REQUIREMENTS=devel - # Radicale-git with filesystem storage (default) + # Radicale-git with filesystem storage - - BUILD=test DAV_SERVER=owncloud REQUIREMENTS=release + - BUILD=test DAV_SERVER=owncloud # Latest ownCloud release - - BUILD=test DAV_SERVER=baikal REQUIREMENTS=release + - BUILD=test DAV_SERVER=baikal # Latest Baikal release - - BUILD=style - # flake8 with plugins - install: - "pip install -U pip" - "pip install wheel" diff --git a/Makefile b/Makefile index 0c02292..495c4fd 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ # If you want to skip the DAV tests against Radicale, use: # make DAV_SERVER=skip # ... -export DAV_SERVER := radicale +export DAV_SERVER := skip +export REMOTESTORAGE_SERVER := skip export RADICALE_BACKEND := filesystem export REQUIREMENTS := release export TESTSERVER_BASE := ./tests/storage/servers/ @@ -19,7 +20,7 @@ export TRAVIS := false install-servers: set -ex; \ - for server in $(DAV_SERVER); do \ + for server in $(DAV_SERVER) $(REMOTESTORAGE_SERVER); do \ if [ ! -d "$(TESTSERVER_BASE)$$server/" ]; then \ git clone --depth=1 \ https://github.com/vdirsyncer/$$server-testserver.git \ diff --git a/docs/config.rst b/docs/config.rst index 64015dd..cb7ebbd 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -120,14 +120,35 @@ These storages generally support reading and changing of their items. Their default value for ``read_only`` is ``false``, but can be set to ``true`` if wished. +CalDAV and CardDAV +++++++++++++++++++ + .. autostorage:: vdirsyncer.storage.dav.CaldavStorage .. autostorage:: vdirsyncer.storage.dav.CarddavStorage +remoteStorage ++++++++++++++ + +`remoteStorage `_ is an open per-user data storage +protocol. Vdirsyncer contains **highly experimental support** for it. + +.. note:: + + Do not use this storage if you're not prepared for data-loss and breakage. + +.. autostorage:: vdirsyncer.storage.remotestorage.RemoteStorageContacts + +.. autostorage:: vdirsyncer.storage.remotestorage.RemoteStorageCalendars + +Local ++++++ + .. autostorage:: vdirsyncer.storage.filesystem.FilesystemStorage .. autostorage:: vdirsyncer.storage.singlefile.SingleFileStorage + Read-only storages ~~~~~~~~~~~~~~~~~~ diff --git a/setup.py b/setup.py index fccda9d..968ebce 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,7 @@ setup( 'click-log', 'click-threading', 'requests', + 'requests-oauthlib', 'lxml>=3.0', # https://github.com/sigmavirus24/requests-toolbelt/pull/28 'requests_toolbelt>=0.4.0', diff --git a/tests/storage/test_remotestorage.py b/tests/storage/test_remotestorage.py new file mode 100644 index 0000000..f14bdd0 --- /dev/null +++ b/tests/storage/test_remotestorage.py @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- + + +import os + +import pytest + +from vdirsyncer.storage.remotestorage import \ + RemoteStorageCalendars, RemoteStorageContacts + +from . import StorageTests, get_server_mixin + +remotestorage_server = os.environ['REMOTESTORAGE_SERVER'] +ServerMixin = get_server_mixin(remotestorage_server) + + +class RemoteStorageTests(ServerMixin, StorageTests): + remotestorage_server = remotestorage_server + + +class TestCalendars(RemoteStorageTests): + storage_class = RemoteStorageCalendars + + @pytest.fixture(params=['VTODO', 'VEVENT']) + def item_type(self, request): + return request.param + + +class TestContacts(RemoteStorageTests): + storage_class = RemoteStorageContacts + supports_collections = False + + @pytest.fixture(params=['VCARD']) + def item_type(self, request): + return request.param diff --git a/vdirsyncer/cli/utils.py b/vdirsyncer/cli/utils.py index 589a044..00980d0 100644 --- a/vdirsyncer/cli/utils.py +++ b/vdirsyncer/cli/utils.py @@ -41,6 +41,10 @@ class _StorageIndex(object): filesystem='vdirsyncer.storage.filesystem.FilesystemStorage', http='vdirsyncer.storage.http.HttpStorage', singlefile='vdirsyncer.storage.singlefile.SingleFileStorage', + remotestorage_contacts=( + 'vdirsyncer.storage.remotestorage.RemoteStorageContacts'), + remotestorage_calendars=( + 'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'), ) def __getitem__(self, name): diff --git a/vdirsyncer/storage/remotestorage.py b/vdirsyncer/storage/remotestorage.py new file mode 100644 index 0000000..3a1153f --- /dev/null +++ b/vdirsyncer/storage/remotestorage.py @@ -0,0 +1,275 @@ +''' +A storage type for accessing contact and calendar data from `remoteStorage +`_. It is highly experimental. + +A few things are hardcoded for now so the user doesn't have to specify those +things, and plugging in an account "just works". + +We also use a custom ``data``-URI for the redirect in OAuth: + +- There is no server that could be compromised. +- With a proper URL, ``access_token`` would be stored in the browser history. + For some reason Firefox doesn't do that with ``data``-URIs. +- ``data``-URIs have no clear domain name that could prevent from phishing + attacks. However, I don't see a way to phish without compromising the + vdirsyncer installation, at which point any hope would already be lost. +- On the downside, redirect URIs are monstrous. + +''' + +import click + +from .base import Item, Storage +from .http import HTTP_STORAGE_PARAMETERS, USERAGENT, prepare_client_cert, \ + prepare_verify +from .. import exceptions, log, utils + +REDIRECT_URI = 'https://vdirsyncer.5apps.com/' +CLIENT_ID = 'https://vdirsyncer.5apps.com' +DRAFT_VERSION = '05' + +logger = log.get(__name__) + +urljoin = utils.compat.urlparse.urljoin +urlquote = utils.compat.urlquote + + +def _ensure_slash(dir): + return dir.rstrip('/') + '/' + + +def _iter_listing(json): + new_listing = '@context' in json # draft-02 and beyond + if new_listing: + json = json['items'] + for name, info in utils.compat.iteritems(json): + if not new_listing: + info = {'ETag': info} + yield name, info + + +class Session(object): + + def __init__(self, account, scope, verify=True, verify_fingerprint=None, + auth_cert=None, access_token=None, collection=None): + from oauthlib.oauth2 import MobileApplicationClient + from requests_oauthlib import OAuth2Session + + self.user, self.host = account.split('@') + + self._settings = { + 'cert': prepare_client_cert(auth_cert) + } + self._settings.update(prepare_verify(verify, verify_fingerprint)) + + self.scope = scope + ':rw' + self._session = OAuth2Session( + CLIENT_ID, client=MobileApplicationClient(CLIENT_ID), + scope=self.scope, + redirect_uri=REDIRECT_URI, + token={'access_token': access_token}, + ) + + subpath = scope + if collection: + subpath = urljoin(_ensure_slash(scope), + _ensure_slash(urlquote(collection))) + + self._discover_endpoints(subpath) + + if not access_token: + self._get_access_token() + + def request(self, method, path, **kwargs): + url = self.endpoints['storage'] + if path: + url = urljoin(url, path) + + settings = dict(self._settings) + settings.update(kwargs) + + return utils.http.request(method, url, + session=self._session, **settings) + + def _get_access_token(self): + authorization_url, state = \ + self._session.authorization_url(self.endpoints['oauth']) + + click.echo('Go to {}'.format(authorization_url)) + click.echo('Follow the instructions on the page.') + raise exceptions.UserError('Aborted!') + + def _discover_endpoints(self, subpath): + r = utils.http.request( + 'GET', 'https://{host}/.well-known/webfinger?resource=acct:{user}' + .format(host=self.host, user=self.user), + **self._settings + ) + j = r.json() + for link in j['links']: + if 'remotestorage' in link['rel']: + break + + storage = urljoin(_ensure_slash(link['href']), + _ensure_slash(subpath)) + props = link['properties'] + oauth = props['http://tools.ietf.org/html/rfc6749#section-4.2'] + self.endpoints = dict(storage=storage, oauth=oauth) + + +class RemoteStorage(Storage): + __doc__ = ''' + :param account: remoteStorage account, ``"user@example.com"``. + ''' + HTTP_STORAGE_PARAMETERS + ''' + ''' + + storage_name = None + item_mimetype = None + fileext = None + + def __init__(self, account, verify=True, verify_fingerprint=None, + auth_cert=None, access_token=None, **kwargs): + super(RemoteStorage, self).__init__(**kwargs) + self.session = Session( + account=account, + verify=verify, + verify_fingerprint=verify_fingerprint, + auth_cert=auth_cert, + access_token=access_token, + collection=self.collection, + scope=self.scope) + + @classmethod + def discover(cls, **base_args): + if base_args.pop('collection', None) is not None: + raise TypeError('collection argument must not be given.') + + session_args, _ = utils.split_dict(base_args, lambda key: key in ( + 'account', 'verify', 'auth', 'verify_fingerprint', 'auth_cert', + 'access_token' + )) + + session = Session(scope=cls.scope, **session_args) + + try: + r = session.request('GET', '') + except exceptions.NotFoundError: + return + + for name, info in _iter_listing(r.json()): + if not name.endswith('/'): + continue # not a folder + + newargs = dict(base_args) + newargs['collection'] = name.rstrip('/') + yield newargs + + @classmethod + def create_collection(cls, collection, **kwargs): + # remoteStorage folders are autocreated. + assert collection + assert '/' not in collection + kwargs['collection'] = collection + return kwargs + + def list(self): + try: + r = self.session.request('GET', '') + except exceptions.NotFoundError: + return + + for name, info in _iter_listing(r.json()): + if not name.endswith(self.fileext): + continue + + etag = info['ETag'] + etag = '"' + etag + '"' + yield name, etag + + def _put(self, href, item, etag): + headers = {'Content-Type': self.item_mimetype + '; charset=UTF-8'} + if etag is None: + headers['If-None-Match'] = '*' + else: + headers['If-Match'] = etag + + response = self.session.request( + 'PUT', + href, + data=item.raw.encode('utf-8'), + headers=headers + ) + if not response.url.endswith('/' + href): + raise exceptions.InvalidResponse('spec doesn\'t allow redirects') + return href, response.headers['etag'] + + def update(self, href, item, etag): + assert etag + href, etag = self._put(href, item, etag) + return etag + + def upload(self, item): + href = utils.generate_href(item.ident) + href = utils.compat.urlquote(href, '@') + self.fileext + return self._put(href, item, None) + + def delete(self, href, etag): + headers = {'If-Match': etag} + self.session.request('DELETE', href, headers=headers) + + def get(self, href): + response = self.session.request('GET', href) + return Item(response.text), response.headers['etag'] + + def get_meta(self, key): + try: + return self.session.request('GET', key).text or None + except exceptions.NotFoundError: + pass + + def set_meta(self, key, value): + self.session.request( + 'PUT', + key, + data=(value or u'').encode('utf-8'), + headers={'Content-Type': 'text/plain; charset=utf-8'} + ) + + +class RemoteStorageContacts(RemoteStorage): + __doc__ = ''' + remoteStorage contacts. Uses the `vdir_contacts` scope. + ''' + RemoteStorage.__doc__ + + storage_name = 'remotestorage_contacts' + fileext = '.vcf' + item_mimetype = 'text/vcard' + scope = 'vdir_contacts' + + def __init__(self, **kwargs): + if kwargs.get('collection'): + raise ValueError( + 'No collections allowed for contacts, ' + 'there is only one addressbook. ' + 'Use the vcard groups construct to categorize your contacts ' + 'into groups.' + ) + + super(RemoteStorageContacts, self).__init__(**kwargs) + + +class RemoteStorageCalendars(RemoteStorage): + __doc__ = ''' + remoteStorage calendars. Uses the `vdir_calendars` scope. + ''' + RemoteStorage.__doc__ + + storage_name = 'remotestorage_calendars' + fileext = '.ics' + item_mimetype = 'text/icalendar' + scope = 'vdir_calendars' + + def __init__(self, **kwargs): + if not kwargs.get('collection'): + raise ValueError('The collections parameter is required.') + + super(RemoteStorageCalendars, self).__init__(**kwargs)