Merge branch 'remotestorage'

.travis.yml

@@ -10,22 +10,34 @@ env:
     - BUILD=test
       # Default build, see Makefile
 
-    - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem REQUIREMENTS=release
+    - BUILD=style
+      # flake8 with plugins
+
+    # REMOTESTORAGE TESTS
+
+  #  - BUILD=test REMOTESTORAGE_SERVER=restore
+      # Testing against reStore
+      # https://github.com/jcoglan/restore/issues/38
+      # https://github.com/jcoglan/restore/issues/37
+
+    # DAV TESTS
+
+    - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem
+      # Radicale-release with filesystem storage
+
+    - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem
       PKGS='lxml==3.0 requests==2.4.1 requests_toolbelt==0.4.0 click==5.0'
       # Minimal requirements
 
     - BUILD=test DAV_SERVER=radicale RADICALE_BACKEND=filesystem REQUIREMENTS=devel
-      # Radicale-git with filesystem storage (default)
+      # Radicale-git with filesystem storage
 
-    - BUILD=test DAV_SERVER=owncloud REQUIREMENTS=release
+    - BUILD=test DAV_SERVER=owncloud
       # Latest ownCloud release
 
-    - BUILD=test DAV_SERVER=baikal REQUIREMENTS=release
+    - BUILD=test DAV_SERVER=baikal
       # Latest Baikal release
 
-    - BUILD=style
-      # flake8 with plugins
-
 install:
     - "pip install -U pip"
     - "pip install wheel"

Makefile

@@ -11,7 +11,8 @@
 # If you want to skip the DAV tests against Radicale, use:
 #     make DAV_SERVER=skip # ...
 
-export DAV_SERVER := radicale
+export DAV_SERVER := skip
+export REMOTESTORAGE_SERVER := skip
 export RADICALE_BACKEND := filesystem
 export REQUIREMENTS := release
 export TESTSERVER_BASE := ./tests/storage/servers/
@@ -19,7 +20,7 @@ export TRAVIS := false
 
 install-servers:
 	set -ex; \
-	for server in $(DAV_SERVER); do \
+	for server in $(DAV_SERVER) $(REMOTESTORAGE_SERVER); do \
 		if [ ! -d "$(TESTSERVER_BASE)$$server/" ]; then \
 			git clone --depth=1 \
 				https://github.com/vdirsyncer/$$server-testserver.git \

docs/config.rst

@@ -120,14 +120,35 @@ These storages generally support reading and changing of their items. Their
 default value for ``read_only`` is ``false``, but can be set to ``true`` if
 wished.
 
+CalDAV and CardDAV
+++++++++++++++++++
+
 .. autostorage:: vdirsyncer.storage.dav.CaldavStorage
 
 .. autostorage:: vdirsyncer.storage.dav.CarddavStorage
 
+remoteStorage
++++++++++++++
+
+`remoteStorage <https://remotestorage.io/>`_ is an open per-user data storage
+protocol. Vdirsyncer contains **highly experimental support** for it.
+
+.. note::
+
+    Do not use this storage if you're not prepared for data-loss and breakage.
+
+.. autostorage:: vdirsyncer.storage.remotestorage.RemoteStorageContacts
+
+.. autostorage:: vdirsyncer.storage.remotestorage.RemoteStorageCalendars
+
+Local
++++++
+
 .. autostorage:: vdirsyncer.storage.filesystem.FilesystemStorage
 
 .. autostorage:: vdirsyncer.storage.singlefile.SingleFileStorage
 
+
 Read-only storages
 ~~~~~~~~~~~~~~~~~~
 

setup.py

@@ -36,6 +36,7 @@ setup(
         'click-log',
         'click-threading',
         'requests',
+        'requests-oauthlib',
         'lxml>=3.0',
         # https://github.com/sigmavirus24/requests-toolbelt/pull/28
         'requests_toolbelt>=0.4.0',

tests/storage/test_remotestorage.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+
+import os
+
+import pytest
+
+from vdirsyncer.storage.remotestorage import \
+    RemoteStorageCalendars, RemoteStorageContacts
+
+from . import StorageTests, get_server_mixin
+
+remotestorage_server = os.environ['REMOTESTORAGE_SERVER']
+ServerMixin = get_server_mixin(remotestorage_server)
+
+
+class RemoteStorageTests(ServerMixin, StorageTests):
+    remotestorage_server = remotestorage_server
+
+
+class TestCalendars(RemoteStorageTests):
+    storage_class = RemoteStorageCalendars
+
+    @pytest.fixture(params=['VTODO', 'VEVENT'])
+    def item_type(self, request):
+        return request.param
+
+
+class TestContacts(RemoteStorageTests):
+    storage_class = RemoteStorageContacts
+    supports_collections = False
+
+    @pytest.fixture(params=['VCARD'])
+    def item_type(self, request):
+        return request.param

vdirsyncer/cli/utils.py

@@ -41,6 +41,10 @@ class _StorageIndex(object):
             filesystem='vdirsyncer.storage.filesystem.FilesystemStorage',
             http='vdirsyncer.storage.http.HttpStorage',
             singlefile='vdirsyncer.storage.singlefile.SingleFileStorage',
+            remotestorage_contacts=(
+                'vdirsyncer.storage.remotestorage.RemoteStorageContacts'),
+            remotestorage_calendars=(
+                'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'),
         )
 
     def __getitem__(self, name):

vdirsyncer/storage/remotestorage.py

@@ -0,0 +1,275 @@
+'''
+A storage type for accessing contact and calendar data from `remoteStorage
+<https://remotestorage.io>`_. It is highly experimental.
+
+A few things are hardcoded for now so the user doesn't have to specify those
+things, and plugging in an account "just works".
+
+We also use a custom ``data``-URI for the redirect in OAuth:
+
+- There is no server that could be compromised.
+- With a proper URL, ``access_token`` would be stored in the browser history.
+  For some reason Firefox doesn't do that with ``data``-URIs.
+- ``data``-URIs have no clear domain name that could prevent from phishing
+  attacks. However, I don't see a way to phish without compromising the
+  vdirsyncer installation, at which point any hope would already be lost.
+- On the downside, redirect URIs are monstrous.
+
+'''
+
+import click
+
+from .base import Item, Storage
+from .http import HTTP_STORAGE_PARAMETERS, USERAGENT, prepare_client_cert, \
+    prepare_verify
+from .. import exceptions, log, utils
+
+REDIRECT_URI = 'https://vdirsyncer.5apps.com/'
+CLIENT_ID = 'https://vdirsyncer.5apps.com'
+DRAFT_VERSION = '05'
+
+logger = log.get(__name__)
+
+urljoin = utils.compat.urlparse.urljoin
+urlquote = utils.compat.urlquote
+
+
+def _ensure_slash(dir):
+    return dir.rstrip('/') + '/'
+
+
+def _iter_listing(json):
+    new_listing = '@context' in json  # draft-02 and beyond
+    if new_listing:
+        json = json['items']
+    for name, info in utils.compat.iteritems(json):
+        if not new_listing:
+            info = {'ETag': info}
+        yield name, info
+
+
+class Session(object):
+
+    def __init__(self, account, scope, verify=True, verify_fingerprint=None,
+                 auth_cert=None, access_token=None, collection=None):
+        from oauthlib.oauth2 import MobileApplicationClient
+        from requests_oauthlib import OAuth2Session
+
+        self.user, self.host = account.split('@')
+
+        self._settings = {
+            'cert': prepare_client_cert(auth_cert)
+        }
+        self._settings.update(prepare_verify(verify, verify_fingerprint))
+
+        self.scope = scope + ':rw'
+        self._session = OAuth2Session(
+            CLIENT_ID, client=MobileApplicationClient(CLIENT_ID),
+            scope=self.scope,
+            redirect_uri=REDIRECT_URI,
+            token={'access_token': access_token},
+        )
+
+        subpath = scope
+        if collection:
+            subpath = urljoin(_ensure_slash(scope),
+                              _ensure_slash(urlquote(collection)))
+
+        self._discover_endpoints(subpath)
+
+        if not access_token:
+            self._get_access_token()
+
+    def request(self, method, path, **kwargs):
+        url = self.endpoints['storage']
+        if path:
+            url = urljoin(url, path)
+
+        settings = dict(self._settings)
+        settings.update(kwargs)
+
+        return utils.http.request(method, url,
+                                  session=self._session, **settings)
+
+    def _get_access_token(self):
+        authorization_url, state = \
+            self._session.authorization_url(self.endpoints['oauth'])
+
+        click.echo('Go to {}'.format(authorization_url))
+        click.echo('Follow the instructions on the page.')
+        raise exceptions.UserError('Aborted!')
+
+    def _discover_endpoints(self, subpath):
+        r = utils.http.request(
+            'GET', 'https://{host}/.well-known/webfinger?resource=acct:{user}'
+            .format(host=self.host, user=self.user),
+            **self._settings
+        )
+        j = r.json()
+        for link in j['links']:
+            if 'remotestorage' in link['rel']:
+                break
+
+        storage = urljoin(_ensure_slash(link['href']),
+                          _ensure_slash(subpath))
+        props = link['properties']
+        oauth = props['http://tools.ietf.org/html/rfc6749#section-4.2']
+        self.endpoints = dict(storage=storage, oauth=oauth)
+
+
+class RemoteStorage(Storage):
+    __doc__ = '''
+    :param account: remoteStorage account, ``"user@example.com"``.
+    ''' + HTTP_STORAGE_PARAMETERS + '''
+    '''
+
+    storage_name = None
+    item_mimetype = None
+    fileext = None
+
+    def __init__(self, account, verify=True, verify_fingerprint=None,
+                 auth_cert=None, access_token=None, **kwargs):
+        super(RemoteStorage, self).__init__(**kwargs)
+        self.session = Session(
+            account=account,
+            verify=verify,
+            verify_fingerprint=verify_fingerprint,
+            auth_cert=auth_cert,
+            access_token=access_token,
+            collection=self.collection,
+            scope=self.scope)
+
+    @classmethod
+    def discover(cls, **base_args):
+        if base_args.pop('collection', None) is not None:
+            raise TypeError('collection argument must not be given.')
+
+        session_args, _ = utils.split_dict(base_args, lambda key: key in (
+            'account', 'verify', 'auth', 'verify_fingerprint', 'auth_cert',
+            'access_token'
+        ))
+
+        session = Session(scope=cls.scope, **session_args)
+
+        try:
+            r = session.request('GET', '')
+        except exceptions.NotFoundError:
+            return
+
+        for name, info in _iter_listing(r.json()):
+            if not name.endswith('/'):
+                continue  # not a folder
+
+            newargs = dict(base_args)
+            newargs['collection'] = name.rstrip('/')
+            yield newargs
+
+    @classmethod
+    def create_collection(cls, collection, **kwargs):
+        # remoteStorage folders are autocreated.
+        assert collection
+        assert '/' not in collection
+        kwargs['collection'] = collection
+        return kwargs
+
+    def list(self):
+        try:
+            r = self.session.request('GET', '')
+        except exceptions.NotFoundError:
+            return
+
+        for name, info in _iter_listing(r.json()):
+            if not name.endswith(self.fileext):
+                continue
+
+            etag = info['ETag']
+            etag = '"' + etag + '"'
+            yield name, etag
+
+    def _put(self, href, item, etag):
+        headers = {'Content-Type': self.item_mimetype + '; charset=UTF-8'}
+        if etag is None:
+            headers['If-None-Match'] = '*'
+        else:
+            headers['If-Match'] = etag
+
+        response = self.session.request(
+            'PUT',
+            href,
+            data=item.raw.encode('utf-8'),
+            headers=headers
+        )
+        if not response.url.endswith('/' + href):
+            raise exceptions.InvalidResponse('spec doesn\'t allow redirects')
+        return href, response.headers['etag']
+
+    def update(self, href, item, etag):
+        assert etag
+        href, etag = self._put(href, item, etag)
+        return etag
+
+    def upload(self, item):
+        href = utils.generate_href(item.ident)
+        href = utils.compat.urlquote(href, '@') + self.fileext
+        return self._put(href, item, None)
+
+    def delete(self, href, etag):
+        headers = {'If-Match': etag}
+        self.session.request('DELETE', href, headers=headers)
+
+    def get(self, href):
+        response = self.session.request('GET', href)
+        return Item(response.text), response.headers['etag']
+
+    def get_meta(self, key):
+        try:
+            return self.session.request('GET', key).text or None
+        except exceptions.NotFoundError:
+            pass
+
+    def set_meta(self, key, value):
+        self.session.request(
+            'PUT',
+            key,
+            data=(value or u'').encode('utf-8'),
+            headers={'Content-Type': 'text/plain; charset=utf-8'}
+        )
+
+
+class RemoteStorageContacts(RemoteStorage):
+    __doc__ = '''
+    remoteStorage contacts. Uses the `vdir_contacts` scope.
+    ''' + RemoteStorage.__doc__
+
+    storage_name = 'remotestorage_contacts'
+    fileext = '.vcf'
+    item_mimetype = 'text/vcard'
+    scope = 'vdir_contacts'
+
+    def __init__(self, **kwargs):
+        if kwargs.get('collection'):
+            raise ValueError(
+                'No collections allowed for contacts, '
+                'there is only one addressbook. '
+                'Use the vcard groups construct to categorize your contacts '
+                'into groups.'
+            )
+
+        super(RemoteStorageContacts, self).__init__(**kwargs)
+
+
+class RemoteStorageCalendars(RemoteStorage):
+    __doc__ = '''
+    remoteStorage calendars. Uses the `vdir_calendars` scope.
+    ''' + RemoteStorage.__doc__
+
+    storage_name = 'remotestorage_calendars'
+    fileext = '.ics'
+    item_mimetype = 'text/icalendar'
+    scope = 'vdir_calendars'
+
+    def __init__(self, **kwargs):
+        if not kwargs.get('collection'):
+            raise ValueError('The collections parameter is required.')
+
+        super(RemoteStorageCalendars, self).__init__(**kwargs)