Refactor Google support into own storage types

docs/config.rst

@@ -116,13 +116,6 @@ Storage Section
 Supported Storages
 ------------------
 
-Read-write storages
-~~~~~~~~~~~~~~~~~~~
-
-These storages generally support reading and changing of their items. Their
-default value for ``read_only`` is ``false``, but can be set to ``true`` if
-wished.
-
 CalDAV and CardDAV
 ++++++++++++++++++
 
@@ -130,6 +123,20 @@ CalDAV and CardDAV
 
 .. autostorage:: vdirsyncer.storage.dav.CarddavStorage
 
+Google
+++++++
+
+At first run you will be asked to authorize application for google account
+access.
+
+To use this storage type, you need to install some additional dependencies::
+
+    pip install vdirsyncer[google]
+
+.. autostorage:: vdirsyncer.storage.google.GoogleCalendarStorage
+
+.. autostorage:: vdirsyncer.storage.google.GoogleContactsStorage
+
 remoteStorage
 +++++++++++++
 
@@ -157,7 +164,7 @@ Local
 
 
 Read-only storages
-~~~~~~~~~~~~~~~~~~
+++++++++++++++++++
 
 These storages don't support writing of their items, consequently ``read_only``
 is set to ``true`` by default. Changing ``read_only`` to ``false`` on them

docs/supported.rst

@@ -213,22 +213,19 @@ frequently.
 ::
 
     [storage cal]
-    type = caldav
+    type = google_calendar
-    url = https://apidata.googleusercontent.com/caldav/v2/
+    token_file = ~/.google-token
-    auth = oauth2_google
 
     [storage card]
-    type = carddav
+    type = google_contacts
-    url = https://www.googleapis.com/carddav/v1/principals/EMAIL/lists/default
+    token_file = ~/.google-token
-    auth = oauth2_google
 
 At first run you will be asked to authorize application for google account
-access. Simply follow the instructions. You'll be asked to modify configuration
+access. Simply follow the instructions.
-file (save `refresh_token` as a password).
 
 - Google's CardDav implementation is very limited, may lead to data loss, use
   with care.
-- You can select which calendars to sync on
+- You can select which calendars to sync on `CalDav settings page
-  `CalDav settings page <https://calendar.google.com/calendar/syncselect>`_
+  <https://calendar.google.com/calendar/syncselect>`_
 
 For more information see :gh:`202` and :gh:`8`.

setup.py

@@ -81,7 +81,7 @@ setup(
     install_requires=requirements,
     extras_require={
         'remotestorage': ['requests-oauthlib'],
-        'oauth2': ['requests-oauthlib'],
+        'google': ['requests-oauthlib'],
     },
     cmdclass={
         'minimal_requirements': PrintRequirements

tests/utils/test_main.py

@@ -23,20 +23,10 @@ def no_debug_output(request):
     logger.setLevel(logging.WARNING)
 
 
-def test_get_class_init_args():
+def test_get_storage_init_args():
-    class Foobar(object):
-        def __init__(self, foo, bar, baz=None):
-            pass
-
-    all, required = utils.get_class_init_args(Foobar)
-    assert all == {'foo', 'bar', 'baz'}
-    assert required == {'foo', 'bar'}
-
-
-def test_get_class_init_args_on_storage():
     from vdirsyncer.storage.memory import MemoryStorage
 
-    all, required = utils.get_class_init_args(MemoryStorage)
+    all, required = utils.get_storage_init_args(MemoryStorage)
     assert all == set(['fileext', 'collection', 'read_only', 'instance_name'])
     assert not required
 

vdirsyncer/cli/utils.py

@@ -17,7 +17,7 @@ import click_threading
 from . import cli_logger
 from .. import DOCS_HOME, exceptions
 from ..sync import IdentConflict, StorageEmpty, SyncConflict
-from ..utils import expand_path, get_class_init_args
+from ..utils import expand_path, get_storage_init_args
 from ..utils.compat import to_native
 
 try:
@@ -46,6 +46,8 @@ class _StorageIndex(object):
                 'vdirsyncer.storage.remotestorage.RemoteStorageContacts'),
             remotestorage_calendars=(
                 'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'),
+            google_calendar='vdirsyncer.storage.google.GoogleCalendarStorage',
+            google_contacts='vdirsyncer.storage.google.GoogleContactsStorage'
         )
 
     def __getitem__(self, name):
@@ -380,7 +382,7 @@ def handle_storage_init_error(cls, config):
     if isinstance(e, (click.Abort, exceptions.UserError, KeyboardInterrupt)):
         raise
 
-    all, required = get_class_init_args(cls)
+    all, required = get_storage_init_args(cls)
     given = set(config)
     missing = required - given
     invalid = given - all
@@ -488,9 +490,9 @@ def format_storage_config(cls, header=True):
     yield 'type = {}'.format(cls.storage_name)
 
     from ..storage.base import Storage
-    from ..utils import get_class_init_specs
+    from ..utils import get_storage_init_specs
     handled = set()
-    for spec in get_class_init_specs(cls, stop_at=Storage):
+    for spec in get_storage_init_specs(cls, stop_at=Storage):
         defaults = spec.defaults or ()
         defaults = dict(zip(spec.args[-len(defaults):], defaults))
         for key in spec.args[1:]:

vdirsyncer/storage/dav.py

@@ -3,8 +3,6 @@
 import datetime
 import logging
 
-import click
-
 from lxml import etree
 
 import requests
@@ -14,17 +12,8 @@ from .base import Item, Storage, normalize_meta_value
 from .http import HTTP_STORAGE_PARAMETERS, USERAGENT, prepare_auth, \
     prepare_client_cert, prepare_verify
 from .. import exceptions, utils
-from ..utils.compat import getargspec_ish, text_type, to_native
+from ..utils.compat import PY2, getargspec_ish, text_type, to_native
 
-OAUTH2_GOOGLE_TOKEN_URL = 'https://accounts.google.com/o/oauth2/v2/auth'
-OAUTH2_GOOGLE_REFRESH_URL = 'https://www.googleapis.com/oauth2/v4/token'
-have_oauth2 = True
-try:
-    from requests_oauthlib import OAuth2Session
-    oauth2_client_id = 'FIXME'
-    oauth2_client_secret = 'FIXME'
-except ImportError:
-    have_oauth2 = False
 
 dav_logger = logging.getLogger(__name__)
 
@@ -114,11 +103,6 @@ def _fuzzy_matches_mimetype(strict, weak):
     return False
 
 
-def _get_collection_from_url(url):
-    _, collection = url.rstrip('/').rsplit('/', 1)
-    return utils.compat.urlunquote(collection)
-
-
 class Discover(object):
     _namespace = None
     _resourcetype = None
@@ -140,6 +124,11 @@ class Discover(object):
         self.session = session
         self.kwargs = kwargs
 
+    @staticmethod
+    def _get_collection_from_url(url):
+        _, collection = url.rstrip('/').rsplit('/', 1)
+        return utils.compat.urlunquote(collection)
+
     def find_dav(self):
         try:
             response = self.session.request(
@@ -218,14 +207,14 @@ class Discover(object):
     def discover(self):
         for c in self.find_collections():
             url = c['href']
-            collection = _get_collection_from_url(url)
+            collection = self._get_collection_from_url(url)
             storage_args = dict(self.kwargs)
             storage_args.update({'url': url, 'collection': collection})
             yield storage_args
 
     def create(self, collection):
         if collection is None:
-            collection = _get_collection_from_url(self.kwargs['url'])
+            collection = self._get_collection_from_url(self.kwargs['url'])
 
         for c in self.discover():
             if c['collection'] == collection:
@@ -306,7 +295,6 @@ class DavSession(object):
     @classmethod
     def init_and_remaining_args(cls, **kwargs):
         argspec = getargspec_ish(cls.__init__)
-        argspec.args
         self_args, remainder = \
             utils.split_dict(kwargs, argspec.args.__contains__)
 
@@ -317,73 +305,23 @@ class DavSession(object):
                  auth_cert=None):
         self._settings = {
             'cert': prepare_client_cert(auth_cert),
+            'auth': prepare_auth(auth, username, password)
         }
         self._settings.update(prepare_verify(verify, verify_fingerprint))
 
         self.useragent = useragent
         self.url = url.rstrip('/') + '/'
-        self.parsed_url = utils.compat.urlparse.urlparse(self.url)
+
-        self._session = None
+        self._session = requests.session()
-        self._token = None
+
-        self._use_oauth2_google = False
+    @utils.cached_property
-        if auth == 'oauth2_google':
+    def parsed_url(self):
-            if not have_oauth2:
+        return utils.compat.urlparse.urlparse(self.url)
-                raise exceptions.UserError("requests-oauthlib not installed")
-            if password:
-                self._token = {
-                    'refresh_token': password,
-                    # Will be derived from refresh_token
-                    'access_token': 'dummy',
-                    'expires_in': -30
-                }
-            self._use_oauth2_google = True
-        else:
-            self._settings['auth'] = prepare_auth(auth, username, password)
 
     def request(self, method, path, **kwargs):
         url = self.url
         if path:
             url = utils.compat.urlparse.urljoin(self.url, path)
-        if self._session is None:
-            self._session = requests.session()
-            if self._use_oauth2_google:
-                self._session = OAuth2Session(
-                    client_id=oauth2_client_id,
-                    token=self._token,
-                    redirect_uri='urn:ietf:wg:oauth:2.0:oob',
-                    scope=['https://www.googleapis.com/auth/calendar',
-                           'https://www.googleapis.com/auth/carddav'],
-                    auto_refresh_url=OAUTH2_GOOGLE_REFRESH_URL,
-                    auto_refresh_kwargs={
-                        'client_id': oauth2_client_id,
-                        'client_secret': oauth2_client_secret,
-                    },
-                    token_updater=lambda x: None
-                )
-                if not self._token:
-                    authorization_url, state = self._session.authorization_url(
-                        OAUTH2_GOOGLE_TOKEN_URL,
-                        # access_type and approval_prompt are Google specific
-                        # extra parameters.
-                        access_type="offline", approval_prompt="force")
-                    click.echo('Opening {} ...'.format(authorization_url))
-                    try:
-                        utils.open_graphical_browser(authorization_url)
-                    except Exception as e:
-                        dav_logger.warning(str(e))
-
-                    click.echo("Follow the instructions on the page.")
-                    code = click.prompt("Paste obtained code")
-                    self._token = self._session.fetch_token(
-                        OAUTH2_GOOGLE_REFRESH_URL,
-                        code=code,
-                        # Google specific extra parameter used for client
-                        # authentication
-                        client_secret=oauth2_client_secret,
-                    )
-                    raise exceptions.UserError(
-                        "Set the following token in a password field: {}".
-                        format(self._token['refresh_token']))
 
         more = dict(self._settings)
         more.update(kwargs)
@@ -401,8 +339,6 @@ class DavStorage(Storage):
     __doc__ = '''
     :param url: Base URL or an URL to a collection.
     ''' + HTTP_STORAGE_PARAMETERS + '''
-    :param unsafe_href_chars: Replace the given characters when generating
-        hrefs. Defaults to ``'@'``.
 
     .. note::
 
@@ -432,12 +368,15 @@ class DavStorage(Storage):
     def __init__(self, **kwargs):
         # defined for _repr_attributes
         self.username = kwargs.get('username')
-        self.url = kwargs['url']
+        self.url = kwargs.get('url')
 
         self.session, kwargs = \
             self.session_class.init_and_remaining_args(**kwargs)
         super(DavStorage, self).__init__(**kwargs)
 
+    if not PY2:
+        import inspect
+        __init__.__signature__ = inspect.signature(session_class.__init__)
 
     @classmethod
     def discover(cls, **kwargs):

vdirsyncer/storage/google.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+
+import click
+
+from . import dav
+from .. import exceptions, utils
+
+logger = logging.getLogger(__name__)
+
+
+TOKEN_URL = 'https://accounts.google.com/o/oauth2/v2/auth'
+REFRESH_URL = 'https://www.googleapis.com/oauth2/v4/token'
+
+CLIENT_ID = ('155040592229-mth5eiq7qt9dtk46j0vcnoometuab9mb'
+             '.apps.googleusercontent.com')
+CLIENT_SECRET = 'flVXF7jB2A2-YC4rg2sUCCX1'
+
+try:
+    from requests_oauthlib import OAuth2Session
+    have_oauth2 = True
+except ImportError:
+    have_oauth2 = False
+
+
+class GoogleSession(dav.DavSession):
+    def __init__(self, token_file, client_id=None, client_secret=None):
+        # Not a default in function signature, otherwise these show up in user
+        # documentation
+        client_id = client_id or CLIENT_ID
+        client_secret = client_secret or CLIENT_SECRET
+
+        self.useragent = client_id
+        self._settings = {}
+
+        if not have_oauth2:
+            raise exceptions.UserError('requests-oauthlib not installed')
+
+        token = None
+        try:
+            with open(token_file) as f:
+                token = json.load(f)
+        except OSError:
+            pass
+
+        def _save_token(token):
+            with open(token_file, 'w') as f:
+                json.dump(token, f)
+
+        self._session = OAuth2Session(
+            client_id=client_id,
+            token=token,
+            redirect_uri='urn:ietf:wg:oauth:2.0:oob',
+            scope=self.scope,
+            auto_refresh_url=REFRESH_URL,
+            auto_refresh_kwargs={
+                'client_id': client_id,
+                'client_secret': client_secret,
+            },
+            token_updater=_save_token
+        )
+
+        if not token:
+            authorization_url, state = self._session.authorization_url(
+                TOKEN_URL,
+                # access_type and approval_prompt are Google specific
+                # extra parameters.
+                access_type='offline', approval_prompt='force')
+            click.echo('Opening {} ...'.format(authorization_url))
+            try:
+                utils.open_graphical_browser(authorization_url)
+            except Exception as e:
+                logger.warning(str(e))
+
+            click.echo("Follow the instructions on the page.")
+            code = click.prompt("Paste obtained code")
+            token = self._session.fetch_token(
+                REFRESH_URL,
+                code=code,
+                # Google specific extra parameter used for client
+                # authentication
+                client_secret=client_secret,
+            )
+            # FIXME: Ugly
+            _save_token(token)
+
+
+GOOGLE_PARAMS_DOCS = '''
+    :param token_file: A filepath where access tokens are stored.
+    :param client_id/client_secret: OAuth credentials. Hardcoded ones are
+        provided, you shouldn't need this unless you hit API rate limits.
+'''
+
+
+class GoogleCalendarStorage(dav.CaldavStorage):
+    __doc__ = '''Google calendar.
+
+    Please refer to :storage:`caldav` regarding
+    the ``item_types`` and timerange parameters.
+    ''' + GOOGLE_PARAMS_DOCS
+
+    class session_class(GoogleSession):
+        url = 'https://apidata.googleusercontent.com/caldav/v2/'
+        scope = ['https://www.googleapis.com/auth/calendar']
+
+    class discovery_class(dav.CalDiscover):
+        @staticmethod
+        def _get_collection_from_url(url):
+            # Google CalDAV has collection URLs like:
+            # /user/foouser/calendars/foocalendar/events/
+            parts = url.rstrip('/').split('/')
+            parts.pop()
+            collection = parts.pop()
+            return utils.compat.urlunquote(collection)
+
+    storage_name = 'google_calendar'
+
+    def __init__(self, token_file, client_id=None, client_secret=None,
+                 start_date=None, end_date=None, item_types=(), **kwargs):
+        super(GoogleContactsStorage, self).__init__(
+            token_file=token_file, client_id=client_id,
+            client_secret=client_secret, start_date=start_date,
+            end_date=end_date, item_types=item_types,
+            **kwargs
+        )
+
+    # This is ugly: We define/override the entire signature computed for the
+    # docs here because the current way we autogenerate those docs are too
+    # simple for our advanced argspec juggling in `vdirsyncer.storage.dav`.
+    __init__._traverse_superclass = False
+
+
+class GoogleContactsStorage(dav.CarddavStorage):
+    __doc__ = '''Google contacts.
+    ''' + GOOGLE_PARAMS_DOCS
+
+    class session_class(GoogleSession):
+        # Apparently Google wants us to submit a PROPFIND to the well-known
+        # URL, instead of looking for a redirect.
+        url = 'https://www.googleapis.com/.well-known/carddav/'
+        scope = ['https://www.googleapis.com/auth/carddav']
+
+    storage_name = 'google_contacts'
+
+    def __init__(self, token_file, client_id=None, client_secret=None,
+                 **kwargs):
+        super(GoogleContactsStorage, self).__init__(
+            token_file=token_file, client_id=client_id,
+            client_secret=client_secret,
+            **kwargs
+        )
+
+    # This is ugly: We define/override the entire signature computed for the
+    # docs here because the current way we autogenerate those docs are too
+    # simple for our advanced argspec juggling in `vdirsyncer.storage.dav`.
+    __init__._traverse_superclass = False

vdirsyncer/storage/http.py

@@ -82,10 +82,7 @@ HTTP_STORAGE_PARAMETERS = '''
         information.
     :param auth: Optional. Either ``basic``, ``digest`` or ``guess``. Default
         ``guess``. If you know yours, consider setting it explicitly for
-        performance. For caldav and carddav, additionaly ``oauth2_google`` is
+        performance.
-        supported. ``password`` setting should point a file for OAuth2 token
-        storage (directory must already exists, but file itself will be created
-        automatically).
     :param auth_cert: Optional. Either a path to a certificate with a client
         certificate and the key or a list of paths to the files with them.
     :param useragent: Default ``vdirsyncer``.

vdirsyncer/utils/__init__.py

@@ -82,16 +82,22 @@ def get_etag_from_fileobject(f):
     return get_etag_from_file(f.name)
 
 
-def get_class_init_specs(cls, stop_at=object):
+def get_storage_init_specs(cls, stop_at=object):
     if cls is stop_at:
         return ()
+
     spec = getargspec_ish(cls.__init__)
-    supercls = next(getattr(x.__init__, '__objclass__', x)
+    if getattr(cls.__init__, '_traverse_superclass', True):
-                    for x in cls.__mro__[1:])
+        supercls = next(getattr(x.__init__, '__objclass__', x)
-    return (spec,) + get_class_init_specs(supercls, stop_at=stop_at)
+                        for x in cls.__mro__[1:])
+        superspecs = get_storage_init_specs(supercls, stop_at=stop_at)
+    else:
+        superspecs = ()
+
+    return (spec,) + superspecs
 
 
-def get_class_init_args(cls, stop_at=object):
+def get_storage_init_args(cls, stop_at=object):
     '''
     Get args which are taken during class initialization. Assumes that all
     classes' __init__ calls super().__init__ with the rest of the arguments.
@@ -102,7 +108,7 @@ def get_class_init_args(cls, stop_at=object):
         requires.
     '''
     all, required = set(), set()
-    for spec in get_class_init_specs(cls, stop_at=stop_at):
+    for spec in get_storage_init_specs(cls, stop_at=stop_at):
         all.update(spec.args[1:])
         required.update(spec.args[1:-len(spec.defaults or ())])