Auto merge of #399 - pimutils:google, r=untitaker

Google contacts and calendar

- [x] Proper documentation
- [x] With the refactor, `get_storage_init_args` now misses a lot of arguments for all DAV and Google storage types

Fix #8

CHANGELOG.rst

@@ -9,6 +9,12 @@ Package maintainers and users who have to manually update their installation
 may want to subscribe to `GitHub's tag feed
 <https://github.com/pimutils/vdirsyncer/tags.atom>`_.
 
+Version 0.10.0
+==============
+
+- New storage types :storage:`google_calendar` and :storage:`google_contacts`
+  have been added.
+
 Version 0.9.3
 =============
 

docs/config.rst

@@ -116,13 +116,6 @@ Storage Section
 Supported Storages
 ------------------
 
-Read-write storages
-~~~~~~~~~~~~~~~~~~~
-
-These storages generally support reading and changing of their items. Their
-default value for ``read_only`` is ``false``, but can be set to ``true`` if
-wished.
-
 CalDAV and CardDAV
 ++++++++++++++++++
 
@@ -130,6 +123,20 @@ CalDAV and CardDAV
 
 .. autostorage:: vdirsyncer.storage.dav.CarddavStorage
 
+Google
+++++++
+
+At first run you will be asked to authorize application for google account
+access.
+
+To use this storage type, you need to install some additional dependencies::
+
+    pip install vdirsyncer[google]
+
+.. autostorage:: vdirsyncer.storage.google.GoogleCalendarStorage
+
+.. autostorage:: vdirsyncer.storage.google.GoogleContactsStorage
+
 remoteStorage
 +++++++++++++
 
@@ -157,7 +164,7 @@ Local
 
 
 Read-only storages
-~~~~~~~~~~~~~~~~~~
+++++++++++++++++++
 
 These storages don't support writing of their items, consequently ``read_only``
 is set to ``true`` by default. Changing ``read_only`` to ``false`` on them

docs/supported.rst

@@ -207,5 +207,25 @@ Vdirsyncer is continuously tested against the latest version of Baikal_.
 Google
 ------
 
-Vdirsyncer doesn't currently support Google accounts fully. For possible
+Using vdirsyncer with Google Calendar is possible, but it is not tested
-solutions see :gh:`202` and :gh:`8`.
+frequently.
+
+::
+
+    [storage cal]
+    type = google_calendar
+    token_file = ~/.google-token
+
+    [storage card]
+    type = google_contacts
+    token_file = ~/.google-token
+
+At first run you will be asked to authorize application for google account
+access. Simply follow the instructions.
+
+- Google's CardDav implementation is very limited, may lead to data loss, use
+  with care.
+- You can select which calendars to sync on `CalDav settings page
+  <https://calendar.google.com/calendar/syncselect>`_
+
+For more information see :gh:`202` and :gh:`8`.

setup.py

@@ -80,7 +80,8 @@ setup(
     },
     install_requires=requirements,
     extras_require={
-        'remotestorage': ['requests-oauthlib']
+        'remotestorage': ['requests-oauthlib'],
+        'google': ['requests-oauthlib'],
     },
     cmdclass={
         'minimal_requirements': PrintRequirements

tests/utils/test_main.py

@@ -23,20 +23,10 @@ def no_debug_output(request):
     logger.setLevel(logging.WARNING)
 
 
-def test_get_class_init_args():
+def test_get_storage_init_args():
-    class Foobar(object):
-        def __init__(self, foo, bar, baz=None):
-            pass
-
-    all, required = utils.get_class_init_args(Foobar)
-    assert all == {'foo', 'bar', 'baz'}
-    assert required == {'foo', 'bar'}
-
-
-def test_get_class_init_args_on_storage():
     from vdirsyncer.storage.memory import MemoryStorage
 
-    all, required = utils.get_class_init_args(MemoryStorage)
+    all, required = utils.get_storage_init_args(MemoryStorage)
     assert all == set(['fileext', 'collection', 'read_only', 'instance_name'])
     assert not required
 

vdirsyncer/cli/utils.py

@@ -17,7 +17,7 @@ import click_threading
 from . import cli_logger
 from .. import DOCS_HOME, exceptions
 from ..sync import IdentConflict, StorageEmpty, SyncConflict
-from ..utils import expand_path, get_class_init_args
+from ..utils import expand_path, get_storage_init_args
 from ..utils.compat import to_native
 
 try:
@@ -46,6 +46,8 @@ class _StorageIndex(object):
                 'vdirsyncer.storage.remotestorage.RemoteStorageContacts'),
             remotestorage_calendars=(
                 'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'),
+            google_calendar='vdirsyncer.storage.google.GoogleCalendarStorage',
+            google_contacts='vdirsyncer.storage.google.GoogleContactsStorage'
         )
 
     def __getitem__(self, name):
@@ -380,7 +382,7 @@ def handle_storage_init_error(cls, config):
     if isinstance(e, (click.Abort, exceptions.UserError, KeyboardInterrupt)):
         raise
 
-    all, required = get_class_init_args(cls)
+    all, required = get_storage_init_args(cls)
     given = set(config)
     missing = required - given
     invalid = given - all
@@ -488,9 +490,9 @@ def format_storage_config(cls, header=True):
     yield 'type = {}'.format(cls.storage_name)
 
     from ..storage.base import Storage
-    from ..utils import get_class_init_specs
+    from ..utils import get_storage_init_specs
     handled = set()
-    for spec in get_class_init_specs(cls, stop_at=Storage):
+    for spec in get_storage_init_specs(cls, stop_at=Storage):
         defaults = spec.defaults or ()
         defaults = dict(zip(spec.args[-len(defaults):], defaults))
         for key in spec.args[1:]:

vdirsyncer/storage/dav.py

@@ -12,7 +12,7 @@ from .base import Item, Storage, normalize_meta_value
 from .http import HTTP_STORAGE_PARAMETERS, USERAGENT, prepare_auth, \
     prepare_client_cert, prepare_verify
 from .. import exceptions, utils
-from ..utils.compat import text_type, to_native
+from ..utils.compat import PY2, getargspec_ish, text_type, to_native
 
 
 dav_logger = logging.getLogger(__name__)
@@ -103,11 +103,6 @@ def _fuzzy_matches_mimetype(strict, weak):
     return False
 
 
-def _get_collection_from_url(url):
-    _, collection = url.rstrip('/').rsplit('/', 1)
-    return utils.compat.urlunquote(collection)
-
-
 class Discover(object):
     _namespace = None
     _resourcetype = None
@@ -122,18 +117,18 @@ class Discover(object):
     </d:propfind>
     """
 
-    def __init__(self, **kwargs):
+    def __init__(self, session, kwargs):
         if kwargs.pop('collection', None) is not None:
             raise TypeError('collection argument must not be given.')
 
-        discover_args, _ = utils.split_dict(kwargs, lambda key: key in (
+        self.session = session
-            'url', 'username', 'password', 'verify', 'auth', 'useragent',
-            'verify_fingerprint', 'auth_cert',
-        ))
-
-        self.session = DavSession(**discover_args)
         self.kwargs = kwargs
 
+    @staticmethod
+    def _get_collection_from_url(url):
+        _, collection = url.rstrip('/').rsplit('/', 1)
+        return utils.compat.urlunquote(collection)
+
     def find_dav(self):
         try:
             response = self.session.request(
@@ -212,14 +207,14 @@ class Discover(object):
     def discover(self):
         for c in self.find_collections():
             url = c['href']
-            collection = _get_collection_from_url(url)
+            collection = self._get_collection_from_url(url)
             storage_args = dict(self.kwargs)
             storage_args.update({'url': url, 'collection': collection})
             yield storage_args
 
     def create(self, collection):
         if collection is None:
-            collection = _get_collection_from_url(self.kwargs['url'])
+            collection = self._get_collection_from_url(self.kwargs['url'])
 
         for c in self.discover():
             if c['collection'] == collection:
@@ -297,26 +292,36 @@ class DavSession(object):
     A helper class to connect to DAV servers.
     '''
 
+    @classmethod
+    def init_and_remaining_args(cls, **kwargs):
+        argspec = getargspec_ish(cls.__init__)
+        self_args, remainder = \
+            utils.split_dict(kwargs, argspec.args.__contains__)
+
+        return cls(**self_args), remainder
+
     def __init__(self, url, username='', password='', verify=True, auth=None,
                  useragent=USERAGENT, verify_fingerprint=None,
                  auth_cert=None):
         self._settings = {
-            'auth': prepare_auth(auth, username, password),
             'cert': prepare_client_cert(auth_cert),
+            'auth': prepare_auth(auth, username, password)
         }
         self._settings.update(prepare_verify(verify, verify_fingerprint))
 
         self.useragent = useragent
         self.url = url.rstrip('/') + '/'
-        self.parsed_url = utils.compat.urlparse.urlparse(self.url)
+
-        self._session = None
+        self._session = requests.session()
+
+    @utils.cached_property
+    def parsed_url(self):
+        return utils.compat.urlparse.urlparse(self.url)
 
     def request(self, method, path, **kwargs):
         url = self.url
         if path:
             url = utils.compat.urlparse.urljoin(self.url, path)
-        if self._session is None:
-            self._session = requests.session()
 
         more = dict(self._settings)
         more.update(kwargs)
@@ -334,8 +339,6 @@ class DavStorage(Storage):
     __doc__ = '''
     :param url: Base URL or an URL to a collection.
     ''' + HTTP_STORAGE_PARAMETERS + '''
-    :param unsafe_href_chars: Replace the given characters when generating
-        hrefs. Defaults to ``'@'``.
 
     .. note::
 
@@ -353,36 +356,38 @@ class DavStorage(Storage):
     get_multi_data_query = None
     # The Discover subclass to use
     discovery_class = None
+    # The DavSession class to use
+    session_class = DavSession
 
-    _session = None
     _repr_attributes = ('username', 'url')
 
     _property_table = {
         'displayname': ('displayname', 'DAV:'),
     }
 
-    def __init__(self, url, username='', password='', verify=True, auth=None,
+    def __init__(self, **kwargs):
-                 useragent=USERAGENT, verify_fingerprint=None, auth_cert=None,
+        # defined for _repr_attributes
-                 **kwargs):
+        self.username = kwargs.get('username')
+        self.url = kwargs.get('url')
+
+        self.session, kwargs = \
+            self.session_class.init_and_remaining_args(**kwargs)
         super(DavStorage, self).__init__(**kwargs)
 
-        url = url.rstrip('/') + '/'
+    if not PY2:
-        self.session = DavSession(url, username, password, verify, auth,
+        import inspect
-                                  useragent, verify_fingerprint,
+        __init__.__signature__ = inspect.signature(session_class.__init__)
-                                  auth_cert)
-
-        # defined for _repr_attributes
-        self.username = username
-        self.url = url
 
     @classmethod
     def discover(cls, **kwargs):
-        d = cls.discovery_class(**kwargs)
+        session, _ = cls.session_class.init_and_remaining_args(**kwargs)
+        d = cls.discovery_class(session, kwargs)
         return d.discover()
 
     @classmethod
     def create_collection(cls, collection, **kwargs):
-        d = cls.discovery_class(**kwargs)
+        session, _ = cls.session_class.init_and_remaining_args(**kwargs)
+        d = cls.discovery_class(session, kwargs)
         return d.create(collection)
 
     def _normalize_href(self, *args, **kwargs):

vdirsyncer/storage/google.py

@@ -0,0 +1,158 @@
+# -*- coding: utf-8 -*-
+
+import json
+import logging
+
+import click
+
+from . import dav
+from .. import exceptions, utils
+
+logger = logging.getLogger(__name__)
+
+
+TOKEN_URL = 'https://accounts.google.com/o/oauth2/v2/auth'
+REFRESH_URL = 'https://www.googleapis.com/oauth2/v4/token'
+
+CLIENT_ID = ('155040592229-mth5eiq7qt9dtk46j0vcnoometuab9mb'
+             '.apps.googleusercontent.com')
+CLIENT_SECRET = 'flVXF7jB2A2-YC4rg2sUCCX1'
+
+try:
+    from requests_oauthlib import OAuth2Session
+    have_oauth2 = True
+except ImportError:
+    have_oauth2 = False
+
+
+class GoogleSession(dav.DavSession):
+    def __init__(self, token_file, client_id=None, client_secret=None):
+        # Not a default in function signature, otherwise these show up in user
+        # documentation
+        client_id = client_id or CLIENT_ID
+        client_secret = client_secret or CLIENT_SECRET
+
+        self.useragent = client_id
+        self._settings = {}
+
+        if not have_oauth2:
+            raise exceptions.UserError('requests-oauthlib not installed')
+
+        token = None
+        try:
+            with open(token_file) as f:
+                token = json.load(f)
+        except OSError:
+            pass
+
+        def _save_token(token):
+            with open(token_file, 'w') as f:
+                json.dump(token, f)
+
+        self._session = OAuth2Session(
+            client_id=client_id,
+            token=token,
+            redirect_uri='urn:ietf:wg:oauth:2.0:oob',
+            scope=self.scope,
+            auto_refresh_url=REFRESH_URL,
+            auto_refresh_kwargs={
+                'client_id': client_id,
+                'client_secret': client_secret,
+            },
+            token_updater=_save_token
+        )
+
+        if not token:
+            authorization_url, state = self._session.authorization_url(
+                TOKEN_URL,
+                # access_type and approval_prompt are Google specific
+                # extra parameters.
+                access_type='offline', approval_prompt='force')
+            click.echo('Opening {} ...'.format(authorization_url))
+            try:
+                utils.open_graphical_browser(authorization_url)
+            except Exception as e:
+                logger.warning(str(e))
+
+            click.echo("Follow the instructions on the page.")
+            code = click.prompt("Paste obtained code")
+            token = self._session.fetch_token(
+                REFRESH_URL,
+                code=code,
+                # Google specific extra parameter used for client
+                # authentication
+                client_secret=client_secret,
+            )
+            # FIXME: Ugly
+            _save_token(token)
+
+
+GOOGLE_PARAMS_DOCS = '''
+    :param token_file: A filepath where access tokens are stored.
+    :param client_id/client_secret: OAuth credentials. Hardcoded ones are
+        provided, you shouldn't need this unless you hit API rate limits.
+'''
+
+
+class GoogleCalendarStorage(dav.CaldavStorage):
+    __doc__ = '''Google calendar.
+
+    Please refer to :storage:`caldav` regarding
+    the ``item_types`` and timerange parameters.
+    ''' + GOOGLE_PARAMS_DOCS
+
+    class session_class(GoogleSession):
+        url = 'https://apidata.googleusercontent.com/caldav/v2/'
+        scope = ['https://www.googleapis.com/auth/calendar']
+
+    class discovery_class(dav.CalDiscover):
+        @staticmethod
+        def _get_collection_from_url(url):
+            # Google CalDAV has collection URLs like:
+            # /user/foouser/calendars/foocalendar/events/
+            parts = url.rstrip('/').split('/')
+            parts.pop()
+            collection = parts.pop()
+            return utils.compat.urlunquote(collection)
+
+    storage_name = 'google_calendar'
+
+    def __init__(self, token_file, client_id=None, client_secret=None,
+                 start_date=None, end_date=None, item_types=(), **kwargs):
+        super(GoogleContactsStorage, self).__init__(
+            token_file=token_file, client_id=client_id,
+            client_secret=client_secret, start_date=start_date,
+            end_date=end_date, item_types=item_types,
+            **kwargs
+        )
+
+    # This is ugly: We define/override the entire signature computed for the
+    # docs here because the current way we autogenerate those docs are too
+    # simple for our advanced argspec juggling in `vdirsyncer.storage.dav`.
+    __init__._traverse_superclass = False
+
+
+class GoogleContactsStorage(dav.CarddavStorage):
+    __doc__ = '''Google contacts.
+    ''' + GOOGLE_PARAMS_DOCS
+
+    class session_class(GoogleSession):
+        # Apparently Google wants us to submit a PROPFIND to the well-known
+        # URL, instead of looking for a redirect.
+        url = 'https://www.googleapis.com/.well-known/carddav/'
+        scope = ['https://www.googleapis.com/auth/carddav']
+
+    storage_name = 'google_contacts'
+
+    def __init__(self, token_file, client_id=None, client_secret=None,
+                 **kwargs):
+        super(GoogleContactsStorage, self).__init__(
+            token_file=token_file, client_id=client_id,
+            client_secret=client_secret,
+            **kwargs
+        )
+
+    # This is ugly: We define/override the entire signature computed for the
+    # docs here because the current way we autogenerate those docs are too
+    # simple for our advanced argspec juggling in `vdirsyncer.storage.dav`.
+    __init__._traverse_superclass = False

vdirsyncer/utils/__init__.py

@@ -82,16 +82,22 @@ def get_etag_from_fileobject(f):
     return get_etag_from_file(f.name)
 
 
-def get_class_init_specs(cls, stop_at=object):
+def get_storage_init_specs(cls, stop_at=object):
     if cls is stop_at:
         return ()
+
     spec = getargspec_ish(cls.__init__)
-    supercls = next(getattr(x.__init__, '__objclass__', x)
+    if getattr(cls.__init__, '_traverse_superclass', True):
-                    for x in cls.__mro__[1:])
+        supercls = next(getattr(x.__init__, '__objclass__', x)
-    return (spec,) + get_class_init_specs(supercls, stop_at=stop_at)
+                        for x in cls.__mro__[1:])
+        superspecs = get_storage_init_specs(supercls, stop_at=stop_at)
+    else:
+        superspecs = ()
+
+    return (spec,) + superspecs
 
 
-def get_class_init_args(cls, stop_at=object):
+def get_storage_init_args(cls, stop_at=object):
     '''
     Get args which are taken during class initialization. Assumes that all
     classes' __init__ calls super().__init__ with the rest of the arguments.
@@ -102,7 +108,7 @@ def get_class_init_args(cls, stop_at=object):
         requires.
     '''
     all, required = set(), set()
-    for spec in get_class_init_specs(cls, stop_at=stop_at):
+    for spec in get_storage_init_specs(cls, stop_at=stop_at):
         all.update(spec.args[1:])
         required.update(spec.args[1:-len(spec.defaults or ())])