From 2888757e1b303a1238ee1fe75e919f0c9a863f64 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 1 Apr 2016 20:48:13 +0200 Subject: [PATCH] Refactor Google support into own storage types --- docs/config.rst | 23 +++-- docs/supported.rst | 17 ++-- setup.py | 2 +- tests/utils/test_main.py | 14 +--- vdirsyncer/cli/utils.py | 10 ++- vdirsyncer/storage/dav.py | 99 +++++----------------- vdirsyncer/storage/google.py | 158 +++++++++++++++++++++++++++++++++++ vdirsyncer/storage/http.py | 5 +- vdirsyncer/utils/__init__.py | 18 ++-- 9 files changed, 221 insertions(+), 125 deletions(-) create mode 100644 vdirsyncer/storage/google.py diff --git a/docs/config.rst b/docs/config.rst index c3f8097..8c3ae1e 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -116,13 +116,6 @@ Storage Section Supported Storages ------------------ -Read-write storages -~~~~~~~~~~~~~~~~~~~ - -These storages generally support reading and changing of their items. Their -default value for ``read_only`` is ``false``, but can be set to ``true`` if -wished. - CalDAV and CardDAV ++++++++++++++++++ @@ -130,6 +123,20 @@ CalDAV and CardDAV .. autostorage:: vdirsyncer.storage.dav.CarddavStorage +Google +++++++ + +At first run you will be asked to authorize application for google account +access. + +To use this storage type, you need to install some additional dependencies:: + + pip install vdirsyncer[google] + +.. autostorage:: vdirsyncer.storage.google.GoogleCalendarStorage + +.. autostorage:: vdirsyncer.storage.google.GoogleContactsStorage + remoteStorage +++++++++++++ @@ -157,7 +164,7 @@ Local Read-only storages -~~~~~~~~~~~~~~~~~~ +++++++++++++++++++ These storages don't support writing of their items, consequently ``read_only`` is set to ``true`` by default. Changing ``read_only`` to ``false`` on them diff --git a/docs/supported.rst b/docs/supported.rst index cfe46b9..e71d02d 100644 --- a/docs/supported.rst +++ b/docs/supported.rst @@ -213,22 +213,19 @@ frequently. :: [storage cal] - type = caldav - url = https://apidata.googleusercontent.com/caldav/v2/ - auth = oauth2_google + type = google_calendar + token_file = ~/.google-token [storage card] - type = carddav - url = https://www.googleapis.com/carddav/v1/principals/EMAIL/lists/default - auth = oauth2_google + type = google_contacts + token_file = ~/.google-token At first run you will be asked to authorize application for google account -access. Simply follow the instructions. You'll be asked to modify configuration -file (save `refresh_token` as a password). +access. Simply follow the instructions. - Google's CardDav implementation is very limited, may lead to data loss, use with care. -- You can select which calendars to sync on - `CalDav settings page `_ +- You can select which calendars to sync on `CalDav settings page + `_ For more information see :gh:`202` and :gh:`8`. diff --git a/setup.py b/setup.py index f7a558e..de1b219 100644 --- a/setup.py +++ b/setup.py @@ -81,7 +81,7 @@ setup( install_requires=requirements, extras_require={ 'remotestorage': ['requests-oauthlib'], - 'oauth2': ['requests-oauthlib'], + 'google': ['requests-oauthlib'], }, cmdclass={ 'minimal_requirements': PrintRequirements diff --git a/tests/utils/test_main.py b/tests/utils/test_main.py index 2c63507..1790f2a 100644 --- a/tests/utils/test_main.py +++ b/tests/utils/test_main.py @@ -23,20 +23,10 @@ def no_debug_output(request): logger.setLevel(logging.WARNING) -def test_get_class_init_args(): - class Foobar(object): - def __init__(self, foo, bar, baz=None): - pass - - all, required = utils.get_class_init_args(Foobar) - assert all == {'foo', 'bar', 'baz'} - assert required == {'foo', 'bar'} - - -def test_get_class_init_args_on_storage(): +def test_get_storage_init_args(): from vdirsyncer.storage.memory import MemoryStorage - all, required = utils.get_class_init_args(MemoryStorage) + all, required = utils.get_storage_init_args(MemoryStorage) assert all == set(['fileext', 'collection', 'read_only', 'instance_name']) assert not required diff --git a/vdirsyncer/cli/utils.py b/vdirsyncer/cli/utils.py index 97d4800..6e49d6a 100644 --- a/vdirsyncer/cli/utils.py +++ b/vdirsyncer/cli/utils.py @@ -17,7 +17,7 @@ import click_threading from . import cli_logger from .. import DOCS_HOME, exceptions from ..sync import IdentConflict, StorageEmpty, SyncConflict -from ..utils import expand_path, get_class_init_args +from ..utils import expand_path, get_storage_init_args from ..utils.compat import to_native try: @@ -46,6 +46,8 @@ class _StorageIndex(object): 'vdirsyncer.storage.remotestorage.RemoteStorageContacts'), remotestorage_calendars=( 'vdirsyncer.storage.remotestorage.RemoteStorageCalendars'), + google_calendar='vdirsyncer.storage.google.GoogleCalendarStorage', + google_contacts='vdirsyncer.storage.google.GoogleContactsStorage' ) def __getitem__(self, name): @@ -380,7 +382,7 @@ def handle_storage_init_error(cls, config): if isinstance(e, (click.Abort, exceptions.UserError, KeyboardInterrupt)): raise - all, required = get_class_init_args(cls) + all, required = get_storage_init_args(cls) given = set(config) missing = required - given invalid = given - all @@ -488,9 +490,9 @@ def format_storage_config(cls, header=True): yield 'type = {}'.format(cls.storage_name) from ..storage.base import Storage - from ..utils import get_class_init_specs + from ..utils import get_storage_init_specs handled = set() - for spec in get_class_init_specs(cls, stop_at=Storage): + for spec in get_storage_init_specs(cls, stop_at=Storage): defaults = spec.defaults or () defaults = dict(zip(spec.args[-len(defaults):], defaults)) for key in spec.args[1:]: diff --git a/vdirsyncer/storage/dav.py b/vdirsyncer/storage/dav.py index 8620adc..d8c9fe5 100644 --- a/vdirsyncer/storage/dav.py +++ b/vdirsyncer/storage/dav.py @@ -3,8 +3,6 @@ import datetime import logging -import click - from lxml import etree import requests @@ -14,17 +12,8 @@ from .base import Item, Storage, normalize_meta_value from .http import HTTP_STORAGE_PARAMETERS, USERAGENT, prepare_auth, \ prepare_client_cert, prepare_verify from .. import exceptions, utils -from ..utils.compat import getargspec_ish, text_type, to_native +from ..utils.compat import PY2, getargspec_ish, text_type, to_native -OAUTH2_GOOGLE_TOKEN_URL = 'https://accounts.google.com/o/oauth2/v2/auth' -OAUTH2_GOOGLE_REFRESH_URL = 'https://www.googleapis.com/oauth2/v4/token' -have_oauth2 = True -try: - from requests_oauthlib import OAuth2Session - oauth2_client_id = 'FIXME' - oauth2_client_secret = 'FIXME' -except ImportError: - have_oauth2 = False dav_logger = logging.getLogger(__name__) @@ -114,11 +103,6 @@ def _fuzzy_matches_mimetype(strict, weak): return False -def _get_collection_from_url(url): - _, collection = url.rstrip('/').rsplit('/', 1) - return utils.compat.urlunquote(collection) - - class Discover(object): _namespace = None _resourcetype = None @@ -140,6 +124,11 @@ class Discover(object): self.session = session self.kwargs = kwargs + @staticmethod + def _get_collection_from_url(url): + _, collection = url.rstrip('/').rsplit('/', 1) + return utils.compat.urlunquote(collection) + def find_dav(self): try: response = self.session.request( @@ -218,14 +207,14 @@ class Discover(object): def discover(self): for c in self.find_collections(): url = c['href'] - collection = _get_collection_from_url(url) + collection = self._get_collection_from_url(url) storage_args = dict(self.kwargs) storage_args.update({'url': url, 'collection': collection}) yield storage_args def create(self, collection): if collection is None: - collection = _get_collection_from_url(self.kwargs['url']) + collection = self._get_collection_from_url(self.kwargs['url']) for c in self.discover(): if c['collection'] == collection: @@ -306,7 +295,6 @@ class DavSession(object): @classmethod def init_and_remaining_args(cls, **kwargs): argspec = getargspec_ish(cls.__init__) - argspec.args self_args, remainder = \ utils.split_dict(kwargs, argspec.args.__contains__) @@ -317,73 +305,23 @@ class DavSession(object): auth_cert=None): self._settings = { 'cert': prepare_client_cert(auth_cert), + 'auth': prepare_auth(auth, username, password) } self._settings.update(prepare_verify(verify, verify_fingerprint)) self.useragent = useragent self.url = url.rstrip('/') + '/' - self.parsed_url = utils.compat.urlparse.urlparse(self.url) - self._session = None - self._token = None - self._use_oauth2_google = False - if auth == 'oauth2_google': - if not have_oauth2: - raise exceptions.UserError("requests-oauthlib not installed") - if password: - self._token = { - 'refresh_token': password, - # Will be derived from refresh_token - 'access_token': 'dummy', - 'expires_in': -30 - } - self._use_oauth2_google = True - else: - self._settings['auth'] = prepare_auth(auth, username, password) + + self._session = requests.session() + + @utils.cached_property + def parsed_url(self): + return utils.compat.urlparse.urlparse(self.url) def request(self, method, path, **kwargs): url = self.url if path: url = utils.compat.urlparse.urljoin(self.url, path) - if self._session is None: - self._session = requests.session() - if self._use_oauth2_google: - self._session = OAuth2Session( - client_id=oauth2_client_id, - token=self._token, - redirect_uri='urn:ietf:wg:oauth:2.0:oob', - scope=['https://www.googleapis.com/auth/calendar', - 'https://www.googleapis.com/auth/carddav'], - auto_refresh_url=OAUTH2_GOOGLE_REFRESH_URL, - auto_refresh_kwargs={ - 'client_id': oauth2_client_id, - 'client_secret': oauth2_client_secret, - }, - token_updater=lambda x: None - ) - if not self._token: - authorization_url, state = self._session.authorization_url( - OAUTH2_GOOGLE_TOKEN_URL, - # access_type and approval_prompt are Google specific - # extra parameters. - access_type="offline", approval_prompt="force") - click.echo('Opening {} ...'.format(authorization_url)) - try: - utils.open_graphical_browser(authorization_url) - except Exception as e: - dav_logger.warning(str(e)) - - click.echo("Follow the instructions on the page.") - code = click.prompt("Paste obtained code") - self._token = self._session.fetch_token( - OAUTH2_GOOGLE_REFRESH_URL, - code=code, - # Google specific extra parameter used for client - # authentication - client_secret=oauth2_client_secret, - ) - raise exceptions.UserError( - "Set the following token in a password field: {}". - format(self._token['refresh_token'])) more = dict(self._settings) more.update(kwargs) @@ -401,8 +339,6 @@ class DavStorage(Storage): __doc__ = ''' :param url: Base URL or an URL to a collection. ''' + HTTP_STORAGE_PARAMETERS + ''' - :param unsafe_href_chars: Replace the given characters when generating - hrefs. Defaults to ``'@'``. .. note:: @@ -432,12 +368,15 @@ class DavStorage(Storage): def __init__(self, **kwargs): # defined for _repr_attributes self.username = kwargs.get('username') - self.url = kwargs['url'] + self.url = kwargs.get('url') self.session, kwargs = \ self.session_class.init_and_remaining_args(**kwargs) super(DavStorage, self).__init__(**kwargs) + if not PY2: + import inspect + __init__.__signature__ = inspect.signature(session_class.__init__) @classmethod def discover(cls, **kwargs): diff --git a/vdirsyncer/storage/google.py b/vdirsyncer/storage/google.py new file mode 100644 index 0000000..9aab290 --- /dev/null +++ b/vdirsyncer/storage/google.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +import json +import logging + +import click + +from . import dav +from .. import exceptions, utils + +logger = logging.getLogger(__name__) + + +TOKEN_URL = 'https://accounts.google.com/o/oauth2/v2/auth' +REFRESH_URL = 'https://www.googleapis.com/oauth2/v4/token' + +CLIENT_ID = ('155040592229-mth5eiq7qt9dtk46j0vcnoometuab9mb' + '.apps.googleusercontent.com') +CLIENT_SECRET = 'flVXF7jB2A2-YC4rg2sUCCX1' + +try: + from requests_oauthlib import OAuth2Session + have_oauth2 = True +except ImportError: + have_oauth2 = False + + +class GoogleSession(dav.DavSession): + def __init__(self, token_file, client_id=None, client_secret=None): + # Not a default in function signature, otherwise these show up in user + # documentation + client_id = client_id or CLIENT_ID + client_secret = client_secret or CLIENT_SECRET + + self.useragent = client_id + self._settings = {} + + if not have_oauth2: + raise exceptions.UserError('requests-oauthlib not installed') + + token = None + try: + with open(token_file) as f: + token = json.load(f) + except OSError: + pass + + def _save_token(token): + with open(token_file, 'w') as f: + json.dump(token, f) + + self._session = OAuth2Session( + client_id=client_id, + token=token, + redirect_uri='urn:ietf:wg:oauth:2.0:oob', + scope=self.scope, + auto_refresh_url=REFRESH_URL, + auto_refresh_kwargs={ + 'client_id': client_id, + 'client_secret': client_secret, + }, + token_updater=_save_token + ) + + if not token: + authorization_url, state = self._session.authorization_url( + TOKEN_URL, + # access_type and approval_prompt are Google specific + # extra parameters. + access_type='offline', approval_prompt='force') + click.echo('Opening {} ...'.format(authorization_url)) + try: + utils.open_graphical_browser(authorization_url) + except Exception as e: + logger.warning(str(e)) + + click.echo("Follow the instructions on the page.") + code = click.prompt("Paste obtained code") + token = self._session.fetch_token( + REFRESH_URL, + code=code, + # Google specific extra parameter used for client + # authentication + client_secret=client_secret, + ) + # FIXME: Ugly + _save_token(token) + + +GOOGLE_PARAMS_DOCS = ''' + :param token_file: A filepath where access tokens are stored. + :param client_id/client_secret: OAuth credentials. Hardcoded ones are + provided, you shouldn't need this unless you hit API rate limits. +''' + + +class GoogleCalendarStorage(dav.CaldavStorage): + __doc__ = '''Google calendar. + + Please refer to :storage:`caldav` regarding + the ``item_types`` and timerange parameters. + ''' + GOOGLE_PARAMS_DOCS + + class session_class(GoogleSession): + url = 'https://apidata.googleusercontent.com/caldav/v2/' + scope = ['https://www.googleapis.com/auth/calendar'] + + class discovery_class(dav.CalDiscover): + @staticmethod + def _get_collection_from_url(url): + # Google CalDAV has collection URLs like: + # /user/foouser/calendars/foocalendar/events/ + parts = url.rstrip('/').split('/') + parts.pop() + collection = parts.pop() + return utils.compat.urlunquote(collection) + + storage_name = 'google_calendar' + + def __init__(self, token_file, client_id=None, client_secret=None, + start_date=None, end_date=None, item_types=(), **kwargs): + super(GoogleContactsStorage, self).__init__( + token_file=token_file, client_id=client_id, + client_secret=client_secret, start_date=start_date, + end_date=end_date, item_types=item_types, + **kwargs + ) + + # This is ugly: We define/override the entire signature computed for the + # docs here because the current way we autogenerate those docs are too + # simple for our advanced argspec juggling in `vdirsyncer.storage.dav`. + __init__._traverse_superclass = False + + +class GoogleContactsStorage(dav.CarddavStorage): + __doc__ = '''Google contacts. + ''' + GOOGLE_PARAMS_DOCS + + class session_class(GoogleSession): + # Apparently Google wants us to submit a PROPFIND to the well-known + # URL, instead of looking for a redirect. + url = 'https://www.googleapis.com/.well-known/carddav/' + scope = ['https://www.googleapis.com/auth/carddav'] + + storage_name = 'google_contacts' + + def __init__(self, token_file, client_id=None, client_secret=None, + **kwargs): + super(GoogleContactsStorage, self).__init__( + token_file=token_file, client_id=client_id, + client_secret=client_secret, + **kwargs + ) + + # This is ugly: We define/override the entire signature computed for the + # docs here because the current way we autogenerate those docs are too + # simple for our advanced argspec juggling in `vdirsyncer.storage.dav`. + __init__._traverse_superclass = False diff --git a/vdirsyncer/storage/http.py b/vdirsyncer/storage/http.py index 8d04684..da5ff07 100644 --- a/vdirsyncer/storage/http.py +++ b/vdirsyncer/storage/http.py @@ -82,10 +82,7 @@ HTTP_STORAGE_PARAMETERS = ''' information. :param auth: Optional. Either ``basic``, ``digest`` or ``guess``. Default ``guess``. If you know yours, consider setting it explicitly for - performance. For caldav and carddav, additionaly ``oauth2_google`` is - supported. ``password`` setting should point a file for OAuth2 token - storage (directory must already exists, but file itself will be created - automatically). + performance. :param auth_cert: Optional. Either a path to a certificate with a client certificate and the key or a list of paths to the files with them. :param useragent: Default ``vdirsyncer``. diff --git a/vdirsyncer/utils/__init__.py b/vdirsyncer/utils/__init__.py index a3321a1..8f97c6d 100644 --- a/vdirsyncer/utils/__init__.py +++ b/vdirsyncer/utils/__init__.py @@ -82,16 +82,22 @@ def get_etag_from_fileobject(f): return get_etag_from_file(f.name) -def get_class_init_specs(cls, stop_at=object): +def get_storage_init_specs(cls, stop_at=object): if cls is stop_at: return () + spec = getargspec_ish(cls.__init__) - supercls = next(getattr(x.__init__, '__objclass__', x) - for x in cls.__mro__[1:]) - return (spec,) + get_class_init_specs(supercls, stop_at=stop_at) + if getattr(cls.__init__, '_traverse_superclass', True): + supercls = next(getattr(x.__init__, '__objclass__', x) + for x in cls.__mro__[1:]) + superspecs = get_storage_init_specs(supercls, stop_at=stop_at) + else: + superspecs = () + + return (spec,) + superspecs -def get_class_init_args(cls, stop_at=object): +def get_storage_init_args(cls, stop_at=object): ''' Get args which are taken during class initialization. Assumes that all classes' __init__ calls super().__init__ with the rest of the arguments. @@ -102,7 +108,7 @@ def get_class_init_args(cls, stop_at=object): requires. ''' all, required = set(), set() - for spec in get_class_init_specs(cls, stop_at=stop_at): + for spec in get_storage_init_specs(cls, stop_at=stop_at): all.update(spec.args[1:]) required.update(spec.args[1:-len(spec.defaults or ())])