From 7e8fa89985ccae9138d3d2b25f58c7b7b7b66aff Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 16 May 2014 16:32:44 +0200 Subject: [PATCH] First version of SingleFileStorage --- tests/__init__.py | 3 +- tests/storage/__init__.py | 4 +- tests/storage/test_singlefile.py | 50 +++++++++++ tests/utils/test_vobject.py | 40 +++++---- vdirsyncer/storage/__init__.py | 4 +- vdirsyncer/storage/base.py | 3 +- vdirsyncer/storage/http.py | 24 ++++-- vdirsyncer/storage/singlefile.py | 139 +++++++++++++++++++++++++++++++ vdirsyncer/utils/vobject.py | 36 ++++++++ 9 files changed, 275 insertions(+), 28 deletions(-) create mode 100644 tests/storage/test_singlefile.py create mode 100644 vdirsyncer/storage/singlefile.py diff --git a/tests/__init__.py b/tests/__init__.py index d853917..67706f2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -85,5 +85,4 @@ END:VCALENDAR''' SIMPLE_TEMPLATE = u'''BEGIN:FOO UID:{r} -END:FOO -''' +END:FOO''' diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index 3fc283b..4111ad8 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -10,14 +10,14 @@ import random import pytest -from .. import assert_item_equals +from .. import assert_item_equals, SIMPLE_TEMPLATE import vdirsyncer.exceptions as exceptions from vdirsyncer.storage.base import Item from vdirsyncer.utils import text_type class StorageTests(object): - item_template = u'X-SOMETHING:{r}' + item_template = SIMPLE_TEMPLATE def _create_bogus_item(self, item_template=None): r = random.random() diff --git a/tests/storage/test_singlefile.py b/tests/storage/test_singlefile.py new file mode 100644 index 0000000..6213c18 --- /dev/null +++ b/tests/storage/test_singlefile.py @@ -0,0 +1,50 @@ +# -*- coding: utf-8 -*- +''' + tests.storage.test_singlefile + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' + +import pytest + +from .. import assert_item_equals +from . import StorageTests +from vdirsyncer.storage.singlefile import SingleFileStorage + + +class TestSingleFileStorage(StorageTests): + + storage_class = SingleFileStorage + + @pytest.fixture(autouse=True) + def setup(self, tmpdir): + self._path = str(tmpdir.join('test.txt')) + + def get_storage_args(self, **kwargs): + return dict(path=self._path, wrapper=u'MYWRAPPER') + + def test_discover(self): + '''This test doesn't make any sense here.''' + + def test_discover_collection_arg(self): + '''This test doesn't make any sense here.''' + + def test_collection_arg(self): + '''This test doesn't make any sense here.''' + + def test_update(self): + '''The original testcase tries to fetch with the old href. But this + storage doesn't have real hrefs, so the href might change if the + underlying UID changes. ''' + + s = self._get_storage() + item = self._create_bogus_item() + href, etag = s.upload(item) + assert_item_equals(s.get(href)[0], item) + + new_item = self._create_bogus_item() + s.update(href, new_item, etag) + ((new_href, new_etag),) = s.list() + assert_item_equals(s.get(new_href)[0], new_item) diff --git a/tests/utils/test_vobject.py b/tests/utils/test_vobject.py index 8b0b222..9fed788 100644 --- a/tests/utils/test_vobject.py +++ b/tests/utils/test_vobject.py @@ -7,29 +7,37 @@ :license: MIT, see LICENSE for more details. ''' -from vdirsyncer.utils.vobject import split_collection +from vdirsyncer.utils.vobject import split_collection, join_collection from .. import normalize_item, SIMPLE_TEMPLATE, BARE_EVENT_TEMPLATE +_simple_joined = u'\r\n'.join(( + u'BEGIN:VADDRESSBOOK', + SIMPLE_TEMPLATE.format(r=123), + SIMPLE_TEMPLATE.format(r=345), + SIMPLE_TEMPLATE.format(r=678), + u'END:VADDRESSBOOK' +)) + +_simple_split = [ + SIMPLE_TEMPLATE.format(r=123), + SIMPLE_TEMPLATE.format(r=345), + SIMPLE_TEMPLATE.format(r=678) +] + + def test_split_collection_simple(): - input = u'\r\n'.join(( - u'BEGIN:VADDRESSBOOK', - SIMPLE_TEMPLATE.format(r=123), - SIMPLE_TEMPLATE.format(r=345), - SIMPLE_TEMPLATE.format(r=678), - u'END:VADDRESSBOOK' - )) + given = split_collection(_simple_joined) + assert [normalize_item(item) for item in given] == \ + [normalize_item(item) for item in _simple_split] - given = split_collection(input) - expected = [ - SIMPLE_TEMPLATE.format(r=123), - SIMPLE_TEMPLATE.format(r=345), - SIMPLE_TEMPLATE.format(r=678) - ] - assert set(normalize_item(item) for item in given) == \ - set(normalize_item(item) for item in expected) +def test_join_collection_simple(): + given = join_collection(_simple_split, wrapper=u'VADDRESSBOOK') + print(given) + print(_simple_joined) + assert normalize_item(given) == normalize_item(_simple_joined) def test_split_collection_timezones(): diff --git a/vdirsyncer/storage/__init__.py b/vdirsyncer/storage/__init__.py index 613f33b..34cb364 100644 --- a/vdirsyncer/storage/__init__.py +++ b/vdirsyncer/storage/__init__.py @@ -15,10 +15,12 @@ from .dav import CarddavStorage, CaldavStorage from .filesystem import FilesystemStorage from .http import HttpStorage +from .singlefile import SingleFileStorage storage_names = { 'caldav': CaldavStorage, 'carddav': CarddavStorage, 'filesystem': FilesystemStorage, - 'http': HttpStorage + 'http': HttpStorage, + 'singlefile': SingleFileStorage } diff --git a/vdirsyncer/storage/base.py b/vdirsyncer/storage/base.py index 1f70332..6b78575 100644 --- a/vdirsyncer/storage/base.py +++ b/vdirsyncer/storage/base.py @@ -44,7 +44,8 @@ class Item(object): self.uid = uid self.raw = u'\n'.join(raw) - self.hash = hashlib.sha256(self.raw.encode('utf-8')).hexdigest() + self.hash = hashlib.sha256( + self.raw.strip().encode('utf-8')).hexdigest() self.ident = self.uid or self.hash diff --git a/vdirsyncer/storage/http.py b/vdirsyncer/storage/http.py index d56866d..856ba1c 100644 --- a/vdirsyncer/storage/http.py +++ b/vdirsyncer/storage/http.py @@ -10,6 +10,7 @@ from .base import Item, Storage from ..utils import expand_path, get_password, request, text_type, urlparse from ..utils.vobject import split_collection +from ..exceptions import NotFoundError USERAGENT = 'vdirsyncer' @@ -36,6 +37,7 @@ def prepare_verify(verify): class HttpStorage(Storage): _repr_attributes = ('username', 'url') + _items = None def __init__(self, url, username='', password='', collection=None, verify=True, auth=None, useragent=USERAGENT, **kwargs): @@ -67,7 +69,6 @@ class HttpStorage(Storage): self.url = url self.parsed_url = urlparse.urlparse(self.url) self.collection = collection - self._items = {} def _default_headers(self): return {'User-Agent': self.useragent} @@ -75,13 +76,24 @@ class HttpStorage(Storage): def list(self): r = request('GET', self.url, **self._settings) r.raise_for_status() - self._items.clear() + self._items = {} + rv = [] for item in split_collection(r.text): item = Item(item) - self._items[self._get_href(item)] = item, item.hash + href = self._get_href(item) + etag = item.hash + self._items[href] = item, etag + rv.append((href, etag)) - for href, (item, etag) in self._items.items(): - yield href, etag + # we can't use yield here because we need to populate our + # dict even if the user doesn't exhaust the iterator + return rv def get(self, href): - return self._items[href] + if self._items is None: + self.list() + + try: + return self._items[href] + except KeyError: + raise NotFoundError(href) diff --git a/vdirsyncer/storage/singlefile.py b/vdirsyncer/storage/singlefile.py new file mode 100644 index 0000000..e71ca8c --- /dev/null +++ b/vdirsyncer/storage/singlefile.py @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +''' + vdirsyncer.storage.singlefile + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: (c) 2014 Markus Unterwaditzer + :license: MIT, see LICENSE for more details. +''' + +import os + +from .base import Item, Storage +import vdirsyncer.exceptions as exceptions +import vdirsyncer.log as log +from vdirsyncer.utils import expand_path, safe_write, itervalues +from vdirsyncer.utils.vobject import split_collection, join_collection + +logger = log.get(__name__) + + +class SingleFileStorage(Storage): + '''Save data in single VCALENDAR file, like Orage -- a calendar app for + XFCE -- and Radicale do. Hashes are etags, UIDs or hashes are hrefs. + + This storage has many raceconditions and is very slow.''' + + _repr_attributes = ('path',) + + _write_mode = 'wb' + _append_mode = 'ab' + _read_mode = 'rb' + + _items = None + + def __init__(self, path, wrapper=None, encoding='utf-8', create=True, + collection=None, **kwargs): + super(SingleFileStorage, self).__init__(**kwargs) + path = expand_path(path) + + if collection is not None: + raise ValueError('collection is not a valid argument for {}' + .format(type(self).__name__)) + + if not os.path.isfile(path): + if os.path.exists(path): + raise IOError('{} is not a file.'.format(path)) + if create: + self._write_mode = 'wb+' + self._append_mode = 'ab+' + else: + raise IOError('File {} does not exist. Use create = ' + 'True in your configuration to automatically ' + 'create it, or create it ' + 'yourself.'.format(path)) + + self.path = path + self.encoding = encoding + self.create = create + self.wrapper = wrapper + + def list(self): + self._items = {} + text = None + + try: + with open(self.path, self._read_mode) as f: + text = f.read().decode(self.encoding) + except IOError as e: + import errno + if e.errno != errno.ENOENT or not self.create: # file not found + raise + return () + + rv = [] + for item in split_collection(text): + item = Item(item) + href = self._get_href(item) + etag = item.hash + self._items[href] = item, etag + rv.append((href, etag)) + + # we can't use yield here because we need to populate our + # dict even if the user doesn't exhaust the iterator + return rv + + def get(self, href): + if self._items is None: + self.list() + + try: + return self._items[href] + except KeyError: + raise exceptions.NotFoundError(href) + + def upload(self, item): + href = self._get_href(item) + self.list() + if href in self._items: + raise exceptions.AlreadyExistingError(href) + + self._items[href] = item, item.hash + self._write() + return href, item.hash + + def update(self, href, item, etag): + self.list() + if href not in self._items: + raise exceptions.NotFoundError(href) + + _, actual_etag = self._items[href] + if etag != actual_etag: + raise exceptions.WrongEtagError(etag, actual_etag) + + self._items[href] = item, item.hash + self._write() + return item.hash + + def delete(self, href, etag): + self.list() + if href not in self._items: + raise exceptions.NotFoundError(href) + + _, actual_etag = self._items[href] + if etag != actual_etag: + raise exceptions.WrongEtagError(etag, actual_etag) + + del self._items[href] + self._write() + + def _write(self): + text = join_collection( + (item.raw for item, etag in itervalues(self._items)), + wrapper=self.wrapper + ) + try: + with safe_write(self.path, self._write_mode) as f: + f.write(text.encode(self.encoding)) + finally: + self._items = None diff --git a/vdirsyncer/utils/vobject.py b/vdirsyncer/utils/vobject.py index 55cacb3..4cdaa01 100644 --- a/vdirsyncer/utils/vobject.py +++ b/vdirsyncer/utils/vobject.py @@ -14,6 +14,7 @@ from . import text_type, itervalues def split_collection(text, inline=(u'VTIMEZONE',), wrap_items_with=(u'VCALENDAR',)): + '''Emits items in the order they occur in the text.''' assert isinstance(text, text_type) collection = icalendar.cal.Component.from_ical(text) items = collection.subcomponents @@ -50,3 +51,38 @@ def to_unicode_lines(item): for content_line in item.content_lines(): if content_line: yield icalendar.parser.foldline(content_line) + + +def join_collection(items, wrapper=None): + timezones = {} + components = [] + + for item in items: + component = icalendar.cal.Component.from_ical(item) + if component.name == u'VCALENDAR': + assert wrapper is None or wrapper == u'VCALENDAR' + wrapper = u'VCALENDAR' + for subcomponent in component.subcomponents: + if subcomponent.name == u'VTIMEZONE': + timezones[subcomponent['TZID']] = subcomponent + else: + components.append(subcomponent) + else: + if component.name == u'VCARD': + assert wrapper is None or wrapper == u'VADDRESSBOOK' + wrapper = u'VADDRESSBOOK' + components.append(component) + + start = end = u'' + if wrapper is not None: + start = u'BEGIN:{}'.format(wrapper) + end = u'END:{}'.format(wrapper) + + lines = [start] + for timezone in itervalues(timezones): + lines.extend(to_unicode_lines(timezone)) + for component in components: + lines.extend(to_unicode_lines(component)) + lines.append(end) + + return u''.join(line + u'\r\n' for line in lines if line)