Mark unused variables as such

Adds support for auto-creating collections when they exist only on
one side and `implicit = 'create'` is set in the pair config.

.builds/archlinux-py313.yml

@@ -10,13 +10,12 @@ packages:
   - python-installer
   - python-setuptools-scm
   # Runtime dependencies:
-  - python-atomicwrites
   - python-click
   - python-click-log
   - python-click-threading
   - python-requests
-  - python-requests-toolbelt
   - python-aiohttp-oauthlib
+  - python-tenacity
   # Test dependencies:
   - python-hypothesis
   - python-pytest-cov
@@ -36,6 +35,8 @@ environment:
   REQUIREMENTS: release
   # TODO: ETESYNC_TESTS
 tasks:
+  - check-python:
+      python --version | grep 'Python 3.13'
   - docker: |
       sudo systemctl start docker
   - setup: |

.builds/tests-minimal.yml

@@ -3,7 +3,7 @@
 # TODO: It might make more sense to test with an older Ubuntu or Fedora version
 # here, and consider that our "oldest suppported environment".
 
-image: alpine/3.17 # python 3.10
+image: alpine/3.19 # python 3.11
 packages:
   - docker
   - docker-cli
@@ -18,7 +18,6 @@ environment:
   CODECOV_TOKEN: b834a3c5-28fa-4808-9bdb-182210069c79
   DAV_SERVER: radicale xandikos
   REQUIREMENTS: minimal
-  # TODO: ETESYNC_TESTS
 tasks:
   - venv: |
       python3 -m venv $HOME/venv
@@ -28,6 +27,8 @@ tasks:
       sudo service docker start
   - setup: |
       cd vdirsyncer
+      # Hack, no idea why it's needed
+      sudo ln -s /usr/include/python3.11/cpython/longintrepr.h /usr/include/python3.11/longintrepr.h
       make -e install-dev
   - test: |
       cd vdirsyncer

.builds/tests-pypi.yml

@@ -29,6 +29,9 @@ tasks:
       cd vdirsyncer
       make -e ci-test
       make -e ci-test-storage
+  - check: |
+      cd vdirsyncer
+      make check
   - check-secrets: |
       # Stop here if this is a PR. PRs can't run with the below secrets.
       [ -f ~/fastmail-secrets ] || complete-build

.envrc

@@ -0,0 +1 @@
+layout python3

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-python37:
-  image: python:3.7
-  before_script:
-    - make -e install-dev
-  script:
-    - make -e ci-test

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
@@ -8,12 +8,8 @@ repos:
       - id: check-toml
       - id: check-added-large-files
       - id: debug-statements
-  - repo: https://github.com/psf/black
-    rev: "24.2.0"
-    hooks:
-      - id: black
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v1.8.0"
+    rev: "v1.15.0"
     hooks:
       - id: mypy
         files: vdirsyncer/.*
@@ -21,12 +17,12 @@ repos:
           - types-setuptools
           - types-docutils
           - types-requests
-          - types-atomicwrites
   - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.2.2'
+    rev: 'v0.11.4'
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
   - repo: local
     hooks:
       - id: typos-syncroniz

.readthedocs.yaml

@@ -0,0 +1,16 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.9"
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

AUTHORS.rst

@@ -19,6 +19,7 @@ In alphabetical order:
 - rEnr3n
 - Thomas Weißschuh
 - Witcher01
+- samm81
 
 Special thanks goes to:
 

CHANGELOG.rst

@@ -9,6 +9,25 @@ Package maintainers and users who have to manually update their installation
 may want to subscribe to `GitHub's tag feed
 <https://github.com/pimutils/vdirsyncer/tags.atom>`_.
 
+Version 0.21.0
+==============
+
+- Implement retrying for ``google`` storage type when a rate limit is reached.
+- ``tenacity`` is now a required dependency.
+- Drop support for Python 3.8.
+- Retry transient network errors for nullipotent requests.
+
+Version 0.20.0
+==============
+
+- Remove dependency on abandoned ``atomicwrites`` library.
+- Implement ``filter_hook`` for the HTTP storage.
+- Drop support for Python 3.7.
+- Add support for Python 3.12 and Python 3.13.
+- Properly close the status database after using. This especially affects tests,
+  where we were leaking a large amount of file descriptors.
+- Extend supported versions of ``aiostream`` to include 0.7.x.
+
 Version 0.19.3
 ==============
 
@@ -18,6 +37,7 @@ Version 0.19.3
 - Require matching ``BEGIN`` and ``END`` lines in vobjects. :gh:`1103`
 - A Docker environment for Vdirsyncer has been added `Vdirsyncer DOCKERIZED <https://github.com/Bleala/Vdirsyncer-DOCKERIZED>`_.
 - Implement digest auth. :gh:`1137`
+- Add ``filter_hook`` parameter to :storage:`http`. :gh:`1136`
 
 Version 0.19.2
 ==============
@@ -53,6 +73,10 @@ Version 0.19.0
 - Add a new ``showconfig`` status. This prints *some* configuration values as
   JSON. This is intended to be used by external tools and helpers that interact
   with ``vdirsyncer``, and considered experimental.
+- Add ``implicit`` option to the :ref:`pair section <pair_config>`. When set to
+  "create", it implicitly creates missing collections during sync without user
+  prompts. This simplifies workflows where collections should be automatically
+  created on both sides.
 - Update TLS-related tests that were failing due to weak MDs. :gh:`903`
 - ``pytest-httpserver`` and ``trustme`` are now required for tests.
 - ``pytest-localserver`` is no longer required for tests.

Makefile

@@ -40,6 +40,11 @@ ci-test-storage:
 	done
 	bash $(CODECOV_PATH) -c
 
+check:
+	ruff check
+	ruff format --diff
+	#mypy vdirsyncer
+
 release-deb:
 	sh scripts/release-deb.sh debian jessie
 	sh scripts/release-deb.sh debian stretch
@@ -49,10 +54,10 @@ release-deb:
 
 install-dev:
 	pip install -U pip setuptools wheel
-	pip install -e .
+	pip install -e '.[test,check,docs]'
-	pip install -Ur test-requirements.txt -r docs-requirements.txt pre-commit
 	set -xe && if [ "$(REQUIREMENTS)" = "minimal" ]; then \
-		pip install -U --force-reinstall $$(python setup.py --quiet minimal_requirements); \
+		pip install pyproject-dependencies && \
+		pip install -U --force-reinstall $$(pyproject-dependencies . | sed 's/>/=/'); \
 	fi
 
 .PHONY: docs

README.rst

@@ -40,7 +40,7 @@ servers. It can also be used to synchronize calendars and/or addressbooks
 between two servers directly.
 
 It aims to be for calendars and contacts what `OfflineIMAP
-<http://offlineimap.org/>`_ is for emails.
+<https://www.offlineimap.org/>`_ is for emails.
 
 .. _programs: https://vdirsyncer.pimutils.org/en/latest/tutorials/
 

contrib/conflict_resolution/resolve_interactively.py

@@ -16,6 +16,7 @@ SPDX-License-Identifier: BSD-3-Clause
 SPDX-FileCopyrightText: 2021 Intevation GmbH <https://intevation.de>
 Author: <bernhard.reiter@intevation.de>
 """
+
 from __future__ import annotations
 
 import re
@@ -53,8 +54,8 @@ def main(ical1_filename, ical2_filename):
             f"{get_summary(ical1)}...\n(full contents: {ical1_filename})\n\n"
             "or the second entry:\n"
             f"{get_summary(ical2)}...\n(full contents: {ical2_filename})?",
+            *additional_args,
         ]
-        + additional_args
     )
 
     if r.returncode == 2:

docs-requirements.txt

@@ -1,4 +0,0 @@
-# This file is used by readthedocs.org
-sphinx != 1.4.7
-sphinx_rtd_theme
-setuptools_scm

docs/conf.py

@@ -20,7 +20,7 @@ copyright = "2014-{}, Markus Unterwaditzer & contributors".format(
 release = get_distribution("vdirsyncer").version
 version = ".".join(release.split(".")[:2])  # The short X.Y version.
 
-rst_epilog = ".. |vdirsyncer_version| replace:: %s" % release
+rst_epilog = f".. |vdirsyncer_version| replace:: {release}"
 
 exclude_patterns = ["_build"]
 
@@ -37,9 +37,7 @@ except ImportError:
     html_theme = "default"
     if not on_rtd:
         print("-" * 74)
-        print(
+        print("Warning: sphinx-rtd-theme not installed, building with default theme.")
-            "Warning: sphinx-rtd-theme not installed, building with default " "theme."
-        )
         print("-" * 74)
 
 html_static_path = ["_static"]

docs/config.rst

@@ -128,6 +128,16 @@ Pair Section
 
   The ``conflict_resolution`` parameter applies for these properties too.
 
+.. _implicit_def:
+
+- ``implicit``: Opt into implicitly creating collections. Example::
+
+      implicit = "create"
+
+  When set to "create", missing collections are automatically created on both
+  sides during sync without prompting the user. This simplifies workflows where
+  all collections should be synchronized bidirectionally.
+
 .. _storage_config:
 
 Storage Section
@@ -484,6 +494,7 @@ leads to an error.
         [storage holidays_remote]
         type = "http"
         url = https://example.com/holidays_from_hicksville.ics
+        #filter_hook = null
 
     Too many WebCAL providers generate UIDs of all ``VEVENT``-components
     on-the-fly, i.e. all UIDs change every time the calendar is downloaded.
@@ -508,3 +519,8 @@ leads to an error.
     :param auth_cert: Optional. Either a path to a certificate with a client
         certificate and the key or a list of paths to the files with them.
     :param useragent: Default ``vdirsyncer``.
+    :param filter_hook: Optional. A filter command to call for each fetched
+        item, passed in raw form to stdin and returned via stdout.
+        If nothing is returned by the filter command, the item is skipped.
+        This can be used to alter fields as needed when dealing with providers
+        generating malformed events.

docs/contributing.rst

@@ -81,7 +81,7 @@ virtualenv_ and run this inside of it::
 
     # Install development dependencies, including:
     #  - vdirsyncer from the repo into the virtualenv
-    #  - stylecheckers (ruff) and code formatters (black)
+    #  - style checks and formatting (ruff)
     make install-dev
 
     # Install git commit hook for some extra linting and checking

docs/installation.rst

@@ -42,7 +42,7 @@ If your distribution doesn't provide a package for vdirsyncer, you still can
 use Python's package manager "pip". First, you'll have to check that the
 following things are installed:
 
-- Python 3.7 to 3.11 and pip.
+- Python 3.9 to 3.13 and pip.
 - ``libxml`` and ``libxslt``
 - ``zlib``
 - Linux or macOS. **Windows is not supported**, see :gh:`535`.
@@ -84,7 +84,7 @@ the above location.
 The dirty, easy way
 ~~~~~~~~~~~~~~~~~~~
 
-If pipx is not available on your distirbution, the easiest way to install
+If pipx is not available on your distribution, the easiest way to install
 vdirsyncer at this point would be to run::
 
     pip install --ignore-installed vdirsyncer

docs/packaging.rst

@@ -46,8 +46,9 @@ You can install the all development dependencies with::
     make install-dev
 
 You probably don't want this since it will use pip to download the
-dependencies. Alternatively you can find the testing dependencies in
+dependencies. Alternatively test dependencies are listed as ``test`` optional
-``test-requirements.txt``, again with lower-bound version requirements.
+dependencies in ``pyproject.toml``, again with lower-bound version
+requirements.
 
 You also have to have vdirsyncer fully installed at this point. Merely
 ``cd``-ing into the tarball will not be sufficient.
@@ -73,10 +74,11 @@ Using Sphinx_ you can generate the documentation you're reading right now in a
 variety of formats, such as HTML, PDF, or even as a manpage. That said, I only
 take care of the HTML docs' formatting.
 
-You can find a list of dependencies in ``docs-requirements.txt``. Again, you
+You can find a list of dependencies in ``pyproject.toml``, in the
-can install those using pip with::
+``project.optional-dependencies`` section as ``docs``. Again, you can install
+those using pip with::
 
-    pip install -r docs-requirements.txt
+    pip install '.[docs]'
 
 Then change into the ``docs/`` directory and build whatever format you want
 using the ``Makefile`` in there (run ``make`` for the formats you can build).

docs/when.rst

@@ -50,7 +50,6 @@ program chosen:
 
 * Such a setup doesn't work at all with smartphones. Vdirsyncer, on the other
   hand, synchronizes with CardDAV/CalDAV servers, which can be accessed with
-  e.g. DAVx⁵_ or the apps by dmfs_.
+  e.g. DAVx⁵_ or other apps bundled with smartphones.
 
 .. _DAVx⁵: https://www.davx5.com/
-.. _dmfs: https://dmfs.org/

publish-release.yaml

@@ -4,8 +4,10 @@
 
 image: alpine/edge
 packages:
+  - py3-build
   - py3-pip
   - py3-setuptools
+  - py3-setuptools_scm
   - py3-wheel
   - twine
 sources:
@@ -23,5 +25,5 @@ tasks:
       git describe --exact-match --tags || complete-build
   - publish: |
       cd vdirsyncer
-      python setup.py sdist bdist_wheel
+      python -m build --no-isolation
       twine upload --non-interactive dist/*

pyproject.toml

@@ -1,18 +1,88 @@
-[tool.ruff]
+# Vdirsyncer synchronizes calendars and contacts.
-select = [
+#
-    "E",
+# Please refer to https://vdirsyncer.pimutils.org/en/stable/packaging.html for
-    "F",
+# how to package vdirsyncer.
-    "W",
-    "B0",
-    "I",
-    "UP",
-    "C4",
-    # "TID",
-    "RSE"
-]
-target-version = "py37"
 
-[tool.ruff.isort]
+[build-system]
+requires = ["setuptools>=64", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vdirsyncer"
+authors = [
+    {name = "Markus Unterwaditzer", email = "markus@unterwaditzer.net"},
+]
+description = "Synchronize calendars and contacts"
+readme = "README.rst"
+requires-python = ">=3.9"
+keywords = ["todo", "task", "icalendar", "cli"]
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Operating System :: POSIX",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.9",
+    "Topic :: Internet",
+    "Topic :: Office/Business :: Scheduling",
+    "Topic :: Utilities",
+]
+dependencies = [
+    "click>=5.0,<9.0",
+    "click-log>=0.3.0,<0.5.0",
+    "requests>=2.20.0",
+    "aiohttp>=3.8.2,<4.0.0",
+    "aiostream>=0.4.3,<0.8.0",
+    "tenacity>=9.0.0",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+google = ["aiohttp-oauthlib"]
+test = [
+  "hypothesis>=6.72.0,<7.0.0",
+  "pytest",
+  "pytest-cov",
+  "pytest-httpserver",
+  "trustme",
+  "pytest-asyncio",
+  "aioresponses",
+]
+docs = [
+  "sphinx!=1.4.7",
+  "sphinx_rtd_theme",
+  "setuptools_scm",
+]
+check = [
+  "mypy",
+  "ruff",
+  "types-docutils",
+  "types-requests",
+  "types-setuptools",
+]
+
+[project.scripts]
+vdirsyncer = "vdirsyncer.cli:app"
+
+[tool.ruff.lint]
+extend-select = [
+    "B0",
+    "C4",
+    "E",
+    "I",
+    "RSE",
+    "SIM",
+    "TID",
+    "UP",
+    "W",
+]
+
+[tool.ruff.lint.isort]
 force-single-line = true
 required-imports = ["from __future__ import annotations"]
 
@@ -26,6 +96,7 @@ addopts = """
 --color=yes
 """
 # filterwarnings=error
+asyncio_default_fixture_loop_scope = "function"
 
 [tool.mypy]
 ignore_missing_imports = true
@@ -34,3 +105,10 @@ ignore_missing_imports = true
 exclude_lines = [
     "if TYPE_CHECKING:",
 ]
+
+[tool.setuptools.packages.find]
+include = ["vdirsyncer*"]
+
+[tool.setuptools_scm]
+write_to = "vdirsyncer/version.py"
+version_scheme = "no-guess-dev"

setup.py

@@ -1,82 +0,0 @@
-"""
-Vdirsyncer synchronizes calendars and contacts.
-
-Please refer to https://vdirsyncer.pimutils.org/en/stable/packaging.html for
-how to package vdirsyncer.
-"""
-
-from __future__ import annotations
-
-from setuptools import Command
-from setuptools import find_packages
-from setuptools import setup
-
-requirements = [
-    # https://github.com/mitsuhiko/click/issues/200
-    "click>=5.0,<9.0",
-    "click-log>=0.3.0, <0.5.0",
-    "requests >=2.20.0",
-    # https://github.com/untitaker/python-atomicwrites/commit/4d12f23227b6a944ab1d99c507a69fdbc7c9ed6d  # noqa
-    "atomicwrites>=0.1.7",
-    "aiohttp>=3.8.0,<4.0.0",
-    "aiostream>=0.4.3,<0.5.0",
-]
-
-
-class PrintRequirements(Command):
-    description = "Prints minimal requirements"
-    user_options: list = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        for requirement in requirements:
-            print(requirement.replace(">", "=").replace(" ", ""))
-
-
-with open("README.rst") as f:
-    long_description = f.read()
-
-
-setup(
-    # General metadata
-    name="vdirsyncer",
-    author="Markus Unterwaditzer",
-    author_email="markus@unterwaditzer.net",
-    url="https://github.com/pimutils/vdirsyncer",
-    description="Synchronize calendars and contacts",
-    license="BSD",
-    long_description=long_description,
-    # Runtime dependencies
-    install_requires=requirements,
-    # Optional dependencies
-    extras_require={
-        "google": ["aiohttp-oauthlib"],
-    },
-    # Build dependencies
-    setup_requires=["setuptools_scm != 1.12.0"],
-    # Other
-    packages=find_packages(exclude=["tests.*", "tests"]),
-    include_package_data=True,
-    cmdclass={"minimal_requirements": PrintRequirements},
-    use_scm_version={"write_to": "vdirsyncer/version.py"},
-    entry_points={"console_scripts": ["vdirsyncer = vdirsyncer.cli:app"]},
-    classifiers=[
-        "Development Status :: 4 - Beta",
-        "Environment :: Console",
-        "License :: OSI Approved :: BSD License",
-        "Operating System :: POSIX",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Programming Language :: Python :: 3.10",
-        "Programming Language :: Python :: 3.11",
-        "Topic :: Internet",
-        "Topic :: Utilities",
-    ],
-)

test-requirements.txt

@@ -1,7 +0,0 @@
-hypothesis>=5.0.0,<7.0.0
-pytest
-pytest-cov
-pytest-httpserver
-trustme
-pytest-asyncio
-aioresponses

tests/__init__.py

@@ -103,10 +103,8 @@ X-SOMETHING:{r}
 HAHA:YES
 END:FOO"""
 
-printable_characters_strategy = st.text(
+printable_characters_strategy = st.text(st.characters(exclude_categories=("Cc", "Cs")))
-    st.characters(blacklist_categories=("Cc", "Cs"))
-)
 
 uid_strategy = st.text(
-    st.characters(blacklist_categories=("Zs", "Zl", "Zp", "Cc", "Cs")), min_size=1
+    st.characters(exclude_categories=("Zs", "Zl", "Zp", "Cc", "Cs")), min_size=1
 ).filter(lambda x: x.strip() == x)

tests/conftest.py

@@ -45,7 +45,7 @@ settings.register_profile(
     "deterministic",
     settings(
         derandomize=True,
-        suppress_health_check=HealthCheck.all(),
+        suppress_health_check=list(HealthCheck),
     ),
 )
 settings.register_profile("dev", settings(suppress_health_check=[HealthCheck.too_slow]))
@@ -59,12 +59,12 @@ else:
 
 
 @pytest_asyncio.fixture
-async def aio_session(event_loop):
+async def aio_session():
     async with aiohttp.ClientSession() as session:
         yield session
 
 
 @pytest_asyncio.fixture
-async def aio_connector(event_loop):
+async def aio_connector():
     async with aiohttp.TCPConnector(limit_per_host=16) as conn:
         yield conn

tests/storage/__init__.py

@@ -10,16 +10,15 @@ import aiostream
 import pytest
 import pytest_asyncio
 
+from tests import EVENT_TEMPLATE
+from tests import TASK_TEMPLATE
+from tests import VCARD_TEMPLATE
+from tests import assert_item_equals
+from tests import normalize_item
 from vdirsyncer import exceptions
 from vdirsyncer.storage.base import normalize_meta_value
 from vdirsyncer.vobject import Item
 
-from .. import EVENT_TEMPLATE
-from .. import TASK_TEMPLATE
-from .. import VCARD_TEMPLATE
-from .. import assert_item_equals
-from .. import normalize_item
-
 
 def get_server_mixin(server_name):
     from . import __name__ as base
@@ -105,7 +104,7 @@ class StorageTests:
         href, etag = await s.upload(get_item())
         if etag is None:
             _, etag = await s.get(href)
-        ((href2, item, etag2),) = await aiostream.stream.list(s.get_multi([href] * 2))
+        ((href2, _item, etag2),) = await aiostream.stream.list(s.get_multi([href] * 2))
         assert href2 == href
         assert etag2 == etag
 
@@ -119,7 +118,7 @@ class StorageTests:
     @pytest.mark.asyncio
     async def test_upload(self, s, get_item):
         item = get_item()
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
         assert_item_equals((await s.get(href))[0], item)
 
     @pytest.mark.asyncio
@@ -147,7 +146,7 @@ class StorageTests:
     @pytest.mark.asyncio
     async def test_wrong_etag(self, s, get_item):
         item = get_item()
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
         with pytest.raises(exceptions.PreconditionFailed):
             await s.update(href, item, '"lolnope"')
         with pytest.raises(exceptions.PreconditionFailed):
@@ -423,7 +422,7 @@ class StorageTests:
             ).strip()
         )
 
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
 
-        item2, etag2 = await s.get(href)
+        item2, _etag2 = await s.get(href)
         assert normalize_item(item) == normalize_item(item2)

tests/storage/dav/__init__.py

@@ -8,12 +8,11 @@ import aiostream
 import pytest
 
 from tests import assert_item_equals
+from tests.storage import StorageTests
+from tests.storage import get_server_mixin
 from vdirsyncer import exceptions
 from vdirsyncer.vobject import Item
 
-from .. import StorageTests
-from .. import get_server_mixin
-
 dav_server = os.environ.get("DAV_SERVER", "skip")
 ServerMixin = get_server_mixin(dav_server)
 
@@ -49,6 +48,6 @@ class DAVStorageTests(ServerMixin, StorageTests):
 
         monkeypatch.setattr(s, "_get_href", lambda item: item.ident + s.fileext)
         item = get_item(uid="град сатану" + str(uuid.uuid4()))
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
-        item2, etag2 = await s.get(href)
+        item2, _etag2 = await s.get(href)
         assert_item_equals(item, item2)

tests/storage/dav/test_caldav.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import datetime
 from textwrap import dedent
 
@@ -11,10 +12,10 @@ from aioresponses import aioresponses
 from tests import EVENT_TEMPLATE
 from tests import TASK_TEMPLATE
 from tests import VCARD_TEMPLATE
+from tests.storage import format_item
 from vdirsyncer import exceptions
 from vdirsyncer.storage.dav import CalDAVStorage
 
-from .. import format_item
 from . import DAVStorageTests
 from . import dav_server
 
@@ -30,18 +31,16 @@ class TestCalDAVStorage(DAVStorageTests):
     async def test_doesnt_accept_vcard(self, item_type, get_storage_args):
         s = self.storage_class(item_types=(item_type,), **await get_storage_args())
 
-        try:
+        # Most storages hard-fail, but xandikos doesn't.
+        with contextlib.suppress(exceptions.Error, aiohttp.ClientResponseError):
             await s.upload(format_item(VCARD_TEMPLATE))
-        except (exceptions.Error, aiohttp.ClientResponseError):
-            # Most storages hard-fail, but xandikos doesn't.
-            pass
 
         assert not await aiostream.stream.list(s.list())
 
     # The `arg` param is not named `item_types` because that would hit
     # https://bitbucket.org/pytest-dev/pytest/issue/745/
     @pytest.mark.parametrize(
-        "arg,calls_num",
+        ("arg", "calls_num"),
         [
             (("VTODO",), 1),
             (("VEVENT",), 1),

tests/storage/dav/test_main.py

@@ -52,7 +52,7 @@ def test_xml_specialchars(char):
 @pytest.mark.parametrize(
     "href",
     [
-        "/dav/calendars/user/testuser/123/UID%253A20210609T084907Z-@synaps-web-54fddfdf7-7kcfm%250A.ics",  # noqa: E501
+        "/dav/calendars/user/testuser/123/UID%253A20210609T084907Z-@synaps-web-54fddfdf7-7kcfm%250A.ics",
     ],
 )
 def test_normalize_href(href):

tests/storage/servers/davical/__init__.py

@@ -13,7 +13,7 @@ try:
         "url": "https://brutus.lostpackets.de/davical-test/caldav.php/",
     }
 except KeyError as e:
-    pytestmark = pytest.mark.skip(f"Missing envkey: {str(e)}")
+    pytestmark = pytest.mark.skip(f"Missing envkey: {e!s}")
 
 
 @pytest.mark.flaky(reruns=5)

tests/storage/servers/fastmail/__init__.py

@@ -8,11 +8,13 @@ import pytest
 class ServerMixin:
     @pytest.fixture
     def get_storage_args(self, slow_create_collection, aio_connector, request):
-        if "item_type" in request.fixturenames:
+        if (
-            if request.getfixturevalue("item_type") == "VTODO":
+            "item_type" in request.fixturenames
-                # Fastmail has non-standard support for TODOs
+            and request.getfixturevalue("item_type") == "VTODO"
-                # See https://github.com/pimutils/vdirsyncer/issues/824
+        ):
-                pytest.skip("Fastmail has non-standard VTODO support.")
+            # Fastmail has non-standard support for TODOs
+            # See https://github.com/pimutils/vdirsyncer/issues/824
+            pytest.skip("Fastmail has non-standard VTODO support.")
 
         async def inner(collection="test"):
             args = {

tests/storage/servers/icloud/__init__.py

@@ -10,7 +10,7 @@ class ServerMixin:
     def get_storage_args(self, item_type, slow_create_collection):
         if item_type != "VEVENT":
             # iCloud collections can either be calendars or task lists.
-            # See https://github.com/pimutils/vdirsyncer/pull/593#issuecomment-285941615  # noqa
+            # See https://github.com/pimutils/vdirsyncer/pull/593#issuecomment-285941615
             pytest.skip("iCloud doesn't support anything else than VEVENT")
 
         async def inner(collection="test"):

tests/storage/test_filesystem.py

@@ -48,7 +48,8 @@ class TestFilesystemStorage(StorageTests):
         s = self.storage_class(str(tmpdir), ".txt")
         await s.upload(Item("UID:a/b/c"))
         (item_file,) = tmpdir.listdir()
-        assert "/" not in item_file.basename and item_file.isfile()
+        assert "/" not in item_file.basename
+        assert item_file.isfile()
 
     @pytest.mark.asyncio
     async def test_ignore_tmp_files(self, tmpdir):
@@ -89,7 +90,7 @@ class TestFilesystemStorage(StorageTests):
         storage = self.storage_class(str(tmpdir), ".txt")
         item = Item("UID:" + "hue" * 600)
 
-        href, etag = await storage.upload(item)
+        href, _etag = await storage.upload(item)
         assert item.uid not in href
 
     @pytest.mark.asyncio

tests/storage/test_http.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import aiohttp
 import pytest
 from aioresponses import CallbackResult
 from aioresponses import aioresponses
@@ -8,6 +9,8 @@ from tests import normalize_item
 from vdirsyncer.exceptions import UserError
 from vdirsyncer.http import BasicAuthMethod
 from vdirsyncer.http import DigestAuthMethod
+from vdirsyncer.http import UsageLimitReached
+from vdirsyncer.http import request
 from vdirsyncer.storage.http import HttpStorage
 from vdirsyncer.storage.http import prepare_auth
 
@@ -38,7 +41,7 @@ async def test_list(aio_connector):
         ),
     ]
 
-    responses = ["\n".join(["BEGIN:VCALENDAR"] + items + ["END:VCALENDAR"])] * 2
+    responses = ["\n".join(["BEGIN:VCALENDAR", *items, "END:VCALENDAR"])] * 2
 
     def callback(url, headers, **kwargs):
         assert headers["User-Agent"].startswith("vdirsyncer/")
@@ -120,3 +123,41 @@ def test_verify_false_disallowed(aio_connector):
         HttpStorage(url="http://example.com", verify=False, connector=aio_connector)
 
     assert "must be a path to a pem-file." in str(excinfo.value).lower()
+
+
+@pytest.mark.asyncio
+async def test_403_usage_limit_exceeded(aio_connector):
+    url = "http://127.0.0.1/test_403"
+    error_body = {
+        "error": {
+            "errors": [
+                {
+                    "domain": "usageLimits",
+                    "message": "Calendar usage limits exceeded.",
+                    "reason": "quotaExceeded",
+                }
+            ],
+            "code": 403,
+            "message": "Calendar usage limits exceeded.",
+        }
+    }
+
+    async with aiohttp.ClientSession(connector=aio_connector) as session:
+        with aioresponses() as m:
+            m.get(url, status=403, payload=error_body, repeat=True)
+            with pytest.raises(UsageLimitReached):
+                await request("GET", url, session)
+
+
+@pytest.mark.asyncio
+async def test_403_without_usage_limits_domain(aio_connector):
+    """A 403 JSON error without the Google 'usageLimits' domain should not be
+    treated as UsageLimitReached and should surface as ClientResponseError.
+    """
+    url = "http://127.0.0.1/test_403_no_usage_limits"
+
+    async with aiohttp.ClientSession(connector=aio_connector) as session:
+        with aioresponses() as m:
+            m.get(url, status=403, repeat=True)
+            with pytest.raises(aiohttp.ClientResponseError):
+                await request("GET", url, session)

tests/storage/test_http_with_singlefile.py

@@ -20,7 +20,7 @@ class CombinedStorage(Storage):
     storage_name = "http_and_singlefile"
 
     def __init__(self, url, path, *, connector, **kwargs):
-        if kwargs.get("collection", None) is not None:
+        if kwargs.get("collection") is not None:
             raise ValueError
 
         super().__init__(**kwargs)

tests/system/cli/test_config.py

@@ -26,7 +26,7 @@ def read_config(tmpdir, monkeypatch):
 
 
 def test_read_config(read_config):
-    errors, c = read_config(
+    _errors, c = read_config(
         """
         [general]
         status_path = "/tmp/status/"
@@ -222,3 +222,62 @@ def test_validate_collections_param():
     x([["c", None, "b"]])
     x([["c", "a", None]])
     x([["c", None, None]])
+
+
+def test_invalid_implicit_value(read_config):
+    expected_message = "`implicit` parameter must be 'create' or absent"
+    with pytest.raises(exceptions.UserError) as excinfo:
+        read_config(
+            """
+            [general]
+            status_path = "/tmp/status/"
+
+            [pair my_pair]
+            a = "my_a"
+            b = "my_b"
+            collections = null
+            implicit = "invalid"
+
+            [storage my_a]
+            type = "filesystem"
+            path = "{base}/path_a/"
+            fileext = ".txt"
+
+            [storage my_b]
+            type = "filesystem"
+            path = "{base}/path_b/"
+            fileext = ".txt"
+            """
+        )
+
+    assert expected_message in str(excinfo.value)
+
+
+def test_implicit_create_only(read_config):
+    """Test that implicit create works."""
+    errors, c = read_config(
+        """
+        [general]
+        status_path = "/tmp/status/"
+
+        [pair my_pair]
+        a = "my_a"
+        b = "my_b"
+        collections = ["from a", "from b"]
+        implicit = "create"
+
+        [storage my_a]
+        type = "filesystem"
+        path = "{base}/path_a/"
+        fileext = ".txt"
+
+        [storage my_b]
+        type = "filesystem"
+        path = "{base}/path_b/"
+        fileext = ".txt"
+        """
+    )
+
+    assert not errors
+    pair = c.pairs["my_pair"]
+    assert pair.implicit == "create"

tests/system/cli/test_discover.py

@@ -161,12 +161,12 @@ def test_null_collection_with_named_collection(tmpdir, runner):
 
     [storage foo]
     type = "filesystem"
-    path = "{str(tmpdir)}/foo/"
+    path = "{tmpdir!s}/foo/"
     fileext = ".txt"
 
     [storage bar]
     type = "singlefile"
-    path = "{str(tmpdir)}/bar.txt"
+    path = "{tmpdir!s}/bar.txt"
     """
         )
     )
@@ -191,7 +191,7 @@ def test_null_collection_with_named_collection(tmpdir, runner):
 
 
 @pytest.mark.parametrize(
-    "a_requires,b_requires",
+    ("a_requires", "b_requires"),
     [
         (True, True),
         (True, False),

tests/system/cli/test_fetchparams.py

@@ -14,12 +14,12 @@ def test_get_password_from_command(tmpdir, runner):
 
         [storage foo]
         type.fetch = ["shell", "echo filesystem"]
-        path = "{str(tmpdir)}/foo/"
+        path = "{tmpdir!s}/foo/"
         fileext.fetch = ["command", "echo", ".txt"]
 
         [storage bar]
         type = "filesystem"
-        path = "{str(tmpdir)}/bar/"
+        path = "{tmpdir!s}/bar/"
         fileext.fetch = ["prompt", "Fileext for bar"]
     """
         )

tests/system/cli/test_repair.py

@@ -58,7 +58,7 @@ def test_repair_uids(storage, runner, repair_uids):
     else:
         opt = ["--no-repair-unsafe-uid"]
 
-    result = runner.invoke(["repair"] + opt + ["foo"], input="y")
+    result = runner.invoke(["repair", *opt, "foo"], input="y")
     assert not result.exception
 
     if repair_uids:

tests/system/cli/test_sync.py

@@ -90,9 +90,7 @@ def test_empty_storage(tmpdir, runner):
     result = runner.invoke(["sync"])
     lines = result.output.splitlines()
     assert lines[0] == "Syncing my_pair"
-    assert lines[1].startswith(
+    assert lines[1].startswith('error: my_pair: Storage "my_b" was completely emptied.')
-        "error: my_pair: " 'Storage "my_b" was completely emptied.'
-    )
     assert result.exception
 
 
@@ -290,12 +288,12 @@ def test_create_collections(collections, tmpdir, runner):
 
     [storage foo]
     type = "filesystem"
-    path = "{str(tmpdir)}/foo/"
+    path = "{tmpdir!s}/foo/"
     fileext = ".txt"
 
     [storage bar]
     type = "filesystem"
-    path = "{str(tmpdir)}/bar/"
+    path = "{tmpdir!s}/bar/"
     fileext = ".txt"
     """
         )
@@ -323,12 +321,12 @@ def test_ident_conflict(tmpdir, runner):
 
     [storage foo]
     type = "filesystem"
-    path = "{str(tmpdir)}/foo/"
+    path = "{tmpdir!s}/foo/"
     fileext = ".txt"
 
     [storage bar]
     type = "filesystem"
-    path = "{str(tmpdir)}/bar/"
+    path = "{tmpdir!s}/bar/"
     fileext = ".txt"
     """
         )
@@ -360,7 +358,7 @@ def test_ident_conflict(tmpdir, runner):
 
 
 @pytest.mark.parametrize(
-    "existing,missing",
+    ("existing", "missing"),
     [
         ("foo", "bar"),
         ("bar", "foo"),
@@ -377,7 +375,7 @@ def test_unknown_storage(tmpdir, runner, existing, missing):
 
     [storage {existing}]
     type = "filesystem"
-    path = "{str(tmpdir)}/{existing}/"
+    path = "{tmpdir!s}/{existing}/"
     fileext = ".txt"
     """
         )
@@ -404,7 +402,7 @@ def test_no_configured_pairs(tmpdir, runner, cmd):
 
 
 @pytest.mark.parametrize(
-    "resolution,expect_foo,expect_bar",
+    ("resolution", "expect_foo", "expect_bar"),
     [(["command", "cp"], "UID:lol\nfööcontent", "UID:lol\nfööcontent")],
 )
 def test_conflict_resolution(tmpdir, runner, resolution, expect_foo, expect_bar):
@@ -420,12 +418,12 @@ def test_conflict_resolution(tmpdir, runner, resolution, expect_foo, expect_bar)
     [storage foo]
     type = "filesystem"
     fileext = ".txt"
-    path = "{str(tmpdir)}/foo"
+    path = "{tmpdir!s}/foo"
 
     [storage bar]
     type = "filesystem"
     fileext = ".txt"
-    path = "{str(tmpdir)}/bar"
+    path = "{tmpdir!s}/bar"
     """
         )
     )
@@ -520,7 +518,7 @@ def test_fetch_only_necessary_params(tmpdir, runner):
         dedent(
             f"""
     set -e
-    touch "{str(fetched_file)}"
+    touch "{fetched_file!s}"
     echo ".txt"
     """
         )
@@ -553,9 +551,7 @@ def test_fetch_only_necessary_params(tmpdir, runner):
     type = "filesystem"
     path = "{path}"
     fileext.fetch = ["command", "sh", "{script}"]
-    """.format(
+    """.format(path=str(tmpdir.mkdir("bogus")), script=str(fetch_script))
-                path=str(tmpdir.mkdir("bogus")), script=str(fetch_script)
-            )
         )
     )
 

tests/system/cli/test_utils.py

@@ -14,7 +14,7 @@ def test_handle_cli_error(capsys):
     except BaseException:
         handle_cli_error()
 
-    out, err = capsys.readouterr()
+    _out, err = capsys.readouterr()
     assert "returned something vdirsyncer doesn't understand" in err
     assert "ayy lmao" in err
 

tests/unit/cli/test_discover.py

@@ -9,7 +9,7 @@ missing = object()
 
 
 @pytest.mark.parametrize(
-    "shortcuts,expected",
+    ("shortcuts", "expected"),
     [
         (
             ["from a"],

tests/unit/sync/test_status.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import contextlib
+
 import hypothesis.strategies as st
 from hypothesis import assume
 from hypothesis import given
@@ -24,13 +26,13 @@ def test_legacy_status(status_dict):
     hrefs_a = {meta_a["href"] for meta_a, meta_b in status_dict.values()}
     hrefs_b = {meta_b["href"] for meta_a, meta_b in status_dict.values()}
     assume(len(hrefs_a) == len(status_dict) == len(hrefs_b))
-    status = SqliteStatus()
+    with contextlib.closing(SqliteStatus()) as status:
-    status.load_legacy_status(status_dict)
+        status.load_legacy_status(status_dict)
-    assert dict(status.to_legacy_status()) == status_dict
+        assert dict(status.to_legacy_status()) == status_dict
 
-    for ident, (meta_a, meta_b) in status_dict.items():
+        for ident, (meta_a, meta_b) in status_dict.items():
-        ident_a, meta2_a = status.get_by_href_a(meta_a["href"])
+            ident_a, meta2_a = status.get_by_href_a(meta_a["href"])
-        ident_b, meta2_b = status.get_by_href_b(meta_b["href"])
+            ident_b, meta2_b = status.get_by_href_b(meta_b["href"])
-        assert meta2_a.to_status() == meta_a
+            assert meta2_a.to_status() == meta_a
-        assert meta2_b.to_status() == meta_b
+            assert meta2_b.to_status() == meta_b
-        assert ident_a == ident_b == ident
+            assert ident_a == ident_b == ident

tests/unit/sync/test_sync.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import asyncio
+import contextlib
 from copy import deepcopy
 
 import aiostream
@@ -25,13 +26,12 @@ from vdirsyncer.sync.status import SqliteStatus
 from vdirsyncer.vobject import Item
 
 
-async def sync(a, b, status, *args, **kwargs):
+async def sync(a, b, status, *args, **kwargs) -> None:
-    new_status = SqliteStatus(":memory:")
+    with contextlib.closing(SqliteStatus(":memory:")) as new_status:
-    new_status.load_legacy_status(status)
+        new_status.load_legacy_status(status)
-    rv = await _sync(a, b, new_status, *args, **kwargs)
+        await _sync(a, b, new_status, *args, **kwargs)
-    status.clear()
+        status.clear()
-    status.update(new_status.to_legacy_status())
+        status.update(new_status.to_legacy_status())
-    return rv
 
 
 def empty_storage(x):
@@ -98,7 +98,8 @@ async def test_read_only_and_prefetch():
     await sync(a, b, status, force_delete=True)
     await sync(a, b, status, force_delete=True)
 
-    assert not items(a) and not items(b)
+    assert not items(a)
+    assert not items(b)
 
 
 @pytest.mark.asyncio
@@ -226,7 +227,8 @@ async def test_insert_hash():
 
     await a.update(href, Item("UID:1\nHAHA:YES"), etag)
     await sync(a, b, status)
-    assert "hash" in status["1"][0] and "hash" in status["1"][1]
+    assert "hash" in status["1"][0]
+    assert "hash" in status["1"][1]
 
 
 @pytest.mark.asyncio
@@ -346,7 +348,7 @@ async def test_uses_get_multi(monkeypatch):
     a = MemoryStorage()
     b = MemoryStorage()
     item = Item("UID:1")
-    expected_href, etag = await a.upload(item)
+    expected_href, _etag = await a.upload(item)
 
     await sync(a, b, {})
     assert get_multi_calls == [[expected_href]]
@@ -383,7 +385,7 @@ async def test_changed_uids():
     a = MemoryStorage()
     b = MemoryStorage()
     href_a, etag_a = await a.upload(Item("UID:A-ONE"))
-    href_b, etag_b = await b.upload(Item("UID:B-ONE"))
+    _href_b, _etag_b = await b.upload(Item("UID:B-ONE"))
     status = {}
     await sync(a, b, status)
 
@@ -437,7 +439,7 @@ async def test_partial_sync_revert():
     assert items(a) == {"UID:2"}
 
 
-@pytest.mark.parametrize("sync_inbetween", (True, False))
+@pytest.mark.parametrize("sync_inbetween", [True, False])
 @pytest.mark.asyncio
 async def test_ident_conflict(sync_inbetween):
     a = MemoryStorage()
@@ -467,7 +469,7 @@ async def test_moved_href():
     a = MemoryStorage()
     b = MemoryStorage()
     status = {}
-    href, etag = await a.upload(Item("UID:haha"))
+    _href, _etag = await a.upload(Item("UID:haha"))
     await sync(a, b, status)
 
     b.items["lol"] = b.items.pop("haha")
@@ -528,7 +530,7 @@ async def test_unicode_hrefs():
     a = MemoryStorage()
     b = MemoryStorage()
     status = {}
-    href, etag = await a.upload(Item("UID:äää"))
+    _href, _etag = await a.upload(Item("UID:äää"))
     await sync(a, b, status)
 
 
@@ -551,7 +553,7 @@ class SyncMachine(RuleBasedStateMachine):
         if flaky_etags:
 
             async def get(href):
-                old_etag, item = s.items[href]
+                _old_etag, item = s.items[href]
                 etag = _random_string()
                 s.items[href] = etag, item
                 return item, etag
@@ -642,10 +644,7 @@ class SyncMachine(RuleBasedStateMachine):
 
             errors = []
 
-            if with_error_callback:
+            error_callback = errors.append if with_error_callback else None
-                error_callback = errors.append
-            else:
-                error_callback = None
 
             try:
                 # If one storage is read-only, double-sync because changes don't
@@ -668,7 +667,8 @@ class SyncMachine(RuleBasedStateMachine):
             except ActionIntentionallyFailed:
                 pass
             except BothReadOnly:
-                assert a.read_only and b.read_only
+                assert a.read_only
+                assert b.read_only
                 assume(False)
             except StorageEmpty:
                 if force_delete:

tests/unit/test_metasync.py

@@ -1,5 +1,7 @@
 from __future__ import annotations
 
+import asyncio
+
 import hypothesis.strategies as st
 import pytest
 import pytest_asyncio
@@ -33,7 +35,8 @@ async def test_basic(monkeypatch):
 
     await a.set_meta("foo", None)
     await metasync(a, b, status, keys=["foo"])
-    assert await a.get_meta("foo") is None and await b.get_meta("foo") is None
+    assert await a.get_meta("foo") is None
+    assert await b.get_meta("foo") is None
 
     await a.set_meta("foo", "bar")
     await metasync(a, b, status, keys=["foo"])
@@ -52,27 +55,24 @@ async def test_basic(monkeypatch):
 
     await b.set_meta("foo", None)
     await metasync(a, b, status, keys=["foo"])
-    assert not await a.get_meta("foo") and not await b.get_meta("foo")
+    assert not await a.get_meta("foo")
+    assert not await b.get_meta("foo")
 
 
 @pytest_asyncio.fixture
-@pytest.mark.asyncio
+async def conflict_state(request):
-async def conflict_state(request, event_loop):
     a = MemoryStorage()
     b = MemoryStorage()
     status = {}
     await a.set_meta("foo", "bar")
     await b.set_meta("foo", "baz")
 
-    def cleanup():
+    async def do_cleanup():
-        async def do_cleanup():
+        assert await a.get_meta("foo") == "bar"
-            assert await a.get_meta("foo") == "bar"
+        assert await b.get_meta("foo") == "baz"
-            assert await b.get_meta("foo") == "baz"
+        assert not status
-            assert not status
 
-        event_loop.run_until_complete(do_cleanup())
+    request.addfinalizer(lambda: asyncio.run(do_cleanup()))
-
-    request.addfinalizer(cleanup)
 
     return a, b, status
 

tests/unit/test_repair.py

@@ -17,7 +17,7 @@ from vdirsyncer.vobject import Item
 
 @given(uid=uid_strategy)
 # Using the random module for UIDs:
-@settings(suppress_health_check=HealthCheck.all())
+@settings(suppress_health_check=list(HealthCheck))
 @pytest.mark.asyncio
 async def test_repair_uids(uid):
     s = MemoryStorage()
@@ -40,12 +40,12 @@ async def test_repair_uids(uid):
 
 @given(uid=uid_strategy.filter(lambda x: not href_safe(x)))
 # Using the random module for UIDs:
-@settings(suppress_health_check=HealthCheck.all())
+@settings(suppress_health_check=list(HealthCheck))
 @pytest.mark.asyncio
 async def test_repair_unsafe_uids(uid):
     s = MemoryStorage()
     item = Item(f"BEGIN:VCARD\nUID:{uid}\nEND:VCARD")
-    href, etag = await s.upload(item)
+    href, _etag = await s.upload(item)
     assert (await s.get(href))[0].uid == uid
     assert not href_safe(uid)
 
@@ -58,7 +58,7 @@ async def test_repair_unsafe_uids(uid):
 
 
 @pytest.mark.parametrize(
-    "uid,href", [("b@dh0mbr3", "perfectly-fine"), ("perfectly-fine", "b@dh0mbr3")]
+    ("uid", "href"), [("b@dh0mbr3", "perfectly-fine"), ("perfectly-fine", "b@dh0mbr3")]
 )
 def test_repair_unsafe_href(uid, href):
     item = Item(f"BEGIN:VCARD\nUID:{uid}\nEND:VCARD")

tests/unit/test_retry.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+from unittest.mock import AsyncMock
+from unittest.mock import Mock
+
+import aiohttp
+import pytest
+
+from vdirsyncer.http import UsageLimitReached
+from vdirsyncer.http import request
+
+
+async def _create_mock_response(status: int, body: str | dict):
+    raw_body = body
+    text_body = json.dumps(body) if isinstance(body, dict) else body
+
+    mock_response = AsyncMock()
+    mock_response.status = status
+    mock_response.ok = 200 <= status < 300
+    mock_response.reason = "OK" if mock_response.ok else "Forbidden"
+    mock_response.headers = (
+        {"Content-Type": "application/json"}
+        if isinstance(raw_body, dict)
+        else {"Content-Type": "text/plain"}
+    )
+    mock_response.text.return_value = text_body
+    if isinstance(raw_body, dict):
+        mock_response.json.return_value = raw_body
+    else:
+        mock_response.json.side_effect = ValueError("Not JSON")
+    mock_response.raise_for_status = Mock(
+        side_effect=(
+            aiohttp.ClientResponseError(
+                request_info=AsyncMock(),
+                history=(),
+                status=status,
+                message=mock_response.reason,
+                headers=mock_response.headers,
+            )
+            if not mock_response.ok
+            else None
+        )
+    )
+
+    return mock_response
+
+
+@pytest.mark.asyncio
+async def test_request_retry_on_usage_limit():
+    url = "http://example.com/api"
+    max_retries = 5  # As configured in the @retry decorator
+
+    mock_session = AsyncMock()
+
+    # Simulate (max_retries - 1) 403 errors and then a 200 OK
+    mock_session.request.side_effect = [
+        await _create_mock_response(
+            403,
+            {
+                "error": {
+                    "errors": [{"domain": "usageLimits", "reason": "quotaExceeded"}]
+                }
+            },
+        )
+        for _ in range(max_retries - 1)
+    ] + [await _create_mock_response(200, "OK")]
+
+    async with (
+        aiohttp.ClientSession()
+    ):  # Dummy session. Will be replaced by mock_session at call
+        response = await request("GET", url, mock_session)
+
+        assert response.status == 200
+        assert mock_session.request.call_count == max_retries
+
+
+@pytest.mark.asyncio
+async def test_request_retry_exceeds_max_attempts():
+    url = "http://example.com/api"
+    max_retries = 5  # As configured in the @retry decorator
+
+    mock_session = AsyncMock()
+    # Simulate max_retries 403 errors and then a 200 OK
+    mock_session.request.side_effect = [
+        await _create_mock_response(
+            403,
+            {
+                "error": {
+                    "errors": [{"domain": "usageLimits", "reason": "quotaExceeded"}]
+                }
+            },
+        )
+        for _ in range(max_retries)
+    ]
+
+    async with (
+        aiohttp.ClientSession()
+    ):  # Dummy session. Will be replaced by mock_session at call
+        with pytest.raises(UsageLimitReached):
+            await request("GET", url, mock_session)
+        assert mock_session.request.call_count == max_retries
+
+
+@pytest.mark.asyncio
+async def test_request_no_retry_on_generic_403_json():
+    url = "http://example.com/api"
+
+    mock_session = AsyncMock()
+    # Generic non-Google 403 error payload (e.g., GitHub-style)
+    mock_session.request.side_effect = [
+        await _create_mock_response(403, {"message": "API rate limit exceeded"})
+    ]
+
+    async with aiohttp.ClientSession():
+        with pytest.raises(aiohttp.ClientResponseError):
+            await request("GET", url, mock_session)
+        # Should not retry because it's not the Google quotaExceeded shape
+        assert mock_session.request.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_request_no_retry_on_generic_403_text():
+    url = "http://example.com/api"
+
+    mock_session = AsyncMock()
+    # Plain-text 403 body mentioning rate limits, but not structured as Google error
+    mock_session.request.side_effect = [
+        await _create_mock_response(403, "Rate limit exceeded")
+    ]
+
+    async with aiohttp.ClientSession():
+        with pytest.raises(aiohttp.ClientResponseError):
+            await request("GET", url, mock_session)
+        # Should not retry because the JSON shape is not Google quotaExceeded
+        assert mock_session.request.call_count == 1

tests/unit/utils/test_vobject.py

@@ -25,7 +25,7 @@ _simple_split = [
 ]
 
 _simple_joined = "\r\n".join(
-    ["BEGIN:VADDRESSBOOK"] + _simple_split + ["END:VADDRESSBOOK\r\n"]
+    ["BEGIN:VADDRESSBOOK", *_simple_split, "END:VADDRESSBOOK\r\n"]
 )
 
 
@@ -124,7 +124,7 @@ def test_split_collection_timezones():
         "END:VTIMEZONE"
     )
 
-    full = "\r\n".join(["BEGIN:VCALENDAR"] + items + [timezone, "END:VCALENDAR"])
+    full = "\r\n".join(["BEGIN:VCALENDAR", *items, timezone, "END:VCALENDAR"])
 
     given = {normalize_item(item) for item in vobject.split_collection(full)}
     expected = {
@@ -154,7 +154,7 @@ def test_hash_item():
 
 
 def test_multiline_uid(benchmark):
-    a = "BEGIN:FOO\r\n" "UID:123456789abcd\r\n" " efgh\r\n" "END:FOO\r\n"
+    a = "BEGIN:FOO\r\nUID:123456789abcd\r\n efgh\r\nEND:FOO\r\n"
     assert benchmark(lambda: vobject.Item(a).uid) == "123456789abcdefgh"
 
 
@@ -299,7 +299,7 @@ def test_input_types():
 
 value_strategy = st.text(
     st.characters(
-        blacklist_categories=("Zs", "Zl", "Zp", "Cc", "Cs"), blacklist_characters=":="
+        exclude_categories=("Zs", "Zl", "Zp", "Cc", "Cs"), exclude_characters=":="
     ),
     min_size=1,
 ).filter(lambda x: x.strip() == x)
@@ -335,7 +335,8 @@ class VobjectMachine(RuleBasedStateMachine):
         assert key in c
         assert c.get(key) == value
         dump = "\r\n".join(c.dump_lines())
-        assert key in dump and value in dump
+        assert key in dump
+        assert value in dump
 
     @rule(
         c=Parsed,
@@ -365,6 +366,16 @@ class VobjectMachine(RuleBasedStateMachine):
 TestVobjectMachine = VobjectMachine.TestCase
 
 
+def test_dupe_consecutive_keys():
+    state = VobjectMachine()
+    unparsed_0 = state.get_unparsed_lines(encoded=False, joined=False)
+    parsed_0 = state.parse(unparsed=unparsed_0)
+    state.add_prop_raw(c=parsed_0, key="0", params=[], value="0")
+    state.add_prop_raw(c=parsed_0, key="0", params=[], value="0")
+    state.add_prop(c=parsed_0, key="0", value="1")
+    state.teardown()
+
+
 def test_component_contains():
     item = vobject._Component.parse(["BEGIN:FOO", "FOO:YES", "END:FOO"])
 
@@ -372,4 +383,4 @@ def test_component_contains():
     assert "BAZ" not in item
 
     with pytest.raises(ValueError):
-        42 in item  # noqa: B015
+        42 in item  # noqa: B015, this check raises.

vdirsyncer/__init__.py

@@ -9,7 +9,7 @@ BUGTRACKER_HOME = PROJECT_HOME + "/issues"
 DOCS_HOME = "https://vdirsyncer.pimutils.org/en/stable"
 
 try:
-    from .version import version as __version__  # noqa
+    from .version import version as __version__
 except ImportError:  # pragma: no cover
     raise ImportError(
         "Failed to find (autogenerated) version.py. "
@@ -17,12 +17,14 @@ except ImportError:  # pragma: no cover
         "use the PyPI ones."
     )
 
+__all__ = ["__version__"]
+
 
 def _check_python_version():
     import sys
 
-    if sys.version_info < (3, 7, 0):  # noqa: UP036
+    if sys.version_info < (3, 9, 0):  # noqa: UP036
-        print("vdirsyncer requires at least Python 3.7.")
+        print("vdirsyncer requires at least Python 3.9.")
         sys.exit(1)
 
 

vdirsyncer/cli/__init__.py

@@ -10,8 +10,8 @@ import aiohttp
 import click
 import click_log
 
-from .. import BUGTRACKER_HOME
+from vdirsyncer import BUGTRACKER_HOME
-from .. import __version__
+from vdirsyncer import __version__
 
 cli_logger = logging.getLogger(__name__)
 click_log.basic_config("vdirsyncer")
@@ -147,7 +147,14 @@ def sync(ctx, collections, force_delete):
                         )
                     )
 
-            await asyncio.gather(*tasks)
+            # `return_exceptions=True` ensures that the event loop lives long enough for
+            # backoffs to be able to finish
+            gathered = await asyncio.gather(*tasks, return_exceptions=True)
+            # but now we need to manually check for and propogate a single failure after
+            # allowing all tasks to finish in order to keep exit status non-zero
+            failures = [e for e in gathered if isinstance(e, BaseException)]
+            if failures:
+                raise failures[0]
 
     asyncio.run(main(collections))
 

vdirsyncer/cli/config.py

@@ -3,17 +3,18 @@ from __future__ import annotations
 import json
 import os
 import string
+from collections.abc import Generator
 from configparser import RawConfigParser
+from functools import cached_property
 from itertools import chain
 from typing import IO
 from typing import Any
-from typing import Generator
 
-from .. import PROJECT_HOME
+from vdirsyncer import PROJECT_HOME
-from .. import exceptions
+from vdirsyncer import exceptions
-from ..utils import cached_property
+from vdirsyncer.utils import expand_path
-from ..utils import expand_path
+from vdirsyncer.vobject import Item
-from ..vobject import Item
+
 from .fetchparams import expand_fetch_params
 from .utils import storage_class_from_config
 
@@ -92,7 +93,15 @@ def _validate_collections_param(collections):
                 raise ValueError("Duplicate value.")
             collection_names.add(collection_name)
         except ValueError as e:
-            raise ValueError(f"`collections` parameter, position {i}: {str(e)}")
+            raise ValueError(f"`collections` parameter, position {i}: {e!s}")
+
+
+def _validate_implicit_param(implicit):
+    if implicit is None:
+        return
+
+    if implicit != "create":
+        raise ValueError("`implicit` parameter must be 'create' or absent.")
 
 
 class _ConfigReader:
@@ -141,7 +150,7 @@ class _ConfigReader:
                     dict(_parse_options(self._parser.items(section), section=section)),
                 )
             except ValueError as e:
-                raise exceptions.UserError(f'Section "{section}": {str(e)}')
+                raise exceptions.UserError(f'Section "{section}": {e!s}')
 
         _validate_general_section(self._general)
         if getattr(self._file, "name", None):
@@ -229,9 +238,10 @@ class PairConfig:
         self.name: str = name
         self.name_a: str = options.pop("a")
         self.name_b: str = options.pop("b")
+        self.implicit = options.pop("implicit", None)
 
         self._partial_sync: str | None = options.pop("partial_sync", None)
-        self.metadata = options.pop("metadata", None) or ()
+        self.metadata: str | tuple[()] = options.pop("metadata", ())
 
         self.conflict_resolution = self._process_conflict_resolution_param(
             options.pop("conflict_resolution", None)
@@ -247,6 +257,7 @@ class PairConfig:
             )
         else:
             _validate_collections_param(self.collections)
+            _validate_implicit_param(self.implicit)
 
         if options:
             raise ValueError("Unknown options: {}".format(", ".join(options)))
@@ -256,7 +267,7 @@ class PairConfig:
     ):
         if conflict_resolution in (None, "a wins", "b wins"):
             return conflict_resolution
-        elif (
+        if (
             isinstance(conflict_resolution, list)
             and len(conflict_resolution) > 1
             and conflict_resolution[0] == "command"
@@ -270,8 +281,7 @@ class PairConfig:
                 return _resolve_conflict_via_command(a, b, command, a_name, b_name)
 
             return resolve
-        else:
+        raise ValueError("Invalid value for `conflict_resolution`.")
-            raise ValueError("Invalid value for `conflict_resolution`.")
 
     # The following parameters are lazily evaluated because evaluating
     # self.config_a would expand all `x.fetch` parameters. This is costly and
@@ -338,7 +348,7 @@ def _resolve_conflict_via_command(
     if _check_call is None:
         from subprocess import check_call as _check_call
 
-    from ..vobject import Item
+    from vdirsyncer.vobject import Item
 
     dir = tempfile.mkdtemp(prefix="vdirsyncer-conflict.")
     try:
@@ -351,7 +361,7 @@ def _resolve_conflict_via_command(
             f.write(b.raw)
 
         command[0] = expand_path(command[0])
-        _check_call(command + [a_tmp, b_tmp])
+        _check_call([*command, a_tmp, b_tmp])
 
         with open(a_tmp) as f:
             new_a = f.read()
@@ -359,7 +369,7 @@ def _resolve_conflict_via_command(
             new_b = f.read()
 
         if new_a != new_b:
-            raise exceptions.UserError("The two files are not completely " "equal.")
+            raise exceptions.UserError("The two files are not completely equal.")
         return Item(new_a)
     finally:
         shutil.rmtree(dir)

vdirsyncer/cli/discover.py

@@ -9,7 +9,8 @@ import sys
 import aiohttp
 import aiostream
 
-from .. import exceptions
+from vdirsyncer import exceptions
+
 from .utils import handle_collection_not_found
 from .utils import handle_storage_init_error
 from .utils import load_status
@@ -59,22 +60,20 @@ async def collections_for_pair(
     cache_key = _get_collections_cache_key(pair)
     if from_cache:
         rv = load_status(status_path, pair.name, data_type="collections")
-        if rv.get("cache_key", None) == cache_key:
+        if rv and rv.get("cache_key", None) == cache_key:
             return list(
                 _expand_collections_cache(
                     rv["collections"], pair.config_a, pair.config_b
                 )
             )
-        elif rv:
+        if rv:
             raise exceptions.UserError(
                 "Detected change in config file, "
                 f"please run `vdirsyncer discover {pair.name}`."
             )
-        else:
+        raise exceptions.UserError(
-            raise exceptions.UserError(
+            f"Please run `vdirsyncer discover {pair.name}`  before synchronization."
-                f"Please run `vdirsyncer discover {pair.name}` "
+        )
-                " before synchronization."
-            )
 
     logger.info(f"Discovering collections for pair {pair.name}")
 
@@ -94,16 +93,23 @@ async def collections_for_pair(
             connector=connector,
         )
 
+    async def _handle_collection_not_found(
+        config, collection, e=None, implicit_create=False
+    ):
+        return await handle_collection_not_found(
+            config, collection, e=e, implicit_create=pair.implicit == "create"
+        )
+
     # We have to use a list here because the special None/null value would get
     # mangled to string (because JSON objects always have string keys).
-    rv = await aiostream.stream.list(
+    rv = await aiostream.stream.list(  #  type: ignore[assignment]
         expand_collections(
             shortcuts=pair.collections,
             config_a=pair.config_a,
             config_b=pair.config_b,
             get_a_discovered=a_discovered.get_self,
             get_b_discovered=b_discovered.get_self,
-            _handle_collection_not_found=handle_collection_not_found,
+            _handle_collection_not_found=_handle_collection_not_found,
         )
     )
 

vdirsyncer/cli/fetchparams.py

@@ -4,9 +4,10 @@ import logging
 
 import click
 
-from .. import exceptions
+from vdirsyncer import exceptions
-from ..utils import expand_path
+from vdirsyncer.utils import expand_path
-from ..utils import synchronized
+from vdirsyncer.utils import synchronized
+
 from . import AppContext
 
 SUFFIX = ".fetch"
@@ -87,7 +88,7 @@ def _strategy_command(*command: str, shell: bool = False):
         return stdout.strip("\n")
     except OSError as e:
         cmd = " ".join(expanded_command)
-        raise exceptions.UserError(f"Failed to execute command: {cmd}\n{str(e)}")
+        raise exceptions.UserError(f"Failed to execute command: {cmd}\n{e!s}")
 
 
 def _strategy_shell(*command: str):

vdirsyncer/cli/tasks.py

@@ -4,8 +4,9 @@ import json
 
 import aiohttp
 
-from .. import exceptions
+from vdirsyncer import exceptions
-from .. import sync
+from vdirsyncer import sync
+
 from .config import CollectionConfig
 from .discover import DiscoverResult
 from .discover import collections_for_pair
@@ -35,10 +36,8 @@ async def prepare_pair(pair_name, collections, config, *, connector):
             config_a, config_b = all_collections[collection_name]
         except KeyError:
             raise exceptions.UserError(
-                "Pair {}: Collection {} not found. These are the "
+                f"Pair {pair_name}: Collection {json.dumps(collection_name)} not found."
-                "configured collections:\n{}".format(
+                f"These are the configured collections:\n{list(all_collections)}"
-                    pair_name, json.dumps(collection_name), list(all_collections)
-                )
             )
 
         collection = CollectionConfig(pair, collection_name, config_a, config_b)
@@ -105,7 +104,7 @@ async def repair_collection(
     *,
     connector: aiohttp.TCPConnector,
 ):
-    from ..repair import repair_storage
+    from vdirsyncer.repair import repair_storage
 
     storage_name, collection = collection, None
     if "/" in storage_name:
@@ -136,7 +135,7 @@ async def repair_collection(
 
 
 async def metasync_collection(collection, general, *, connector: aiohttp.TCPConnector):
-    from ..metasync import metasync
+    from vdirsyncer.metasync import metasync
 
     pair = collection.pair
     status_name = get_status_name(pair.name, collection.name)

vdirsyncer/cli/utils.py

@@ -10,19 +10,20 @@ from typing import Any
 
 import aiohttp
 import click
-from atomicwrites import atomic_write
 
-from .. import BUGTRACKER_HOME
+from vdirsyncer import BUGTRACKER_HOME
-from .. import DOCS_HOME
+from vdirsyncer import DOCS_HOME
-from .. import exceptions
+from vdirsyncer import exceptions
-from ..storage.base import Storage
+from vdirsyncer.storage.base import Storage
-from ..sync.exceptions import IdentConflict
+from vdirsyncer.sync.exceptions import IdentConflict
-from ..sync.exceptions import PartialSync
+from vdirsyncer.sync.exceptions import PartialSync
-from ..sync.exceptions import StorageEmpty
+from vdirsyncer.sync.exceptions import StorageEmpty
-from ..sync.exceptions import SyncConflict
+from vdirsyncer.sync.exceptions import SyncConflict
-from ..sync.status import SqliteStatus
+from vdirsyncer.sync.status import SqliteStatus
-from ..utils import expand_path
+from vdirsyncer.utils import atomic_write
-from ..utils import get_storage_init_args
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import get_storage_init_args
+
 from . import cli_logger
 
 STATUS_PERMISSIONS = 0o600
@@ -30,7 +31,7 @@ STATUS_DIR_PERMISSIONS = 0o700
 
 
 class _StorageIndex:
-    def __init__(self):
+    def __init__(self) -> None:
         self._storages: dict[str, str] = {
             "caldav": "vdirsyncer.storage.dav.CalDAVStorage",
             "carddav": "vdirsyncer.storage.dav.CardDAVStorage",
@@ -78,13 +79,11 @@ def handle_cli_error(status_name=None, e=None):
         cli_logger.critical(e)
     except StorageEmpty as e:
         cli_logger.error(
-            '{status_name}: Storage "{name}" was completely emptied. If you '
+            f'{status_name}: Storage "{e.empty_storage.instance_name}" was '
-            "want to delete ALL entries on BOTH sides, then use "
+            "completely emptied. If you want to delete ALL entries on BOTH sides,"
-            "`vdirsyncer sync --force-delete {status_name}`. "
+            f"then use `vdirsyncer sync --force-delete {status_name}`. "
-            "Otherwise delete the files for {status_name} in your status "
+            f"Otherwise delete the files for {status_name} in your status "
-            "directory.".format(
+            "directory."
-                name=e.empty_storage.instance_name, status_name=status_name
-            )
         )
     except PartialSync as e:
         cli_logger.error(
@@ -232,7 +231,8 @@ def manage_sync_status(base_path: str, pair_name: str, collection_name: str):
         prepare_status_path(path)
         status = SqliteStatus(path)
 
-    yield status
+    with contextlib.closing(status):
+        yield status
 
 
 def save_status(
@@ -286,15 +286,14 @@ async def storage_instance_from_config(
     except exceptions.CollectionNotFound as e:
         if create:
             config = await handle_collection_not_found(
-                config, config.get("collection", None), e=str(e)
+                config, config.get("collection", None), e=str(e), implicit_create=True
             )
             return await storage_instance_from_config(
                 config,
                 create=False,
                 connector=connector,
             )
-        else:
+        raise
-            raise
     except Exception:
         return handle_storage_init_error(cls, new_config)
 
@@ -342,7 +341,9 @@ def assert_permissions(path: str, wanted: int) -> None:
         os.chmod(path, wanted)
 
 
-async def handle_collection_not_found(config, collection, e=None):
+async def handle_collection_not_found(
+    config, collection, e=None, implicit_create=False
+):
     storage_name = config.get("instance_name", None)
 
     cli_logger.warning(
@@ -351,7 +352,7 @@ async def handle_collection_not_found(config, collection, e=None):
         )
     )
 
-    if click.confirm("Should vdirsyncer attempt to create it?"):
+    if implicit_create or click.confirm("Should vdirsyncer attempt to create it?"):
         storage_type = config["type"]
         cls, config = storage_class_from_config(config)
         config["collection"] = collection

vdirsyncer/http.py

@@ -1,14 +1,24 @@
 from __future__ import annotations
 
+import asyncio
 import logging
+import os
+import platform
 import re
-from abc import ABC, abstractmethod
+from abc import ABC
+from abc import abstractmethod
 from base64 import b64encode
 from ssl import create_default_context
 
 import aiohttp
 import requests.auth
+from aiohttp import ServerDisconnectedError
+from aiohttp import ServerTimeoutError
 from requests.utils import parse_dict_header
+from tenacity import retry
+from tenacity import retry_if_exception_type
+from tenacity import stop_after_attempt
+from tenacity import wait_exponential
 
 from . import __version__
 from . import exceptions
@@ -17,6 +27,13 @@ from .utils import expand_path
 logger = logging.getLogger(__name__)
 USERAGENT = f"vdirsyncer/{__version__}"
 
+# 'hack' to prevent aiohttp from loading the netrc config,
+# but still allow it to read PROXY_* env vars.
+# Otherwise, if our host is defined in the netrc config,
+# aiohttp will overwrite our Authorization header.
+# https://github.com/pimutils/vdirsyncer/issues/1138
+os.environ["NETRC"] = "NUL" if platform.system() == "Windows" else "/dev/null"
+
 
 class AuthMethod(ABC):
     def __init__(self, username, password):
@@ -34,7 +51,11 @@ class AuthMethod(ABC):
     def __eq__(self, other):
         if not isinstance(other, AuthMethod):
             return False
-        return self.__class__ == other.__class__ and self.username == other.username and self.password == other.password
+        return (
+            self.__class__ == other.__class__
+            and self.username == other.username
+            and self.password == other.password
+        )
 
 
 class BasicAuthMethod(AuthMethod):
@@ -43,20 +64,19 @@ class BasicAuthMethod(AuthMethod):
 
     def get_auth_header(self, _method, _url):
         auth_str = f"{self.username}:{self.password}"
-        return "Basic " + b64encode(auth_str.encode('utf-8')).decode("utf-8")
+        return "Basic " + b64encode(auth_str.encode("utf-8")).decode("utf-8")
 
 
 class DigestAuthMethod(AuthMethod):
     # make class var to 'cache' the state, which is more efficient because otherwise
     # each request would first require another 'initialization' request.
-    _auth_helpers = {}
+    _auth_helpers: dict[tuple[str, str], requests.auth.HTTPDigestAuth] = {}
 
-    def __init__(self, username, password):
+    def __init__(self, username: str, password: str):
         super().__init__(username, password)
 
         self._auth_helper = self._auth_helpers.get(
-            (username, password),
+            (username, password), requests.auth.HTTPDigestAuth(username, password)
-            requests.auth.HTTPDigestAuth(username, password)
         )
         self._auth_helpers[(username, password)] = self._auth_helper
 
@@ -78,7 +98,7 @@ class DigestAuthMethod(AuthMethod):
 
         if not self.auth_helper_vars.chal:
             # Need to do init request first
-            return ''
+            return ""
 
         return self._auth_helper.build_digest_header(method, url)
 
@@ -87,13 +107,16 @@ def prepare_auth(auth, username, password):
     if username and password:
         if auth == "basic" or auth is None:
             return BasicAuthMethod(username, password)
-        elif auth == "digest":
+        if auth == "digest":
             return DigestAuthMethod(username, password)
-        elif auth == "guess":
+        if auth == "guess":
-            raise exceptions.UserError(f"'Guess' authentication is not supported in this version of vdirsyncer. \n"
+            raise exceptions.UserError(
-                                       f"Please explicitly specify either 'basic' or 'digest' auth instead. \n"
+                "'Guess' authentication is not supported in this version of "
-                                       f"See the following issue for more information: "
+                "vdirsyncer.\n"
-                                       f"https://github.com/pimutils/vdirsyncer/issues/1015")
+                "Please explicitly specify either 'basic' or 'digest' auth instead. \n"
+                "See the following issue for more information: "
+                "https://github.com/pimutils/vdirsyncer/issues/1015"
+            )
         else:
             raise exceptions.UserError(f"Unknown authentication method: {auth}")
     elif auth:
@@ -132,6 +155,64 @@ def prepare_client_cert(cert):
     return cert
 
 
+class TransientNetworkError(exceptions.Error):
+    """Transient network condition that should be retried."""
+
+
+def _is_safe_to_retry_method(method: str) -> bool:
+    """Returns True if the HTTP method is safe/idempotent to retry.
+
+    We consider these safe for our WebDAV usage:
+    - GET, HEAD, OPTIONS: standard safe methods
+    - PROPFIND, REPORT: read-only DAV queries used for listing/fetching
+    """
+    return method.upper() in {"GET", "HEAD", "OPTIONS", "PROPFIND", "REPORT"}
+
+
+class UsageLimitReached(exceptions.Error):
+    pass
+
+
+async def _is_quota_exceeded_google(response: aiohttp.ClientResponse) -> bool:
+    """Return True if the response JSON indicates Google-style `usageLimits` exceeded.
+
+    Expected shape:
+    {"error": {"errors": [{"domain": "usageLimits", ...}], ...}}
+
+    See https://developers.google.com/workspace/calendar/api/guides/errors#403_usage_limits_exceeded
+    """
+    try:
+        data = await response.json(content_type=None)
+    except Exception:
+        return False
+
+    if not isinstance(data, dict):
+        return False
+
+    error = data.get("error")
+    if not isinstance(error, dict):
+        return False
+
+    errors = error.get("errors")
+    if not isinstance(errors, list):
+        return False
+
+    for entry in errors:
+        if isinstance(entry, dict) and entry.get("domain") == "usageLimits":
+            return True
+
+    return False
+
+
+@retry(
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    retry=(
+        retry_if_exception_type(UsageLimitReached)
+        | retry_if_exception_type(TransientNetworkError)
+    ),
+    reraise=True,
+)
 async def request(
     method,
     url,
@@ -165,7 +246,7 @@ async def request(
     logger.debug("=" * 20)
     logger.debug(f"{method} {url}")
     logger.debug(kwargs.get("headers", {}))
-    logger.debug(kwargs.get("data", None))
+    logger.debug(kwargs.get("data"))
     logger.debug("Sending request...")
 
     assert isinstance(kwargs.get("data", b""), bytes)
@@ -177,23 +258,43 @@ async def request(
         kwargs["ssl"] = ssl_context
 
     headers = kwargs.pop("headers", {})
-    num_401 = 0
+    response: aiohttp.ClientResponse | None = None
-    while num_401 < 2:
+    for _attempt in range(2):
         if auth:
             headers["Authorization"] = auth.get_auth_header(method, url)
-        response = await session.request(method, url, headers=headers, **kwargs)
+        try:
+            response = await session.request(method, url, headers=headers, **kwargs)
+        except (
+            ServerDisconnectedError,
+            ServerTimeoutError,
+            asyncio.TimeoutError,
+        ) as e:
+            # Retry only if the method is safe/idempotent for our DAV use
+            if _is_safe_to_retry_method(method):
+                logger.debug(
+                    f"Transient network error on {method} {url}: {e}. Will retry."
+                )
+                raise TransientNetworkError(str(e)) from e
+            raise e from None
+
+        if response is None:
+            raise RuntimeError("No HTTP response obtained")
 
         if response.ok or not auth:
             # we don't need to do the 401-loop if we don't do auth in the first place
             break
 
         if response.status == 401:
-            num_401 += 1
             auth.handle_401(response)
+            # retry once more after handling the 401 challenge
+            continue
         else:
             # some other error, will be handled later on
             break
 
+    if response is None:
+        raise RuntimeError("No HTTP response obtained")
+
     # See https://github.com/kennethreitz/requests/issues/2042
     content_type = response.headers.get("Content-Type", "")
     if (
@@ -208,10 +309,18 @@ async def request(
     logger.debug(response.headers)
     logger.debug(response.content)
 
+    if logger.getEffectiveLevel() <= logging.DEBUG and response.status >= 400:
+        # https://github.com/pimutils/vdirsyncer/issues/1186
+        logger.debug(await response.text())
+
+    if response.status == 403 and await _is_quota_exceeded_google(response):
+        raise UsageLimitReached(response.reason)
     if response.status == 412:
         raise exceptions.PreconditionFailed(response.reason)
     if response.status in (404, 410):
         raise exceptions.NotFoundError(response.reason)
+    if response.status == 429:
+        raise UsageLimitReached(response.reason)
 
     response.raise_for_status()
     return response

vdirsyncer/metasync.py

@@ -57,7 +57,7 @@ async def metasync(storage_a, storage_b, status, keys, conflict_resolution=None)
         logger.debug(f"B: {b}")
         logger.debug(f"S: {s}")
 
-        if a != s and b != s or storage_a.read_only or storage_b.read_only:
+        if (a != s and b != s) or storage_a.read_only or storage_b.read_only:
             await _resolve_conflict()
         elif a != s and b == s:
             await _a_to_b()

vdirsyncer/repair.py

@@ -56,9 +56,7 @@ def repair_item(href, item, seen_uids, repair_unsafe_uid):
         new_item = item.with_uid(generate_href())
     elif not href_safe(item.uid) or not href_safe(basename(href)):
         if not repair_unsafe_uid:
-            logger.warning(
+            logger.warning("UID may cause problems, add --repair-unsafe-uid to repair.")
-                "UID may cause problems, add " "--repair-unsafe-uid to repair."
-            )
         else:
             logger.warning("UID or href is unsafe, assigning random UID.")
             new_item = item.with_uid(generate_href())

vdirsyncer/storage/base.py

@@ -4,13 +4,12 @@ import contextlib
 import functools
 from abc import ABCMeta
 from abc import abstractmethod
-from typing import Iterable
+from collections.abc import Iterable
 
+from vdirsyncer import exceptions
+from vdirsyncer.utils import uniq
 from vdirsyncer.vobject import Item
 
-from .. import exceptions
-from ..utils import uniq
-
 
 def mutating_storage_method(f):
     """Wrap a method and fail if the instance is readonly."""
@@ -76,7 +75,7 @@ class Storage(metaclass=StorageMeta):
     read_only = False
 
     # The attribute values to show in the representation of the storage.
-    _repr_attributes: list[str] = []
+    _repr_attributes: tuple[str, ...] = ()
 
     def __init__(
         self,
@@ -141,10 +140,8 @@ class Storage(metaclass=StorageMeta):
         except ValueError:
             pass
 
-        return "<{}(**{})>".format(
+        attrs = {x: getattr(self, x) for x in self._repr_attributes}
-            self.__class__.__name__,
+        return f"<{self.__class__.__name__}(**{attrs})>"
-            {x: getattr(self, x) for x in self._repr_attributes},
-        )
 
     @abstractmethod
     async def list(self) -> list[tuple]:
@@ -153,7 +150,7 @@ class Storage(metaclass=StorageMeta):
         """
 
     @abstractmethod
-    async def get(self, href: str):
+    async def get(self, href: str) -> tuple[Item, str]:
         """Fetch a single item.
 
         :param href: href to fetch

vdirsyncer/storage/dav.py

@@ -1,26 +1,28 @@
 from __future__ import annotations
 
+import contextlib
 import datetime
 import logging
 import urllib.parse as urlparse
 import xml.etree.ElementTree as etree
 from abc import abstractmethod
+from functools import cached_property
 from inspect import getfullargspec
 from inspect import signature
 
 import aiohttp
 import aiostream
 
+from vdirsyncer import exceptions
+from vdirsyncer import http
+from vdirsyncer import utils
 from vdirsyncer.exceptions import Error
+from vdirsyncer.http import USERAGENT
+from vdirsyncer.http import prepare_auth
+from vdirsyncer.http import prepare_client_cert
+from vdirsyncer.http import prepare_verify
 from vdirsyncer.vobject import Item
 
-from .. import exceptions
-from .. import http
-from .. import utils
-from ..http import USERAGENT
-from ..http import prepare_auth
-from ..http import prepare_client_cert
-from ..http import prepare_verify
 from .base import Storage
 from .base import normalize_meta_value
 
@@ -113,10 +115,8 @@ def _fuzzy_matches_mimetype(strict, weak):
     if strict is None or weak is None:
         return True
 
-    mediatype, subtype = strict.split("/")
+    _mediatype, subtype = strict.split("/")
-    if subtype in weak:
+    return subtype in weak
-        return True
-    return False
 
 
 class Discover:
@@ -219,10 +219,8 @@ class Discover:
 
     async def find_collections(self):
         rv = None
-        try:
+        with contextlib.suppress(aiohttp.ClientResponseError, exceptions.Error):
             rv = await aiostream.stream.list(self._find_collections_impl(""))
-        except (aiohttp.ClientResponseError, exceptions.Error):
-            pass
 
         if rv:
             return rv
@@ -237,7 +235,7 @@ class Discover:
             return True
 
         props = _merge_xml(response.findall("{DAV:}propstat/{DAV:}prop"))
-        if props is None or not len(props):
+        if props is None or not props:
             dav_logger.debug("Skipping, missing <prop>: %s", response)
             return False
         if props.find("{DAV:}resourcetype/" + self._resourcetype) is None:
@@ -261,7 +259,7 @@ class Discover:
 
             href = response.find("{DAV:}href")
             if href is None:
-                raise InvalidXMLResponse("Missing href tag for collection " "props.")
+                raise InvalidXMLResponse("Missing href tag for collection props.")
             href = urlparse.urljoin(str(r.url), href.text)
             if href not in done:
                 done.add(href)
@@ -310,9 +308,7 @@ class Discover:
             </mkcol>
         """.format(
             etree.tostring(etree.Element(self._resourcetype), encoding="unicode")
-        ).encode(
+        ).encode("utf-8")
-            "utf-8"
-        )
 
         response = await self.session.request(
             "MKCOL",
@@ -325,7 +321,7 @@ class Discover:
 
 class CalDiscover(Discover):
     _namespace = "urn:ietf:params:xml:ns:caldav"
-    _resourcetype = "{%s}calendar" % _namespace
+    _resourcetype = f"{{{_namespace}}}calendar"
     _homeset_xml = b"""
     <propfind xmlns="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
         <prop>
@@ -333,13 +329,13 @@ class CalDiscover(Discover):
         </prop>
     </propfind>
     """
-    _homeset_tag = "{%s}calendar-home-set" % _namespace
+    _homeset_tag = f"{{{_namespace}}}calendar-home-set"
     _well_known_uri = "/.well-known/caldav"
 
 
 class CardDiscover(Discover):
     _namespace = "urn:ietf:params:xml:ns:carddav"
-    _resourcetype: str | None = "{%s}addressbook" % _namespace
+    _resourcetype: str | None = f"{{{_namespace}}}addressbook"
     _homeset_xml = b"""
     <propfind xmlns="DAV:" xmlns:c="urn:ietf:params:xml:ns:carddav">
         <prop>
@@ -347,7 +343,7 @@ class CardDiscover(Discover):
         </prop>
     </propfind>
     """
-    _homeset_tag = "{%s}addressbook-home-set" % _namespace
+    _homeset_tag = f"{{{_namespace}}}addressbook-home-set"
     _well_known_uri = "/.well-known/carddav"
 
 
@@ -395,7 +391,7 @@ class DAVSession:
         self.url = url.rstrip("/") + "/"
         self.connector = connector
 
-    @utils.cached_property
+    @cached_property
     def parsed_url(self):
         return urlparse.urlparse(self.url)
 
@@ -456,7 +452,7 @@ class DAVStorage(Storage):
 
     connector: aiohttp.TCPConnector
 
-    _repr_attributes = ["username", "url"]
+    _repr_attributes = ("username", "url")
 
     _property_table = {
         "displayname": ("displayname", "DAV:"),
@@ -501,8 +497,12 @@ class DAVStorage(Storage):
     def _is_item_mimetype(self, mimetype):
         return _fuzzy_matches_mimetype(self.item_mimetype, mimetype)
 
-    async def get(self, href: str):
+    async def get(self, href: str) -> tuple[Item, str]:
-        ((actual_href, item, etag),) = await aiostream.stream.list(
+        actual_href: str
+        item: Item
+        etag: str
+
+        ((actual_href, item, etag),) = await aiostream.stream.list(  # type: ignore[misc]
             self.get_multi([href])
         )
         assert href == actual_href
@@ -628,7 +628,7 @@ class DAVStorage(Storage):
                 continue
 
             props = response.findall("{DAV:}propstat/{DAV:}prop")
-            if props is None or not len(props):
+            if props is None or not props:
                 dav_logger.debug(f"Skipping {href!r}, properties are missing.")
                 continue
             else:
@@ -740,9 +740,7 @@ class DAVStorage(Storage):
         """.format(
             etree.tostring(element, encoding="unicode"),
             action=action,
-        ).encode(
+        ).encode("utf-8")
-            "utf-8"
-        )
 
         await self.session.request(
             "PROPPATCH",
@@ -796,7 +794,7 @@ class CalDAVStorage(DAVStorage):
         self.item_types = tuple(item_types)
         if (start_date is None) != (end_date is None):
             raise exceptions.UserError(
-                "If start_date is given, " "end_date has to be given too."
+                "If start_date is given, end_date has to be given too."
             )
         elif start_date is not None and end_date is not None:
             namespace = dict(datetime.__dict__)

vdirsyncer/storage/filesystem.py

@@ -1,18 +1,19 @@
 from __future__ import annotations
 
+import contextlib
 import errno
 import logging
 import os
 import subprocess
 
-from atomicwrites import atomic_write
+from vdirsyncer import exceptions
+from vdirsyncer.utils import atomic_write
+from vdirsyncer.utils import checkdir
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import generate_href
+from vdirsyncer.utils import get_etag_from_file
+from vdirsyncer.vobject import Item
 
-from .. import exceptions
-from ..utils import checkdir
-from ..utils import expand_path
-from ..utils import generate_href
-from ..utils import get_etag_from_file
-from ..vobject import Item
 from .base import Storage
 from .base import normalize_meta_value
 
@@ -21,7 +22,7 @@ logger = logging.getLogger(__name__)
 
 class FilesystemStorage(Storage):
     storage_name = "filesystem"
-    _repr_attributes = ["path"]
+    _repr_attributes = ("path",)
 
     def __init__(
         self,
@@ -65,9 +66,7 @@ class FilesystemStorage(Storage):
     def _validate_collection(cls, path):
         if not os.path.isdir(path):
             return False
-        if os.path.basename(path).startswith("."):
+        return not os.path.basename(path).startswith(".")
-            return False
-        return True
 
     @classmethod
     async def create_collection(cls, collection, **kwargs):
@@ -99,7 +98,7 @@ class FilesystemStorage(Storage):
             ):
                 yield fname, get_etag_from_file(fpath)
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         fpath = self._get_filepath(href)
         try:
             with open(fpath, "rb") as f:
@@ -178,7 +177,7 @@ class FilesystemStorage(Storage):
         try:
             subprocess.call([self.post_hook, fpath])
         except OSError as e:
-            logger.warning(f"Error executing external hook: {str(e)}")
+            logger.warning(f"Error executing external hook: {e!s}")
 
     def _run_pre_deletion_hook(self, fpath):
         logger.info(
@@ -187,7 +186,7 @@ class FilesystemStorage(Storage):
         try:
             subprocess.call([self.pre_deletion_hook, fpath])
         except OSError as e:
-            logger.warning(f"Error executing external hook: {str(e)}")
+            logger.warning(f"Error executing external hook: {e!s}")
 
     async def get_meta(self, key):
         fpath = os.path.join(self.path, key)
@@ -205,10 +204,8 @@ class FilesystemStorage(Storage):
 
         fpath = os.path.join(self.path, key)
         if value is None:
-            try:
+            with contextlib.suppress(OSError):
                 os.remove(fpath)
-            except OSError:
-                pass
         else:
             with atomic_write(fpath, mode="wb", overwrite=True) as f:
                 f.write(value.encode(self.encoding))

vdirsyncer/storage/google.py

@@ -11,12 +11,13 @@ from threading import Thread
 
 import aiohttp
 import click
-from atomicwrites import atomic_write
 
-from .. import exceptions
+from vdirsyncer import exceptions
-from ..utils import checkdir
+from vdirsyncer.utils import atomic_write
-from ..utils import expand_path
+from vdirsyncer.utils import checkdir
-from ..utils import open_graphical_browser
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import open_graphical_browser
+
 from . import base
 from . import dav
 from .google_helpers import _RedirectWSGIApp
@@ -98,6 +99,7 @@ class GoogleSession(dav.DAVSession):
             token_updater=self._save_token,
             connector=self.connector,
             connector_owner=False,
+            trust_env=True,
         )
 
     async def _init_token(self):
@@ -128,7 +130,7 @@ class GoogleSession(dav.DAVSession):
             async with self._session as session:
                 # Fail fast if the address is occupied
 
-                authorization_url, state = session.authorization_url(
+                authorization_url, _state = session.authorization_url(
                     TOKEN_URL,
                     # access_type and approval_prompt are Google specific
                     # extra parameters.

vdirsyncer/storage/google_helpers.py

@@ -7,9 +7,9 @@ from __future__ import annotations
 import logging
 import wsgiref.simple_server
 import wsgiref.util
+from collections.abc import Iterable
 from typing import Any
 from typing import Callable
-from typing import Iterable
 
 logger = logging.getLogger(__name__)
 

vdirsyncer/storage/http.py

@@ -1,24 +1,29 @@
 from __future__ import annotations
 
+import logging
+import subprocess
 import urllib.parse as urlparse
 
 import aiohttp
 
-from .. import exceptions
+from vdirsyncer import exceptions
-from ..http import USERAGENT
+from vdirsyncer.http import USERAGENT
-from ..http import prepare_auth
+from vdirsyncer.http import prepare_auth
-from ..http import prepare_client_cert
+from vdirsyncer.http import prepare_client_cert
-from ..http import prepare_verify
+from vdirsyncer.http import prepare_verify
-from ..http import request
+from vdirsyncer.http import request
-from ..vobject import Item
+from vdirsyncer.vobject import Item
-from ..vobject import split_collection
+from vdirsyncer.vobject import split_collection
+
 from .base import Storage
 
+logger = logging.getLogger(__name__)
+
 
 class HttpStorage(Storage):
     storage_name = "http"
     read_only = True
-    _repr_attributes = ["username", "url"]
+    _repr_attributes = ("username", "url")
     _items = None
 
     # Required for tests.
@@ -34,6 +39,7 @@ class HttpStorage(Storage):
         useragent=USERAGENT,
         verify_fingerprint=None,
         auth_cert=None,
+        filter_hook=None,
         *,
         connector,
         **kwargs,
@@ -56,6 +62,7 @@ class HttpStorage(Storage):
         self.useragent = useragent
         assert connector is not None
         self.connector = connector
+        self._filter_hook = filter_hook
 
         collection = kwargs.get("collection")
         if collection is not None:
@@ -66,6 +73,19 @@ class HttpStorage(Storage):
     def _default_headers(self):
         return {"User-Agent": self.useragent}
 
+    def _run_filter_hook(self, raw_item):
+        try:
+            result = subprocess.run(
+                [self._filter_hook],
+                input=raw_item,
+                capture_output=True,
+                encoding="utf-8",
+            )
+            return result.stdout
+        except OSError as e:
+            logger.warning(f"Error executing external command: {e!s}")
+            return raw_item
+
     async def list(self):
         async with aiohttp.ClientSession(
             connector=self.connector,
@@ -82,8 +102,13 @@ class HttpStorage(Storage):
             )
         self._items = {}
 
-        for item in split_collection((await r.read()).decode("utf-8")):
+        for raw_item in split_collection((await r.read()).decode("utf-8")):
-            item = Item(item)
+            if self._filter_hook:
+                raw_item = self._run_filter_hook(raw_item)
+            if not raw_item:
+                continue
+
+            item = Item(raw_item)
             if self._ignore_uids:
                 item = item.with_uid(item.hash)
 
@@ -92,11 +117,12 @@ class HttpStorage(Storage):
         for href, (_, etag) in self._items.items():
             yield href, etag
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         if self._items is None:
             async for _ in self.list():
                 pass
 
+        assert self._items is not None  # type assertion
         try:
             return self._items[href]
         except KeyError:

vdirsyncer/storage/memory.py

@@ -2,7 +2,9 @@ from __future__ import annotations
 
 import random
 
-from .. import exceptions
+from vdirsyncer import exceptions
+from vdirsyncer.vobject import Item
+
 from .base import Storage
 from .base import normalize_meta_value
 
@@ -33,7 +35,7 @@ class MemoryStorage(Storage):
         for href, (etag, _item) in self.items.items():
             yield href, etag
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         etag, item = self.items[href]
         return item, etag
 

vdirsyncer/storage/singlefile.py

@@ -6,24 +6,31 @@ import functools
 import glob
 import logging
 import os
-from typing import Iterable
+from collections.abc import Iterable
 
-from atomicwrites import atomic_write
+from vdirsyncer import exceptions
+from vdirsyncer.utils import atomic_write
+from vdirsyncer.utils import checkfile
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import get_etag_from_file
+from vdirsyncer.utils import uniq
+from vdirsyncer.vobject import Item
+from vdirsyncer.vobject import join_collection
+from vdirsyncer.vobject import split_collection
 
-from .. import exceptions
-from ..utils import checkfile
-from ..utils import expand_path
-from ..utils import get_etag_from_file
-from ..utils import uniq
-from ..vobject import Item
-from ..vobject import join_collection
-from ..vobject import split_collection
 from .base import Storage
 
 logger = logging.getLogger(__name__)
 
 
 def _writing_op(f):
+    """Implement at_once for write operations.
+
+    Wrap an operation which writes to the storage, implementing `at_once` if it has been
+    requested. Changes are stored in-memory until the at_once block finishes, at which
+    time they are all written at once.
+    """
+
     @functools.wraps(f)
     async def inner(self, *args, **kwargs):
         if self._items is None or not self._at_once:
@@ -40,7 +47,7 @@ def _writing_op(f):
 
 class SingleFileStorage(Storage):
     storage_name = "singlefile"
-    _repr_attributes = ["path"]
+    _repr_attributes = ("path",)
 
     _write_mode = "wb"
     _append_mode = "ab"
@@ -95,7 +102,7 @@ class SingleFileStorage(Storage):
                 path = path % (collection,)
             except TypeError:
                 raise ValueError(
-                    "Exactly one %s required in path " "if collection is not null."
+                    "Exactly one %s required in path if collection is not null."
                 )
 
         checkfile(path, create=True)
@@ -126,11 +133,12 @@ class SingleFileStorage(Storage):
 
                 yield href, etag
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         if self._items is None or not self._at_once:
             async for _ in self.list():
                 pass
 
+        assert self._items is not None  # type assertion
         try:
             return self._items[href]
         except KeyError:

vdirsyncer/sync/__init__.py

@@ -16,11 +16,11 @@ import contextlib
 import itertools
 import logging
 
+from vdirsyncer.exceptions import UserError
 from vdirsyncer.storage.base import Storage
+from vdirsyncer.utils import uniq
 from vdirsyncer.vobject import Item
 
-from ..exceptions import UserError
-from ..utils import uniq
 from .exceptions import BothReadOnly
 from .exceptions import IdentAlreadyExists
 from .exceptions import PartialSync
@@ -136,9 +136,13 @@ async def sync(
         raise BothReadOnly
 
     if conflict_resolution == "a wins":
-        conflict_resolution = lambda a, b: a  # noqa: E731
+
+        def conflict_resolution(a, b):
+            return a
     elif conflict_resolution == "b wins":
-        conflict_resolution = lambda a, b: b  # noqa: E731
+
+        def conflict_resolution(a, b):
+            return b
 
     status_nonempty = bool(next(status.iter_old(), None))
 

vdirsyncer/sync/exceptions.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from .. import exceptions
+from vdirsyncer import exceptions
 
 
 class SyncError(exceptions.Error):

vdirsyncer/sync/status.py

@@ -169,6 +169,11 @@ class SqliteStatus(_StatusBase):
             ); """
             )
 
+    def close(self):
+        if self._c:
+            self._c.close()
+            self._c = None
+
     def _is_latest_version(self):
         try:
             return bool(

vdirsyncer/utils.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import contextlib
 import functools
 import os
 import sys
+import tempfile
 import uuid
 from inspect import getfullargspec
 from typing import Callable
@@ -22,8 +24,7 @@ _missing = object()
 def expand_path(p: str) -> str:
     """Expand $HOME in a path and normalise slashes."""
     p = os.path.expanduser(p)
-    p = os.path.normpath(p)
+    return os.path.normpath(p)
-    return p
 
 
 def split_dict(d: dict, f: Callable):
@@ -76,7 +77,7 @@ def get_storage_init_specs(cls, stop_at=object):
     spec = getfullargspec(cls.__init__)
     traverse_superclass = getattr(cls.__init__, "_traverse_superclass", True)
     if traverse_superclass:
-        if traverse_superclass is True:  # noqa
+        if traverse_superclass is True:
             supercls = next(
                 getattr(x.__init__, "__objclass__", x) for x in cls.__mro__[1:]
             )
@@ -86,7 +87,7 @@ def get_storage_init_specs(cls, stop_at=object):
     else:
         superspecs = ()
 
-    return (spec,) + superspecs
+    return (spec, *superspecs)
 
 
 def get_storage_init_args(cls, stop_at=object):
@@ -125,12 +126,13 @@ def checkdir(path: str, create: bool = False, mode: int = 0o750) -> None:
             raise exceptions.CollectionNotFound(f"Directory {path} does not exist.")
 
 
-def checkfile(path, create=False):
+def checkfile(path, create=False) -> None:
-    """
+    """Check whether ``path`` is a file.
-    Check whether ``path`` is a file.
 
     :param create: Whether to create the file's parent directories if they do
         not exist.
+    :raises CollectionNotFound: if path does not exist.
+    :raises OSError: if path exists but is not a file.
     """
     checkdir(os.path.dirname(path), create=create)
     if not os.path.isfile(path):
@@ -143,24 +145,6 @@ def checkfile(path, create=False):
             raise exceptions.CollectionNotFound(f"File {path} does not exist.")
 
 
-class cached_property:
-    """A read-only @property that is only evaluated once. Only usable on class
-    instances' methods.
-    """
-
-    def __init__(self, fget, doc=None):
-        self.__name__ = fget.__name__
-        self.__module__ = fget.__module__
-        self.__doc__ = doc or fget.__doc__
-        self.fget = fget
-
-    def __get__(self, obj, cls):
-        if obj is None:  # pragma: no cover
-            return self
-        obj.__dict__[self.__name__] = result = self.fget(obj)
-        return result
-
-
 def href_safe(ident, safe=SAFE_UID_CHARS):
     return not bool(set(ident) - set(safe))
 
@@ -174,8 +158,7 @@ def generate_href(ident=None, safe=SAFE_UID_CHARS):
     """
     if not ident or not href_safe(ident, safe):
         return str(uuid.uuid4())
-    else:
+    return ident
-        return ident
 
 
 def synchronized(lock=None):
@@ -208,7 +191,7 @@ def open_graphical_browser(url, new=0, autoraise=True):
 
     cli_names = {"www-browser", "links", "links2", "elinks", "lynx", "w3m"}
 
-    if webbrowser._tryorder is None:  # Python 3.7
+    if webbrowser._tryorder is None:  # Python 3.8
         webbrowser.register_standard_browsers()
 
     for name in webbrowser._tryorder:
@@ -219,4 +202,28 @@ def open_graphical_browser(url, new=0, autoraise=True):
         if browser.open(url, new, autoraise):
             return
 
-    raise RuntimeError("No graphical browser found. Please open the URL " "manually.")
+    raise RuntimeError("No graphical browser found. Please open the URL manually.")
+
+
+@contextlib.contextmanager
+def atomic_write(dest, mode="wb", overwrite=False):
+    if "w" not in mode:
+        raise RuntimeError("`atomic_write` requires write access")
+
+    fd, src = tempfile.mkstemp(prefix=os.path.basename(dest), dir=os.path.dirname(dest))
+    file = os.fdopen(fd, mode=mode)
+
+    try:
+        yield file
+    except Exception:
+        os.unlink(src)
+        raise
+    else:
+        file.flush()
+        file.close()
+
+        if overwrite:
+            os.rename(src, dest)
+        else:
+            os.link(src, dest)
+            os.unlink(src)

vdirsyncer/vobject.py

@@ -1,10 +1,10 @@
 from __future__ import annotations
 
 import hashlib
+from functools import cached_property
 from itertools import chain
 from itertools import tee
 
-from .utils import cached_property
 from .utils import uniq
 
 IGNORE_PROPS = (
@@ -188,7 +188,7 @@ def join_collection(items, wrappers=_default_join_wrappers):
     """
 
     items1, items2 = tee((_Component.parse(x) for x in items), 2)
-    item_type, wrapper_type = _get_item_type(items1, wrappers)
+    _item_type, wrapper_type = _get_item_type(items1, wrappers)
     wrapper_props = []
 
     def _get_item_components(x):
@@ -231,8 +231,7 @@ def _get_item_type(components, wrappers):
 
     if not i:
         return None, None
-    else:
+    raise ValueError("Not sure how to join components.")
-        raise ValueError("Not sure how to join components.")
 
 
 class _Component:
@@ -303,10 +302,9 @@ class _Component:
 
         if multiple:
             return rv
-        elif len(rv) != 1:
+        if len(rv) != 1:
             raise ValueError(f"Found {len(rv)} components, expected one.")
-        else:
+        return rv[0]
-            return rv[0]
 
     def dump_lines(self):
         yield f"BEGIN:{self.name}"
@@ -323,13 +321,12 @@ class _Component:
             for line in lineiter:
                 if line.startswith(prefix):
                     break
-                else:
+                new_lines.append(line)
-                    new_lines.append(line)
             else:
                 break
 
             for line in lineiter:
-                if not line.startswith((" ", "\t")):
+                if not line.startswith((" ", "\t", *prefix)):
                     new_lines.append(line)
                     break
 
@@ -347,10 +344,9 @@ class _Component:
             return obj not in self.subcomponents and not any(
                 obj in x for x in self.subcomponents
             )
-        elif isinstance(obj, str):
+        if isinstance(obj, str):
             return self.get(obj, None) is not None
-        else:
+        raise ValueError(obj)
-            raise ValueError(obj)
 
     def __getitem__(self, key):
         prefix_without_params = f"{key}:"
@@ -360,7 +356,7 @@ class _Component:
             if line.startswith(prefix_without_params):
                 rv = line[len(prefix_without_params) :]
                 break
-            elif line.startswith(prefix_with_params):
+            if line.startswith(prefix_with_params):
                 rv = line[len(prefix_with_params) :].split(":", 1)[-1]
                 break
         else: