Mark unused variables as such

Adds support for auto-creating collections when they exist only on
one side and `implicit = 'create'` is set in the pair config.

.builds/archlinux-py313.yml

@@ -5,16 +5,17 @@ packages:
   - docker
   - docker-compose
   # Build dependencies:
-  - python-pip
   - python-wheel
+  - python-build
+  - python-installer
+  - python-setuptools-scm
   # Runtime dependencies:
-  - python-atomicwrites
   - python-click
   - python-click-log
   - python-click-threading
   - python-requests
-  - python-requests-toolbelt
   - python-aiohttp-oauthlib
+  - python-tenacity
   # Test dependencies:
   - python-hypothesis
   - python-pytest-cov
@@ -34,11 +35,14 @@ environment:
   REQUIREMENTS: release
   # TODO: ETESYNC_TESTS
 tasks:
-  - setup: |
+  - check-python:
+      python --version | grep 'Python 3.13'
+  - docker: |
       sudo systemctl start docker
+  - setup: |
       cd vdirsyncer
-      python setup.py build
-      sudo pip install --no-index .
+      python -m build --wheel --skip-dependency-check --no-isolation
+      sudo python -m installer dist/*.whl
   - test: |
       cd vdirsyncer
       make -e ci-test

.builds/tests-minimal.yml

@@ -3,11 +3,13 @@
 # TODO: It might make more sense to test with an older Ubuntu or Fedora version
 # here, and consider that our "oldest suppported environment".
 
-image: archlinux
+image: alpine/3.19 # python 3.11
 packages:
   - docker
+  - docker-cli
   - docker-compose
-  - python-pip
+  - py3-pip
+  - python3-dev
 sources:
   - https://github.com/pimutils/vdirsyncer
 environment:
@@ -16,15 +18,19 @@ environment:
   CODECOV_TOKEN: b834a3c5-28fa-4808-9bdb-182210069c79
   DAV_SERVER: radicale xandikos
   REQUIREMENTS: minimal
-  # TODO: ETESYNC_TESTS
 tasks:
+  - venv: |
+      python3 -m venv $HOME/venv
+      echo "export PATH=$HOME/venv/bin:$PATH" >> $HOME/.buildenv
+  - docker: |
+      sudo addgroup $(whoami) docker
+      sudo service docker start
   - setup: |
-      sudo systemctl start docker
       cd vdirsyncer
+      # Hack, no idea why it's needed
+      sudo ln -s /usr/include/python3.11/cpython/longintrepr.h /usr/include/python3.11/longintrepr.h
       make -e install-dev
   - test: |
       cd vdirsyncer
-      # Non-system python is used for packages:
-      export PATH=$PATH:~/.local/bin/
       make -e ci-test
       make -e ci-test-storage

.builds/tests-pypi.yml

@@ -17,16 +17,21 @@ environment:
   REQUIREMENTS: release
   # TODO: ETESYNC_TESTS
 tasks:
-  - setup: |
+  - venv: |
+      python -m venv $HOME/venv
+      echo "export PATH=$HOME/venv/bin:$PATH" >> $HOME/.buildenv
+  - docker: |
       sudo systemctl start docker
+  - setup: |
       cd vdirsyncer
-      make -e install-dev -e install-docs
+      make -e install-dev
   - test: |
       cd vdirsyncer
-      # Non-system python is used for packages:
-      export PATH=$PATH:~/.local/bin/
       make -e ci-test
       make -e ci-test-storage
+  - check: |
+      cd vdirsyncer
+      make check
   - check-secrets: |
       # Stop here if this is a PR. PRs can't run with the below secrets.
       [ -f ~/fastmail-secrets ] || complete-build

.envrc

@@ -0,0 +1 @@
+layout python3

.gitlab-ci.yml

@@ -1,6 +0,0 @@
-python37:
-  image: python:3.7
-  before_script:
-    - make -e install-dev
-  script:
-    - make -e ci-test

.pre-commit-config.yaml

@@ -1,6 +1,6 @@
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.3.0
+    rev: v5.0.0
     hooks:
       - id: trailing-whitespace
         args: [--markdown-linebreak-ext=md]
@@ -8,27 +8,8 @@ repos:
       - id: check-toml
       - id: check-added-large-files
       - id: debug-statements
-  - repo: https://github.com/PyCQA/flake8
-    rev: "5.0.4"
-    hooks:
-      - id: flake8
-        additional_dependencies: [flake8-import-order, flake8-bugbear]
-  - repo: https://github.com/psf/black
-    rev: "22.10.0"
-    hooks:
-      - id: black
-  - repo: https://github.com/pycqa/isort
-    rev: 5.10.1
-    hooks:
-      - id: isort
-        name: isort (python)
-  - repo: https://github.com/asottile/pyupgrade
-    rev: v3.2.2
-    hooks:
-    -   id: pyupgrade
-        args: [--py37-plus]
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: "v0.990"
+    rev: "v1.15.0"
     hooks:
       - id: mypy
         files: vdirsyncer/.*
@@ -36,4 +17,23 @@ repos:
           - types-setuptools
           - types-docutils
           - types-requests
-          - types-atomicwrites
+  - repo: https://github.com/charliermarsh/ruff-pre-commit
+    rev: 'v0.11.4'
+    hooks:
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix]
+      - id: ruff-format
+  - repo: local
+    hooks:
+      - id: typos-syncroniz
+        name: typos-syncroniz
+        language: system
+        # Not how you spell "synchronise"
+        entry: sh -c "git grep -i syncroniz"
+        files: ".*/.*"
+      - id: typos-text-icalendar
+        name: typos-text-icalendar
+        language: system
+        # It's "text/calendar", no "i".
+        entry: sh -c "git grep -i 'text/icalendar'"
+        files: ".*/.*"

.readthedocs.yaml

@@ -0,0 +1,16 @@
+version: 2
+
+sphinx:
+  configuration: docs/conf.py
+
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.9"
+
+python:
+  install:
+    - method: pip
+      path: .
+      extra_requirements:
+        - docs

AUTHORS.rst

@@ -4,10 +4,13 @@ Contributors
 In alphabetical order:
 
 - Ben Boeckel
+- Bleala
 - Christian Geier
 - Clément Mondon
 - Corey Hinshaw
+- Kai Herlemann
 - Hugo Osvaldo Barrera
+- Jason Cox
 - Julian Mehne
 - Malte Kiefer
 - Marek Marczykowski-Górecki
@@ -16,6 +19,7 @@ In alphabetical order:
 - rEnr3n
 - Thomas Weißschuh
 - Witcher01
+- samm81
 
 Special thanks goes to:
 

CHANGELOG.rst

@@ -9,6 +9,60 @@ Package maintainers and users who have to manually update their installation
 may want to subscribe to `GitHub's tag feed
 <https://github.com/pimutils/vdirsyncer/tags.atom>`_.
 
+Version 0.21.0
+==============
+
+- Implement retrying for ``google`` storage type when a rate limit is reached.
+- ``tenacity`` is now a required dependency.
+- Drop support for Python 3.8.
+- Retry transient network errors for nullipotent requests.
+
+Version 0.20.0
+==============
+
+- Remove dependency on abandoned ``atomicwrites`` library.
+- Implement ``filter_hook`` for the HTTP storage.
+- Drop support for Python 3.7.
+- Add support for Python 3.12 and Python 3.13.
+- Properly close the status database after using. This especially affects tests,
+  where we were leaking a large amount of file descriptors.
+- Extend supported versions of ``aiostream`` to include 0.7.x.
+
+Version 0.19.3
+==============
+
+- Added a no_delete option to the storage configuration. :gh:`1090`
+- Fix crash when running ``vdirsyncer repair`` on a collection. :gh:`1019`
+- Add an option to request vCard v4.0.  :gh:`1066`
+- Require matching ``BEGIN`` and ``END`` lines in vobjects. :gh:`1103`
+- A Docker environment for Vdirsyncer has been added `Vdirsyncer DOCKERIZED <https://github.com/Bleala/Vdirsyncer-DOCKERIZED>`_.
+- Implement digest auth. :gh:`1137`
+- Add ``filter_hook`` parameter to :storage:`http`. :gh:`1136`
+
+Version 0.19.2
+==============
+
+- Improve the performance of ``SingleFileStorage``. :gh:`818`
+- Properly document some caveats of the Google Contacts storage.
+- Fix crash when using auth certs. :gh:`1033`
+- The ``filesystem`` storage can be specified with ``type =
+  "filesystem/icalendar"`` or ``type = "filesystem/vcard"``. This has not
+  functional impact, and is merely for forward compatibility with the Rust
+  implementation of vdirsyncer.
+- Python 3.10 and 3.11 are officially supported.
+- Instructions for integrating with Google CalDav/CardDav have changed.
+  Applications now need to be registered as "Desktop applications". Using "Web
+  application" no longer works due to changes on Google's side. :gh:`1078`
+
+Version 0.19.1
+==============
+
+- Fixed crash when operating on Google Contacts. :gh:`994`
+- The ``HTTP_PROXY`` and ``HTTPS_PROXY`` are now respected. :gh:`1031`
+- Instructions for integrating with Google CalDav/CardDav have changed.
+  Applications now need to be registered as "Web Application". :gh:`975`
+- Various documentation updates.
+
 Version 0.19.0
 ==============
 
@@ -19,6 +73,10 @@ Version 0.19.0
 - Add a new ``showconfig`` status. This prints *some* configuration values as
   JSON. This is intended to be used by external tools and helpers that interact
   with ``vdirsyncer``, and considered experimental.
+- Add ``implicit`` option to the :ref:`pair section <pair_config>`. When set to
+  "create", it implicitly creates missing collections during sync without user
+  prompts. This simplifies workflows where collections should be automatically
+  created on both sides.
 - Update TLS-related tests that were failing due to weak MDs. :gh:`903`
 - ``pytest-httpserver`` and ``trustme`` are now required for tests.
 - ``pytest-localserver`` is no longer required for tests.

Makefile

@@ -20,14 +20,8 @@ export CI := false
 # Whether to generate coverage data while running tests.
 export COVERAGE := $(CI)
 
-# Additional arguments that should be passed to py.test.
-PYTEST_ARGS =
-
 # Variables below this line are not very interesting for getting started.
 
-TEST_EXTRA_PACKAGES =
-
-PYTEST = py.test $(PYTEST_ARGS)
 CODECOV_PATH = /tmp/codecov.sh
 
 all:
@@ -35,32 +29,21 @@ all:
 
 ci-test:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
-	$(PYTEST) --cov vdirsyncer --cov-append tests/unit/ tests/system/
+	pytest --cov vdirsyncer --cov-append tests/unit/ tests/system/
 	bash $(CODECOV_PATH) -c
 
 ci-test-storage:
 	curl -s https://codecov.io/bash > $(CODECOV_PATH)
 	set -ex; \
 	for server in $(DAV_SERVER); do \
-		DAV_SERVER=$$server $(PYTEST) --cov vdirsyncer --cov-append tests/storage; \
+		DAV_SERVER=$$server pytest --cov vdirsyncer --cov-append tests/storage; \
 	done
 	bash $(CODECOV_PATH) -c
 
-test:
-	$(PYTEST)
-
-style:
-	pre-commit run --all
-	! git grep -i syncroniz */*
-	! git grep -i 'text/icalendar' */*
-	sphinx-build -W -b html ./docs/ ./docs/_build/html/
-
-install-docs:
-	pip install -Ur docs-requirements.txt
-
-docs:
-	cd docs && make html
-	sphinx-build -W -b linkcheck ./docs/ ./docs/_build/linkcheck/
+check:
+	ruff check
+	ruff format --diff
+	#mypy vdirsyncer
 
 release-deb:
 	sh scripts/release-deb.sh debian jessie
@@ -71,11 +54,10 @@ release-deb:
 
 install-dev:
 	pip install -U pip setuptools wheel
-	pip install -e .
-	pip install -Ur test-requirements.txt $(TEST_EXTRA_PACKAGES)
-	pip install pre-commit
+	pip install -e '.[test,check,docs]'
 	set -xe && if [ "$(REQUIREMENTS)" = "minimal" ]; then \
-		pip install -U --force-reinstall $$(python setup.py --quiet minimal_requirements); \
+		pip install pyproject-dependencies && \
+		pip install -U --force-reinstall $$(pyproject-dependencies . | sed 's/>/=/'); \
 	fi
 
 .PHONY: docs

README.rst

@@ -40,7 +40,7 @@ servers. It can also be used to synchronize calendars and/or addressbooks
 between two servers directly.
 
 It aims to be for calendars and contacts what `OfflineIMAP
-<http://offlineimap.org/>`_ is for emails.
+<https://www.offlineimap.org/>`_ is for emails.
 
 .. _programs: https://vdirsyncer.pimutils.org/en/latest/tutorials/
 
@@ -59,6 +59,15 @@ Links of interest
 
 * `Donations <https://vdirsyncer.pimutils.org/en/stable/donations.html>`_
 
+Dockerized
+=================
+If you want to run `Vdirsyncer <https://vdirsyncer.pimutils.org/en/stable/>`_ in a
+Docker environment, you can check out the following GitHub Repository:
+
+* `Vdirsyncer DOCKERIZED <https://github.com/Bleala/Vdirsyncer-DOCKERIZED>`_
+
+Note: This is an unofficial Docker build, it is maintained by `Bleala <https://github.com/Bleala>`_.
+
 License
 =======
 

contrib/conflict_resolution/resolve_interactively.py

@@ -16,6 +16,9 @@ SPDX-License-Identifier: BSD-3-Clause
 SPDX-FileCopyrightText: 2021 Intevation GmbH <https://intevation.de>
 Author: <bernhard.reiter@intevation.de>
 """
+
+from __future__ import annotations
+
 import re
 import subprocess
 import sys
@@ -51,8 +54,8 @@ def main(ical1_filename, ical2_filename):
             f"{get_summary(ical1)}...\n(full contents: {ical1_filename})\n\n"
             "or the second entry:\n"
             f"{get_summary(ical2)}...\n(full contents: {ical2_filename})?",
+            *additional_args,
         ]
-        + additional_args
     )
 
     if r.returncode == 2:

docs-requirements.txt

@@ -1,3 +0,0 @@
-sphinx != 1.4.7
-sphinx_rtd_theme
-setuptools_scm

docs/conf.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import datetime
 import os
 
@@ -18,7 +20,7 @@ copyright = "2014-{}, Markus Unterwaditzer & contributors".format(
 release = get_distribution("vdirsyncer").version
 version = ".".join(release.split(".")[:2])  # The short X.Y version.
 
-rst_epilog = ".. |vdirsyncer_version| replace:: %s" % release
+rst_epilog = f".. |vdirsyncer_version| replace:: {release}"
 
 exclude_patterns = ["_build"]
 
@@ -35,9 +37,7 @@ except ImportError:
     html_theme = "default"
     if not on_rtd:
         print("-" * 74)
-        print(
-            "Warning: sphinx-rtd-theme not installed, building with default " "theme."
-        )
+        print("Warning: sphinx-rtd-theme not installed, building with default theme.")
         print("-" * 74)
 
 html_static_path = ["_static"]
@@ -76,7 +76,7 @@ def github_issue_role(name, rawtext, text, lineno, inliner, options=None, conten
     try:
         issue_num = int(text)
         if issue_num <= 0:
-            raise ValueError()
+            raise ValueError
     except ValueError:
         msg = inliner.reporter.error(f"Invalid GitHub issue: {text}", line=lineno)
         prb = inliner.problematic(rawtext, rawtext, msg)

docs/config.rst

@@ -61,7 +61,8 @@ Pair Section
   sync`` is executed. See also :ref:`collections_tutorial`.
 
   The special values ``"from a"`` and ``"from b"``, tell vdirsyncer to try
-  autodiscovery on a specific storage.
+  autodiscovery on a specific storage. It means all the collections on side A /
+  side B.
 
   If the collection you want to sync doesn't have the same name on each side,
   you may also use a value of the form ``["config_name", "name_a", "name_b"]``.
@@ -71,8 +72,8 @@ Pair Section
 
   Examples:
 
-  - ``collections = ["from b", "foo", "bar"]`` makes vdirsyncer synchronize the
-    collections from side B, and also the collections named "foo" and "bar".
+  - ``collections = ["from b", "foo", "bar"]`` makes vdirsyncer synchronize all
+    the collections from side B, and also the collections named "foo" and "bar".
 
   - ``collections = ["from b", "from a"]`` makes vdirsyncer synchronize all
     existing collections on either side.
@@ -127,6 +128,16 @@ Pair Section
 
   The ``conflict_resolution`` parameter applies for these properties too.
 
+.. _implicit_def:
+
+- ``implicit``: Opt into implicitly creating collections. Example::
+
+      implicit = "create"
+
+  When set to "create", missing collections are automatically created on both
+  sides during sync without prompting the user. This simplifies workflows where
+  all collections should be synchronized bidirectionally.
+
 .. _storage_config:
 
 Storage Section
@@ -238,6 +249,7 @@ CalDAV and CardDAV
      #useragent = "vdirsyncer/0.16.4"
      #verify_fingerprint = null
      #auth_cert = null
+     #use_vcard_4 = false
 
    :param url: Base URL or an URL to an addressbook.
    :param username: Username for authentication.
@@ -255,6 +267,7 @@ CalDAV and CardDAV
                      certificate and the key or a list of paths to the files
                      with them.
    :param useragent: Default ``vdirsyncer``.
+   :param use_vcard_4: Whether the server use vCard 4.0.
 
 Google
 ++++++
@@ -268,6 +281,14 @@ in terms of data safety**. See `this blog post
 <https://evertpot.com/google-carddav-issues/>`_ for the details.  Always back
 up your data.
 
+Another caveat is that Google group labels are not synced with vCard's
+`CATEGORIES <https://www.rfc-editor.org/rfc/rfc6350#section-6.7.1>`_ property
+(also see :gh:`814` and
+`upstream issue #36761530 <https://issuetracker.google.com/issues/36761530>`_
+for reference) and the
+`BDAY <https://www.rfc-editor.org/rfc/rfc6350#section-6.2.5>`_ property is not
+synced when only partial date information is present (e.g. the year is missing).
+
 At first run you will be asked to authorize application for Google account
 access.
 
@@ -279,25 +300,29 @@ Furthermore you need to register vdirsyncer as an application yourself to
 obtain ``client_id`` and ``client_secret``, as it is against Google's Terms of
 Service to hardcode those into opensource software [googleterms]_:
 
-1. Go to the `Google API Manager <https://console.developers.google.com>`_ and
-   create a new project under any name.
+1. Go to the `Google API Manager <https://console.developers.google.com>`_
+
+2. Create a new project under any name.
 
 2. Within that project, enable the "CalDAV" and "CardDAV" APIs (**not** the
    Calendar and Contacts APIs, those are different and won't work). There should
-   be a searchbox where you can just enter those terms.
+   be a search box where you can just enter those terms.
 
-3. In the sidebar, select "Credentials" and create a new "OAuth Client ID". The
-   application type is "Other".
+3. In the sidebar, select "Credentials", then "Create Credentials" and create a
+   new "OAuth Client ID".
 
    You'll be prompted to create a OAuth consent screen first. Fill out that
    form however you like.
 
+   After setting up the consent screen, finish creating the new "OAuth Client
+   ID'. The correct application type is "Desktop application".
+
 4. Finally you should have a Client ID and a Client secret. Provide these in
    your storage config.
 
-The ``token_file`` parameter should be a filepath where vdirsyncer can later
-store authentication-related data. You do not need to create the file itself
-or write anything to it.
+The ``token_file`` parameter should be a path to a file where vdirsyncer can
+later store authentication-related data. You do not need to create the file
+itself or write anything to it.
 
 .. [googleterms] See `ToS <https://developers.google.com/terms/?hl=th>`_,
    section "Confidential Matters".
@@ -305,7 +330,7 @@ or write anything to it.
 .. note::
 
     You need to configure which calendars Google should offer vdirsyncer using
-    a rather hidden `settings page
+    a secret `settings page
     <https://calendar.google.com/calendar/syncselect>`_.
 
 .. storage:: google_calendar
@@ -345,6 +370,10 @@ or write anything to it.
    :param client_id/client_secret: OAuth credentials, obtained from the Google
                                    API Manager.
 
+The current flow is not ideal, but Google has deprecated the previous APIs used
+for this without providing a suitable replacement. See :gh:`975` for discussion
+on the topic.
+
 Local
 +++++
 
@@ -360,6 +389,7 @@ Local
       fileext = "..."
       #encoding = "utf-8"
       #post_hook = null
+      #pre_deletion_hook = null
       #fileignoreext = ".tmp"
 
     Can be used with `khal <http://lostpackets.de/khal/>`_. See :doc:`vdir` for
@@ -381,6 +411,8 @@ Local
     :param post_hook: A command to call for each item creation and
         modification. The command will be called with the path of the
         new/updated file.
+    :param pre_deletion_hook: A command to call for each item deletion.
+        The command will be called with the path of the deleted file.
     :param fileeignoreext: The file extention to ignore. It is only useful
         if fileext is set to the empty string. The default is ``.tmp``.
 
@@ -462,6 +494,7 @@ leads to an error.
         [storage holidays_remote]
         type = "http"
         url = https://example.com/holidays_from_hicksville.ics
+        #filter_hook = null
 
     Too many WebCAL providers generate UIDs of all ``VEVENT``-components
     on-the-fly, i.e. all UIDs change every time the calendar is downloaded.
@@ -486,3 +519,8 @@ leads to an error.
     :param auth_cert: Optional. Either a path to a certificate with a client
         certificate and the key or a list of paths to the files with them.
     :param useragent: Default ``vdirsyncer``.
+    :param filter_hook: Optional. A filter command to call for each fetched
+        item, passed in raw form to stdin and returned via stdout.
+        If nothing is returned by the filter command, the item is skipped.
+        This can be used to alter fields as needed when dealing with providers
+        generating malformed events.

docs/contact.rst

@@ -9,7 +9,4 @@ Support and Contact
 * Open `a GitHub issue <https://github.com/pimutils/vdirsyncer/issues/>`_ for
   concrete bug reports and feature requests.
 
-* Lastly, you can also `contact the author directly
-  <https://unterwaditzer.net/contact.html>`_. Do this for security issues. If
-  that doesn't work out (i.e. if I don't respond within one week), use
-  ``contact@pimutils.org``.
+* For security issues, contact ``contact@pimutils.org``.

docs/contributing.rst

@@ -81,7 +81,7 @@ virtualenv_ and run this inside of it::
 
     # Install development dependencies, including:
     #  - vdirsyncer from the repo into the virtualenv
-    #  - stylecheckers (flake8) and code formatters (autopep8)
+    #  - style checks and formatting (ruff)
     make install-dev
 
     # Install git commit hook for some extra linting and checking
@@ -89,9 +89,10 @@ virtualenv_ and run this inside of it::
 
 Then you can run::
 
-    make test   # The normal testsuite
-    make style  # Stylechecker
-    make docs   # Build the HTML docs, output is at docs/_build/html/
+    pytest                # The normal testsuite
+    pre-commit run --all  # Run all linters (which also run via pre-commit)
+    make -C docs html     # Build the HTML docs, output is at docs/_build/html/
+    make -C docs linkcheck  # Check docs for any broken links
 
 The ``Makefile`` has a lot of options that allow you to control which tests are
 run, and which servers are tested. Take a look at its code where they are all

docs/installation.rst

@@ -10,14 +10,15 @@ OS/distro packages
 The following packages are community-contributed and were up-to-date at the
 time of writing:
 
-- `ArchLinux <https://www.archlinux.org/packages/community/any/vdirsyncer/>`_
+- `Arch Linux <https://archlinux.org/packages/extra/any/vdirsyncer/>`_
 - `Ubuntu and Debian, x86_64-only
   <https://packagecloud.io/pimutils/vdirsyncer>`_ (packages also exist
   in the official repositories but may be out of date)
-- `GNU Guix <https://www.gnu.org/software/guix/package-list.html#vdirsyncer>`_
+- `GNU Guix <https://packages.guix.gnu.org/packages/vdirsyncer/>`_
 - `macOS (homebrew) <https://formulae.brew.sh/formula/vdirsyncer>`_
-- `BSD (pkgsrc) <http://pkgsrc.se/time/py-vdirsyncer>`_
+- `NetBSD <https://ftp.netbsd.org/pub/pkgsrc/current/pkgsrc/time/py-vdirsyncer/index.html>`_
 - `OpenBSD <http://ports.su/productivity/vdirsyncer>`_
+- `Slackware (SlackBuild at Slackbuilds.org) <https://slackbuilds.org/repository/15.0/network/vdirsyncer/>`_
 
 We only support the latest version of vdirsyncer, which is at the time of this
 writing |vdirsyncer_version|. Please **do not file bugs if you use an older
@@ -41,7 +42,7 @@ If your distribution doesn't provide a package for vdirsyncer, you still can
 use Python's package manager "pip". First, you'll have to check that the
 following things are installed:
 
-- Python 3.7+ and pip.
+- Python 3.9 to 3.13 and pip.
 - ``libxml`` and ``libxslt``
 - ``zlib``
 - Linux or macOS. **Windows is not supported**, see :gh:`535`.
@@ -49,19 +50,45 @@ following things are installed:
 On Linux systems, using the distro's package manager is the best
 way to do this, for example, using Ubuntu::
 
-    sudo apt-get install libxml2 libxslt1.1 zlib1g python
+    sudo apt-get install libxml2 libxslt1.1 zlib1g python3
 
 Then you have several options. The following text applies for most Python
 software by the way.
 
+pipx: The clean, easy way
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+pipx_ is a new package manager for Python-based software that automatically
+sets up a virtual environment for each program it installs. Please note that
+installing via pipx will not include manual pages nor systemd services.
+
+pipx will install vdirsyncer into ``~/.local/pipx/venvs/vdirsyncer``
+
+Assuming that pipx is installed, vdirsyncer can be installed with::
+
+    pipx install vdirsyncer
+
+It can later be updated to the latest version with::
+
+    pipx upgrade vdirsyncer
+
+And can be uninstalled with::
+
+    pipx uninstall vdirsyncer
+
+This last command will remove vdirsyncer and any dependencies installed into
+the above location.
+
+.. _pipx: https://github.com/pipxproject/pipx
+
 The dirty, easy way
 ~~~~~~~~~~~~~~~~~~~
 
-The easiest way to install vdirsyncer at this point would be to run::
+If pipx is not available on your distribution, the easiest way to install
+vdirsyncer at this point would be to run::
 
-    pip install --user --ignore-installed vdirsyncer
+    pip install --ignore-installed vdirsyncer
 
-- ``--user`` is to install without root rights (into your home directory)
 - ``--ignore-installed`` is to work around Debian's potentially broken packages
   (see :ref:`debian-urllib3`).
 
@@ -92,25 +119,4 @@ This method has two advantages:
   distro-specific issues.
 - You can delete ``~/vdirsyncer_env/`` to uninstall vdirsyncer entirely.
 
-The clean, easy way
-~~~~~~~~~~~~~~~~~~~
-
-pipx_ is a new package manager for Python-based software that automatically
-sets up a virtualenv for each program you install. Assuming you have it
-installed on your operating system, you can do::
-
-    pipx install vdirsyncer
-
-and ``~/.local/pipx/venvs/vdirsyncer`` will be your new vdirsyncer installation. To
-update vdirsyncer to the latest version::
-
-    pipx upgrade vdirsyncer
-
-If you're done with vdirsyncer, you can do::
-
-    pipx uninstall vdirsyncer
-
-and vdirsyncer will be uninstalled, including its dependencies.
-
 .. _virtualenv: https://virtualenv.readthedocs.io/
-.. _pipx: https://github.com/pipxproject/pipx

docs/keyring.rst

@@ -78,3 +78,19 @@ You can also simply prompt for the password::
     type = "caldav"
     username = "myusername"
     password.fetch = ["prompt", "Password for CalDAV"]
+
+Environment variable
+===============
+
+To read the password from an environment variable::
+
+    [storage foo]
+    type = "caldav"
+    username = "myusername"
+    password.fetch = ["command", "printenv", "DAV_PW"]
+
+This is especially handy if you use the same password multiple times
+(say, for a CardDAV and a CalDAV storage).
+On bash, you can read and export the password without printing::
+
+    read -s DAV_PW "DAV Password: " && export DAV_PW

docs/packaging.rst

@@ -46,15 +46,16 @@ You can install the all development dependencies with::
     make install-dev
 
 You probably don't want this since it will use pip to download the
-dependencies. Alternatively you can find the testing dependencies in
-``test-requirements.txt``, again with lower-bound version requirements.
+dependencies. Alternatively test dependencies are listed as ``test`` optional
+dependencies in ``pyproject.toml``, again with lower-bound version
+requirements.
 
 You also have to have vdirsyncer fully installed at this point. Merely
 ``cd``-ing into the tarball will not be sufficient.
 
 Running the tests happens with::
 
-    make test
+    pytest
 
 Hypothesis will randomly generate test input. If you care about deterministic
 tests, set the ``DETERMINISTIC_TESTS`` variable to ``"true"``::
@@ -73,10 +74,11 @@ Using Sphinx_ you can generate the documentation you're reading right now in a
 variety of formats, such as HTML, PDF, or even as a manpage. That said, I only
 take care of the HTML docs' formatting.
 
-You can find a list of dependencies in ``docs-requirements.txt``. Again, you
-can install those using pip with::
+You can find a list of dependencies in ``pyproject.toml``, in the
+``project.optional-dependencies`` section as ``docs``. Again, you can install
+those using pip with::
 
-    make install-docs
+    pip install '.[docs]'
 
 Then change into the ``docs/`` directory and build whatever format you want
 using the ``Makefile`` in there (run ``make`` for the formats you can build).

docs/problems.rst

@@ -18,5 +18,5 @@ package that don't play well with packages assuming a normal ``requests``. This
 is due to stubbornness on both sides.
 
 See :gh:`82` and :gh:`140` for past discussions. You have one option to work
-around this, that is, to install vdirsyncer in a virtualenv, see
+around this, that is, to install vdirsyncer in a virtual environment, see
 :ref:`manual-installation`.

docs/ssl-tutorial.rst

@@ -14,14 +14,14 @@ To pin the certificate by fingerprint::
     [storage foo]
     type = "caldav"
     ...
-    verify_fingerprint = "94:FD:7A:CB:50:75:A4:69:82:0A:F8:23:DF:07:FC:69:3E:CD:90:CA"
+    verify_fingerprint = "6D:83:EA:32:6C:39:BA:08:ED:EB:C9:BC:BE:12:BB:BF:0F:D9:83:00:CC:89:7E:C7:32:05:94:96:CA:C5:59:5E"
 
-SHA256-Fingerprints can be used. CA validation is disabled when pinning a
-fingerprint.
+SHA256-Fingerprints must be used, MD5 and SHA-1 are insecure and not supported.
+CA validation is disabled when pinning a fingerprint.
 
-You can use the following command for obtaining a SHA-1 fingerprint::
+You can use the following command for obtaining a SHA256 fingerprint::
 
-    echo -n | openssl s_client -connect unterwaditzer.net:443 | openssl x509 -noout -fingerprint
+    echo -n | openssl s_client -connect unterwaditzer.net:443 | openssl x509 -noout -fingerprint -sha256
 
 However, please consider using `Let's Encrypt <https://letsencrypt.org/>`_ such
 that you can forget about all of that. It is easier to deploy a free
@@ -40,22 +40,16 @@ To point vdirsyncer to a custom set of root CAs::
     ...
     verify = "/path/to/cert.pem"
 
-Vdirsyncer uses the requests_ library, which, by default, `uses its own set of
-trusted CAs
-<http://www.python-requests.org/en/latest/user/advanced/#ca-certificates>`_.
+Vdirsyncer uses the aiohttp_ library, which uses the default `ssl.SSLContext
+https://docs.python.org/3/library/ssl.html#ssl.SSLContext`_ by default.
 
-However, the actual behavior depends on how you have installed it. Many Linux
-distributions patch their ``python-requests`` package to use the system
-certificate CAs. Normally these two stores are similar enough for you to not
-care.
+There are cases where certificate validation fails even though you can access
+the server fine through e.g. your browser. This usually indicates that your
+installation of ``python`` or the ``aiohttp`` or library is somehow broken. In
+such cases, it makes sense to explicitly set ``verify`` or
+``verify_fingerprint`` as shown above.
 
-But there are cases where certificate validation fails even though you can
-access the server fine through e.g. your browser. This usually indicates that
-your installation of the ``requests`` library is somehow broken. In such cases,
-it makes sense to explicitly set ``verify`` or ``verify_fingerprint`` as shown
-above.
-
-.. _requests: http://www.python-requests.org/
+.. _aiohttp: https://docs.aiohttp.org/en/stable/index.html
 
 .. _ssl-client-certs:
 

docs/tutorial.rst

@@ -176,8 +176,11 @@ as a file called ``color`` within the calendar folder.
 More information about collections
 ----------------------------------
 
-"Collection" is a collective term for addressbooks and calendars. Each
-collection from a storage has a "collection name", a unique identifier for each
+"Collection" is a collective term for addressbooks and calendars. A Cardav or
+Caldav server can contains several "collections" which correspond to several
+addressbooks or calendar.
+
+Each collection from a storage has a "collection name", a unique identifier for each
 collection. In the case of :storage:`filesystem`-storage, this is the name of the
 directory that represents the collection, in the case of the DAV-storages this
 is the last segment of the URL. We use this identifier in the ``collections``

docs/tutorials/todoman.rst

@@ -48,10 +48,9 @@ instance to subfolders of ``~/.calendar/``.
 Setting up todoman
 ==================
 
-Write this to ``~/.config/todoman/todoman.conf``::
+Write this to ``~/.config/todoman/config.py``::
 
-    [main]
-    path = ~/.calendars/*
+    path = "~/.calendars/*"
 
 The glob_ pattern in ``path`` will match all subfolders in ``~/.calendars/``,
 which is exactly the tasklists we want. Now you can use ``todoman`` as

docs/when.rst

@@ -50,7 +50,6 @@ program chosen:
 
 * Such a setup doesn't work at all with smartphones. Vdirsyncer, on the other
   hand, synchronizes with CardDAV/CalDAV servers, which can be accessed with
-  e.g. DAVx⁵_ or the apps by dmfs_.
+  e.g. DAVx⁵_ or other apps bundled with smartphones.
 
 .. _DAVx⁵: https://www.davx5.com/
-.. _dmfs: https://dmfs.org/

publish-release.yaml

@@ -1,11 +1,13 @@
-# Run new version to PyPI.
+# Push new version to PyPI.
 #
 # Usage: hut builds submit publish-release.yaml --follow
 
 image: alpine/edge
 packages:
+  - py3-build
   - py3-pip
   - py3-setuptools
+  - py3-setuptools_scm
   - py3-wheel
   - twine
 sources:
@@ -23,5 +25,5 @@ tasks:
       git describe --exact-match --tags || complete-build
   - publish: |
       cd vdirsyncer
-      python setup.py sdist bdist_wheel
-      twine upload dist/*
+      python -m build --no-isolation
+      twine upload --non-interactive dist/*

pyproject.toml

@@ -0,0 +1,114 @@
+# Vdirsyncer synchronizes calendars and contacts.
+#
+# Please refer to https://vdirsyncer.pimutils.org/en/stable/packaging.html for
+# how to package vdirsyncer.
+
+[build-system]
+requires = ["setuptools>=64", "setuptools_scm>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "vdirsyncer"
+authors = [
+    {name = "Markus Unterwaditzer", email = "markus@unterwaditzer.net"},
+]
+description = "Synchronize calendars and contacts"
+readme = "README.rst"
+requires-python = ">=3.9"
+keywords = ["todo", "task", "icalendar", "cli"]
+license = "BSD-3-Clause"
+license-files = ["LICENSE"]
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Environment :: Console",
+    "Operating System :: POSIX",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.9",
+    "Topic :: Internet",
+    "Topic :: Office/Business :: Scheduling",
+    "Topic :: Utilities",
+]
+dependencies = [
+    "click>=5.0,<9.0",
+    "click-log>=0.3.0,<0.5.0",
+    "requests>=2.20.0",
+    "aiohttp>=3.8.2,<4.0.0",
+    "aiostream>=0.4.3,<0.8.0",
+    "tenacity>=9.0.0",
+]
+dynamic = ["version"]
+
+[project.optional-dependencies]
+google = ["aiohttp-oauthlib"]
+test = [
+  "hypothesis>=6.72.0,<7.0.0",
+  "pytest",
+  "pytest-cov",
+  "pytest-httpserver",
+  "trustme",
+  "pytest-asyncio",
+  "aioresponses",
+]
+docs = [
+  "sphinx!=1.4.7",
+  "sphinx_rtd_theme",
+  "setuptools_scm",
+]
+check = [
+  "mypy",
+  "ruff",
+  "types-docutils",
+  "types-requests",
+  "types-setuptools",
+]
+
+[project.scripts]
+vdirsyncer = "vdirsyncer.cli:app"
+
+[tool.ruff.lint]
+extend-select = [
+    "B0",
+    "C4",
+    "E",
+    "I",
+    "RSE",
+    "SIM",
+    "TID",
+    "UP",
+    "W",
+]
+
+[tool.ruff.lint.isort]
+force-single-line = true
+required-imports = ["from __future__ import annotations"]
+
+[tool.pytest.ini_options]
+addopts = """
+--tb=short
+--cov-config .coveragerc
+--cov=vdirsyncer
+--cov-report=term-missing:skip-covered
+--no-cov-on-fail
+--color=yes
+"""
+# filterwarnings=error
+asyncio_default_fixture_loop_scope = "function"
+
+[tool.mypy]
+ignore_missing_imports = true
+
+[tool.coverage.report]
+exclude_lines = [
+    "if TYPE_CHECKING:",
+]
+
+[tool.setuptools.packages.find]
+include = ["vdirsyncer*"]
+
+[tool.setuptools_scm]
+write_to = "vdirsyncer/version.py"
+version_scheme = "no-guess-dev"

scripts/release-deb.sh

@@ -5,8 +5,10 @@ set -xeu
 SCRIPT_PATH=$(realpath "$0")
 SCRIPT_DIR=$(dirname "$SCRIPT_PATH")
 
-DISTRO=$1
-DISTROVER=$2
+# E.g.: debian, ubuntu
+DISTRO=${DISTRO:1}
+# E.g.: bullseye, bookwork
+DISTROVER=${DISTROVER:2}
 CONTAINER_NAME="vdirsyncer-${DISTRO}-${DISTROVER}"
 CONTEXT="$(mktemp -d)"
 
@@ -21,7 +23,7 @@ trap cleanup EXIT
 cp scripts/_build_deb_in_container.bash "$CONTEXT"
 python setup.py sdist -d "$CONTEXT"
 
-podman run -it \
+docker run -it \
   --name "$CONTAINER_NAME" \
   --volume "$CONTEXT:/source" \
   "$DISTRO:$DISTROVER" \

setup.cfg

@@ -1,28 +0,0 @@
-[tool:pytest]
-addopts =
-  --tb=short
-  --cov-config .coveragerc
-  --cov=vdirsyncer
-  --cov-report=term-missing:skip-covered
-  --no-cov-on-fail
-  --color=yes
-# filterwarnings=error
-
-[flake8]
-application-import-names = tests,vdirsyncer
-extend-ignore =
-  E203,  # Black-incompatible colon spacing.
-  W503,  # Line jump before binary operator.
-  I100,
-  I202
-max-line-length = 88
-exclude = .eggs,build
-import-order-style = smarkets
-
-[isort]
-force_single_line=true
-
-[mypy]
-ignore_missing_imports = True
-# See https://github.com/python/mypy/issues/7511:
-warn_no_return = False

setup.py

@@ -1,80 +0,0 @@
-"""
-Vdirsyncer synchronizes calendars and contacts.
-
-Please refer to https://vdirsyncer.pimutils.org/en/stable/packaging.html for
-how to package vdirsyncer.
-"""
-from setuptools import Command
-from setuptools import find_packages
-from setuptools import setup
-
-requirements = [
-    # https://github.com/mitsuhiko/click/issues/200
-    "click>=5.0,<9.0",
-    "click-log>=0.3.0, <0.5.0",
-    "requests >=2.20.0",
-    # https://github.com/sigmavirus24/requests-toolbelt/pull/28
-    # And https://github.com/sigmavirus24/requests-toolbelt/issues/54
-    "requests_toolbelt >=0.4.0",
-    # https://github.com/untitaker/python-atomicwrites/commit/4d12f23227b6a944ab1d99c507a69fdbc7c9ed6d  # noqa
-    "atomicwrites>=0.1.7",
-    "aiohttp>=3.8.0,<4.0.0",
-    "aiostream>=0.4.3,<0.5.0",
-]
-
-
-class PrintRequirements(Command):
-    description = "Prints minimal requirements"
-    user_options: list = []
-
-    def initialize_options(self):
-        pass
-
-    def finalize_options(self):
-        pass
-
-    def run(self):
-        for requirement in requirements:
-            print(requirement.replace(">", "=").replace(" ", ""))
-
-
-with open("README.rst") as f:
-    long_description = f.read()
-
-
-setup(
-    # General metadata
-    name="vdirsyncer",
-    author="Markus Unterwaditzer",
-    author_email="markus@unterwaditzer.net",
-    url="https://github.com/pimutils/vdirsyncer",
-    description="Synchronize calendars and contacts",
-    license="BSD",
-    long_description=long_description,
-    # Runtime dependencies
-    install_requires=requirements,
-    # Optional dependencies
-    extras_require={
-        "google": ["aiohttp-oauthlib"],
-    },
-    # Build dependencies
-    setup_requires=["setuptools_scm != 1.12.0"],
-    # Other
-    packages=find_packages(exclude=["tests.*", "tests"]),
-    include_package_data=True,
-    cmdclass={"minimal_requirements": PrintRequirements},
-    use_scm_version={"write_to": "vdirsyncer/version.py"},
-    entry_points={"console_scripts": ["vdirsyncer = vdirsyncer.cli:main"]},
-    classifiers=[
-        "Development Status :: 4 - Beta",
-        "Environment :: Console",
-        "License :: OSI Approved :: BSD License",
-        "Operating System :: POSIX",
-        "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.7",
-        "Programming Language :: Python :: 3.8",
-        "Programming Language :: Python :: 3.9",
-        "Topic :: Internet",
-        "Topic :: Utilities",
-    ],
-)

test-requirements.txt

@@ -1,7 +0,0 @@
-hypothesis>=5.0.0,<7.0.0
-pytest
-pytest-cov
-pytest-httpserver
-trustme
-pytest-asyncio
-aioresponses

tests/__init__.py

@@ -1,6 +1,9 @@
 """
 Test suite for vdirsyncer.
 """
+
+from __future__ import annotations
+
 import hypothesis.strategies as st
 import urllib3.exceptions
 
@@ -100,10 +103,8 @@ X-SOMETHING:{r}
 HAHA:YES
 END:FOO"""
 
-printable_characters_strategy = st.text(
-    st.characters(blacklist_categories=("Cc", "Cs"))
-)
+printable_characters_strategy = st.text(st.characters(exclude_categories=("Cc", "Cs")))
 
 uid_strategy = st.text(
-    st.characters(blacklist_categories=("Zs", "Zl", "Zp", "Cc", "Cs")), min_size=1
+    st.characters(exclude_categories=("Zs", "Zl", "Zp", "Cc", "Cs")), min_size=1
 ).filter(lambda x: x.strip() == x)

tests/conftest.py

@@ -1,6 +1,9 @@
 """
 General-purpose fixtures for vdirsyncer's testsuite.
 """
+
+from __future__ import annotations
+
 import logging
 import os
 
@@ -42,7 +45,7 @@ settings.register_profile(
     "deterministic",
     settings(
         derandomize=True,
-        suppress_health_check=HealthCheck.all(),
+        suppress_health_check=list(HealthCheck),
     ),
 )
 settings.register_profile("dev", settings(suppress_health_check=[HealthCheck.too_slow]))
@@ -56,12 +59,12 @@ else:
 
 
 @pytest_asyncio.fixture
-async def aio_session(event_loop):
+async def aio_session():
     async with aiohttp.ClientSession() as session:
         yield session
 
 
 @pytest_asyncio.fixture
-async def aio_connector(event_loop):
+async def aio_connector():
     async with aiohttp.TCPConnector(limit_per_host=16) as conn:
         yield conn

tests/storage/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import random
 import textwrap
 import uuid
@@ -8,16 +10,15 @@ import aiostream
 import pytest
 import pytest_asyncio
 
+from tests import EVENT_TEMPLATE
+from tests import TASK_TEMPLATE
+from tests import VCARD_TEMPLATE
+from tests import assert_item_equals
+from tests import normalize_item
 from vdirsyncer import exceptions
 from vdirsyncer.storage.base import normalize_meta_value
 from vdirsyncer.vobject import Item
 
-from .. import EVENT_TEMPLATE
-from .. import TASK_TEMPLATE
-from .. import VCARD_TEMPLATE
-from .. import assert_item_equals
-from .. import normalize_item
-
 
 def get_server_mixin(server_name):
     from . import __name__ as base
@@ -49,7 +50,7 @@ class StorageTests:
 
         :param collection: The name of the collection to create and use.
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @pytest_asyncio.fixture
     async def s(self, get_storage_args):
@@ -103,7 +104,7 @@ class StorageTests:
         href, etag = await s.upload(get_item())
         if etag is None:
             _, etag = await s.get(href)
-        ((href2, item, etag2),) = await aiostream.stream.list(s.get_multi([href] * 2))
+        ((href2, _item, etag2),) = await aiostream.stream.list(s.get_multi([href] * 2))
         assert href2 == href
         assert etag2 == etag
 
@@ -117,7 +118,7 @@ class StorageTests:
     @pytest.mark.asyncio
     async def test_upload(self, s, get_item):
         item = get_item()
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
         assert_item_equals((await s.get(href))[0], item)
 
     @pytest.mark.asyncio
@@ -145,7 +146,7 @@ class StorageTests:
     @pytest.mark.asyncio
     async def test_wrong_etag(self, s, get_item):
         item = get_item()
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
         with pytest.raises(exceptions.PreconditionFailed):
             await s.update(href, item, '"lolnope"')
         with pytest.raises(exceptions.PreconditionFailed):
@@ -193,8 +194,7 @@ class StorageTests:
         )
         assert {href: etag for href, item, etag in items} == info
 
-    @pytest.mark.asyncio
-    def test_repr(self, s, get_storage_args):  # XXX: unused param
+    def test_repr(self, s):
         assert self.storage_class.__name__ in repr(s)
         assert s.instance_name is None
 
@@ -384,7 +384,7 @@ class StorageTests:
         uid = str(uuid.uuid4())
         item = Item(
             textwrap.dedent(
-                """
+                f"""
         BEGIN:VCALENDAR
         VERSION:2.0
         BEGIN:VEVENT
@@ -418,13 +418,11 @@ class StorageTests:
         TRANSP:OPAQUE
         END:VEVENT
         END:VCALENDAR
-        """.format(
-                    uid=uid
-                )
+        """
             ).strip()
         )
 
-        href, etag = await s.upload(item)
+        href, _etag = await s.upload(item)
 
-        item2, etag2 = await s.get(href)
+        item2, _etag2 = await s.get(href)
         assert normalize_item(item) == normalize_item(item2)

tests/storage/conftest.py

@@ -1,9 +1,10 @@
+from __future__ import annotations
+
 import asyncio
 import contextlib
 import subprocess
 import time
 import uuid
-from typing import Type
 
 import aiostream
 import pytest
@@ -90,7 +91,7 @@ async def slow_create_collection(request, aio_connector):
     # storage limits.
     to_delete = []
 
-    async def inner(cls: Type, args: dict, collection_name: str) -> dict:
+    async def inner(cls: type, args: dict, collection_name: str) -> dict:
         """Create a collection
 
         Returns args necessary to create a Storage instance pointing to it.

tests/storage/dav/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import uuid
 
@@ -6,12 +8,11 @@ import aiostream
 import pytest
 
 from tests import assert_item_equals
+from tests.storage import StorageTests
+from tests.storage import get_server_mixin
 from vdirsyncer import exceptions
 from vdirsyncer.vobject import Item
 
-from .. import StorageTests
-from .. import get_server_mixin
-
 dav_server = os.environ.get("DAV_SERVER", "skip")
 ServerMixin = get_server_mixin(dav_server)
 
@@ -47,6 +48,6 @@ class DAVStorageTests(ServerMixin, StorageTests):
 
         monkeypatch.setattr(s, "_get_href", lambda item: item.ident + s.fileext)
         item = get_item(uid="град сатану" + str(uuid.uuid4()))
-        href, etag = await s.upload(item)
-        item2, etag2 = await s.get(href)
+        href, _etag = await s.upload(item)
+        item2, _etag2 = await s.get(href)
         assert_item_equals(item, item2)

tests/storage/dav/test_caldav.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+import contextlib
 import datetime
 from textwrap import dedent
 
@@ -9,10 +12,10 @@ from aioresponses import aioresponses
 from tests import EVENT_TEMPLATE
 from tests import TASK_TEMPLATE
 from tests import VCARD_TEMPLATE
+from tests.storage import format_item
 from vdirsyncer import exceptions
 from vdirsyncer.storage.dav import CalDAVStorage
 
-from .. import format_item
 from . import DAVStorageTests
 from . import dav_server
 
@@ -28,18 +31,16 @@ class TestCalDAVStorage(DAVStorageTests):
     async def test_doesnt_accept_vcard(self, item_type, get_storage_args):
         s = self.storage_class(item_types=(item_type,), **await get_storage_args())
 
-        try:
+        # Most storages hard-fail, but xandikos doesn't.
+        with contextlib.suppress(exceptions.Error, aiohttp.ClientResponseError):
             await s.upload(format_item(VCARD_TEMPLATE))
-        except (exceptions.Error, aiohttp.ClientResponseError):
-            # Most storages hard-fail, but xandikos doesn't.
-            pass
 
         assert not await aiostream.stream.list(s.list())
 
     # The `arg` param is not named `item_types` because that would hit
     # https://bitbucket.org/pytest-dev/pytest/issue/745/
     @pytest.mark.parametrize(
-        "arg,calls_num",
+        ("arg", "calls_num"),
         [
             (("VTODO",), 1),
             (("VEVENT",), 1),

tests/storage/dav/test_carddav.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from vdirsyncer.storage.dav import CardDAVStorage

tests/storage/dav/test_main.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from vdirsyncer.storage.dav import _BAD_XML_CHARS
@@ -39,8 +41,8 @@ def test_xml_utilities():
 def test_xml_specialchars(char):
     x = _parse_xml(
         '<?xml version="1.0" encoding="UTF-8" ?>'
-        "<foo>ye{}s\r\n"
-        "hello</foo>".format(chr(char)).encode("ascii")
+        f"<foo>ye{chr(char)}s\r\n"
+        "hello</foo>".encode("ascii")
     )
 
     if char in _BAD_XML_CHARS:
@@ -50,7 +52,7 @@ def test_xml_specialchars(char):
 @pytest.mark.parametrize(
     "href",
     [
-        "/dav/calendars/user/testuser/123/UID%253A20210609T084907Z-@synaps-web-54fddfdf7-7kcfm%250A.ics",  # noqa: E501
+        "/dav/calendars/user/testuser/123/UID%253A20210609T084907Z-@synaps-web-54fddfdf7-7kcfm%250A.ics",
     ],
 )
 def test_normalize_href(href):

tests/storage/servers/baikal/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 

tests/storage/servers/davical/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 import uuid
 
@@ -11,7 +13,7 @@ try:
         "url": "https://brutus.lostpackets.de/davical-test/caldav.php/",
     }
 except KeyError as e:
-    pytestmark = pytest.mark.skip(f"Missing envkey: {str(e)}")
+    pytestmark = pytest.mark.skip(f"Missing envkey: {e!s}")
 
 
 @pytest.mark.flaky(reruns=5)
@@ -23,7 +25,7 @@ class ServerMixin:
         elif self.storage_class.fileext == ".vcf":
             pytest.skip("No carddav")
         else:
-            raise RuntimeError()
+            raise RuntimeError
 
     @pytest.fixture
     def get_storage_args(self, davical_args, request):

tests/storage/servers/fastmail/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 
 import pytest
@@ -6,11 +8,13 @@ import pytest
 class ServerMixin:
     @pytest.fixture
     def get_storage_args(self, slow_create_collection, aio_connector, request):
-        if "item_type" in request.fixturenames:
-            if request.getfixturevalue("item_type") == "VTODO":
-                # Fastmail has non-standard support for TODOs
-                # See https://github.com/pimutils/vdirsyncer/issues/824
-                pytest.skip("Fastmail has non-standard VTODO support.")
+        if (
+            "item_type" in request.fixturenames
+            and request.getfixturevalue("item_type") == "VTODO"
+        ):
+            # Fastmail has non-standard support for TODOs
+            # See https://github.com/pimutils/vdirsyncer/issues/824
+            pytest.skip("Fastmail has non-standard VTODO support.")
 
         async def inner(collection="test"):
             args = {
@@ -24,7 +28,7 @@ class ServerMixin:
             elif self.storage_class.fileext == ".vcf":
                 args["url"] = "https://carddav.fastmail.com/"
             else:
-                raise RuntimeError()
+                raise RuntimeError
 
             if collection is not None:
                 args = await slow_create_collection(

tests/storage/servers/icloud/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 
 import pytest
@@ -8,7 +10,7 @@ class ServerMixin:
     def get_storage_args(self, item_type, slow_create_collection):
         if item_type != "VEVENT":
             # iCloud collections can either be calendars or task lists.
-            # See https://github.com/pimutils/vdirsyncer/pull/593#issuecomment-285941615  # noqa
+            # See https://github.com/pimutils/vdirsyncer/pull/593#issuecomment-285941615
             pytest.skip("iCloud doesn't support anything else than VEVENT")
 
         async def inner(collection="test"):
@@ -22,7 +24,7 @@ class ServerMixin:
             elif self.storage_class.fileext == ".vcf":
                 args["url"] = "https://contacts.icloud.com/"
             else:
-                raise RuntimeError()
+                raise RuntimeError
 
             if collection is not None:
                 args = slow_create_collection(self.storage_class, args, collection)

tests/storage/servers/radicale/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 

tests/storage/servers/skip/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 

tests/storage/servers/xandikos/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 

tests/storage/test_filesystem.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import subprocess
 
 import aiostream
@@ -46,7 +48,8 @@ class TestFilesystemStorage(StorageTests):
         s = self.storage_class(str(tmpdir), ".txt")
         await s.upload(Item("UID:a/b/c"))
         (item_file,) = tmpdir.listdir()
-        assert "/" not in item_file.basename and item_file.isfile()
+        assert "/" not in item_file.basename
+        assert item_file.isfile()
 
     @pytest.mark.asyncio
     async def test_ignore_tmp_files(self, tmpdir):
@@ -87,13 +90,13 @@ class TestFilesystemStorage(StorageTests):
         storage = self.storage_class(str(tmpdir), ".txt")
         item = Item("UID:" + "hue" * 600)
 
-        href, etag = await storage.upload(item)
+        href, _etag = await storage.upload(item)
         assert item.uid not in href
 
     @pytest.mark.asyncio
     async def test_post_hook_inactive(self, tmpdir, monkeypatch):
         def check_call_mock(*args, **kwargs):
-            raise AssertionError()
+            raise AssertionError
 
         monkeypatch.setattr(subprocess, "call", check_call_mock)
 

tests/storage/test_http.py

@@ -1,10 +1,16 @@
+from __future__ import annotations
+
+import aiohttp
 import pytest
-from aiohttp import BasicAuth
 from aioresponses import CallbackResult
 from aioresponses import aioresponses
 
 from tests import normalize_item
 from vdirsyncer.exceptions import UserError
+from vdirsyncer.http import BasicAuthMethod
+from vdirsyncer.http import DigestAuthMethod
+from vdirsyncer.http import UsageLimitReached
+from vdirsyncer.http import request
 from vdirsyncer.storage.http import HttpStorage
 from vdirsyncer.storage.http import prepare_auth
 
@@ -35,7 +41,7 @@ async def test_list(aio_connector):
         ),
     ]
 
-    responses = ["\n".join(["BEGIN:VCALENDAR"] + items + ["END:VCALENDAR"])] * 2
+    responses = ["\n".join(["BEGIN:VCALENDAR", *items, "END:VCALENDAR"])] * 2
 
     def callback(url, headers, **kwargs):
         assert headers["User-Agent"].startswith("vdirsyncer/")
@@ -89,16 +95,14 @@ def test_readonly_param(aio_connector):
 def test_prepare_auth():
     assert prepare_auth(None, "", "") is None
 
-    assert prepare_auth(None, "user", "pwd") == BasicAuth("user", "pwd")
-    assert prepare_auth("basic", "user", "pwd") == BasicAuth("user", "pwd")
+    assert prepare_auth(None, "user", "pwd") == BasicAuthMethod("user", "pwd")
+    assert prepare_auth("basic", "user", "pwd") == BasicAuthMethod("user", "pwd")
 
     with pytest.raises(ValueError) as excinfo:
         assert prepare_auth("basic", "", "pwd")
     assert "you need to specify username and password" in str(excinfo.value).lower()
 
-    from requests.auth import HTTPDigestAuth
-
-    assert isinstance(prepare_auth("digest", "user", "pwd"), HTTPDigestAuth)
+    assert isinstance(prepare_auth("digest", "user", "pwd"), DigestAuthMethod)
 
     with pytest.raises(ValueError) as excinfo:
         prepare_auth("ladida", "user", "pwd")
@@ -106,20 +110,12 @@ def test_prepare_auth():
     assert "unknown authentication method" in str(excinfo.value).lower()
 
 
-def test_prepare_auth_guess(monkeypatch):
-    import requests_toolbelt.auth.guess
-
-    assert isinstance(
-        prepare_auth("guess", "user", "pwd"),
-        requests_toolbelt.auth.guess.GuessAuth,
-    )
-
-    monkeypatch.delattr(requests_toolbelt.auth.guess, "GuessAuth")
-
+def test_prepare_auth_guess():
+    # guess auth is currently not supported
     with pytest.raises(UserError) as excinfo:
-        prepare_auth("guess", "user", "pwd")
+        prepare_auth("guess", "usr", "pwd")
 
-    assert "requests_toolbelt is too old" in str(excinfo.value).lower()
+    assert "not supported" in str(excinfo.value).lower()
 
 
 def test_verify_false_disallowed(aio_connector):
@@ -127,3 +123,41 @@ def test_verify_false_disallowed(aio_connector):
         HttpStorage(url="http://example.com", verify=False, connector=aio_connector)
 
     assert "must be a path to a pem-file." in str(excinfo.value).lower()
+
+
+@pytest.mark.asyncio
+async def test_403_usage_limit_exceeded(aio_connector):
+    url = "http://127.0.0.1/test_403"
+    error_body = {
+        "error": {
+            "errors": [
+                {
+                    "domain": "usageLimits",
+                    "message": "Calendar usage limits exceeded.",
+                    "reason": "quotaExceeded",
+                }
+            ],
+            "code": 403,
+            "message": "Calendar usage limits exceeded.",
+        }
+    }
+
+    async with aiohttp.ClientSession(connector=aio_connector) as session:
+        with aioresponses() as m:
+            m.get(url, status=403, payload=error_body, repeat=True)
+            with pytest.raises(UsageLimitReached):
+                await request("GET", url, session)
+
+
+@pytest.mark.asyncio
+async def test_403_without_usage_limits_domain(aio_connector):
+    """A 403 JSON error without the Google 'usageLimits' domain should not be
+    treated as UsageLimitReached and should surface as ClientResponseError.
+    """
+    url = "http://127.0.0.1/test_403_no_usage_limits"
+
+    async with aiohttp.ClientSession(connector=aio_connector) as session:
+        with aioresponses() as m:
+            m.get(url, status=403, repeat=True)
+            with pytest.raises(aiohttp.ClientResponseError):
+                await request("GET", url, session)

tests/storage/test_http_with_singlefile.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import aiostream
 import pytest
 from aioresponses import CallbackResult
@@ -18,8 +20,8 @@ class CombinedStorage(Storage):
     storage_name = "http_and_singlefile"
 
     def __init__(self, url, path, *, connector, **kwargs):
-        if kwargs.get("collection", None) is not None:
-            raise ValueError()
+        if kwargs.get("collection") is not None:
+            raise ValueError
 
         super().__init__(**kwargs)
         self.url = url

tests/storage/test_memory.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from vdirsyncer.storage.memory import MemoryStorage
@@ -6,7 +8,6 @@ from . import StorageTests
 
 
 class TestMemoryStorage(StorageTests):
-
     storage_class = MemoryStorage
     supports_collections = False
 

tests/storage/test_singlefile.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from vdirsyncer.storage.singlefile import SingleFileStorage
@@ -6,7 +8,6 @@ from . import StorageTests
 
 
 class TestSingleFileStorage(StorageTests):
-
     storage_class = SingleFileStorage
     supports_metadata = False
 

tests/system/cli/conftest.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from textwrap import dedent
 
 import pytest

tests/system/cli/test_config.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import io
 from textwrap import dedent
 
@@ -24,7 +26,7 @@ def read_config(tmpdir, monkeypatch):
 
 
 def test_read_config(read_config):
-    errors, c = read_config(
+    _errors, c = read_config(
         """
         [general]
         status_path = "/tmp/status/"
@@ -220,3 +222,62 @@ def test_validate_collections_param():
     x([["c", None, "b"]])
     x([["c", "a", None]])
     x([["c", None, None]])
+
+
+def test_invalid_implicit_value(read_config):
+    expected_message = "`implicit` parameter must be 'create' or absent"
+    with pytest.raises(exceptions.UserError) as excinfo:
+        read_config(
+            """
+            [general]
+            status_path = "/tmp/status/"
+
+            [pair my_pair]
+            a = "my_a"
+            b = "my_b"
+            collections = null
+            implicit = "invalid"
+
+            [storage my_a]
+            type = "filesystem"
+            path = "{base}/path_a/"
+            fileext = ".txt"
+
+            [storage my_b]
+            type = "filesystem"
+            path = "{base}/path_b/"
+            fileext = ".txt"
+            """
+        )
+
+    assert expected_message in str(excinfo.value)
+
+
+def test_implicit_create_only(read_config):
+    """Test that implicit create works."""
+    errors, c = read_config(
+        """
+        [general]
+        status_path = "/tmp/status/"
+
+        [pair my_pair]
+        a = "my_a"
+        b = "my_b"
+        collections = ["from a", "from b"]
+        implicit = "create"
+
+        [storage my_a]
+        type = "filesystem"
+        path = "{base}/path_a/"
+        fileext = ".txt"
+
+        [storage my_b]
+        type = "filesystem"
+        path = "{base}/path_b/"
+        fileext = ".txt"
+        """
+    )
+
+    assert not errors
+    pair = c.pairs["my_pair"]
+    assert pair.implicit == "create"

tests/system/cli/test_discover.py

@@ -1,6 +1,7 @@
+from __future__ import annotations
+
 import json
 from textwrap import dedent
-from typing import List
 
 import pytest
 
@@ -152,7 +153,7 @@ def test_discover_direct_path(tmpdir, runner):
 def test_null_collection_with_named_collection(tmpdir, runner):
     runner.write_with_general(
         dedent(
-            """
+            f"""
     [pair foobar]
     a = "foo"
     b = "bar"
@@ -160,15 +161,13 @@ def test_null_collection_with_named_collection(tmpdir, runner):
 
     [storage foo]
     type = "filesystem"
-    path = "{base}/foo/"
+    path = "{tmpdir!s}/foo/"
     fileext = ".txt"
 
     [storage bar]
     type = "singlefile"
-    path = "{base}/bar.txt"
-    """.format(
-                base=str(tmpdir)
-            )
+    path = "{tmpdir!s}/bar.txt"
+    """
         )
     )
 
@@ -192,7 +191,7 @@ def test_null_collection_with_named_collection(tmpdir, runner):
 
 
 @pytest.mark.parametrize(
-    "a_requires,b_requires",
+    ("a_requires", "b_requires"),
     [
         (True, True),
         (True, False),
@@ -207,13 +206,13 @@ def test_collection_required(a_requires, b_requires, tmpdir, runner, monkeypatch
         def __init__(self, require_collection, **kw):
             if require_collection:
                 assert not kw.get("collection")
-                raise exceptions.CollectionRequired()
+                raise exceptions.CollectionRequired
 
         async def get(self, href: str):
-            raise NotImplementedError()
+            raise NotImplementedError
 
-        async def list(self) -> List[tuple]:
-            raise NotImplementedError()
+        async def list(self) -> list[tuple]:
+            raise NotImplementedError
 
     from vdirsyncer.cli.utils import storage_names
 
@@ -221,7 +220,7 @@ def test_collection_required(a_requires, b_requires, tmpdir, runner, monkeypatch
 
     runner.write_with_general(
         dedent(
-            """
+            f"""
     [pair foobar]
     a = "foo"
     b = "bar"
@@ -229,14 +228,12 @@ def test_collection_required(a_requires, b_requires, tmpdir, runner, monkeypatch
 
     [storage foo]
     type = "test"
-    require_collection = {a}
+    require_collection = {json.dumps(a_requires)}
 
     [storage bar]
     type = "test"
-    require_collection = {b}
-    """.format(
-                a=json.dumps(a_requires), b=json.dumps(b_requires)
-            )
+    require_collection = {json.dumps(b_requires)}
+    """
         )
     )
 

tests/system/cli/test_fetchparams.py

@@ -1,10 +1,12 @@
+from __future__ import annotations
+
 from textwrap import dedent
 
 
 def test_get_password_from_command(tmpdir, runner):
     runner.write_with_general(
         dedent(
-            """
+            f"""
         [pair foobar]
         a = "foo"
         b = "bar"
@@ -12,16 +14,14 @@ def test_get_password_from_command(tmpdir, runner):
 
         [storage foo]
         type.fetch = ["shell", "echo filesystem"]
-        path = "{base}/foo/"
+        path = "{tmpdir!s}/foo/"
         fileext.fetch = ["command", "echo", ".txt"]
 
         [storage bar]
         type = "filesystem"
-        path = "{base}/bar/"
+        path = "{tmpdir!s}/bar/"
         fileext.fetch = ["prompt", "Fileext for bar"]
-    """.format(
-                base=str(tmpdir)
-            )
+    """
         )
     )
 

tests/system/cli/test_repair.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from textwrap import dedent
 
 import pytest
@@ -56,7 +58,7 @@ def test_repair_uids(storage, runner, repair_uids):
     else:
         opt = ["--no-repair-unsafe-uid"]
 
-    result = runner.invoke(["repair"] + opt + ["foo"], input="y")
+    result = runner.invoke(["repair", *opt, "foo"], input="y")
     assert not result.exception
 
     if repair_uids:

tests/system/cli/test_sync.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import sys
 from textwrap import dedent
@@ -88,9 +90,7 @@ def test_empty_storage(tmpdir, runner):
     result = runner.invoke(["sync"])
     lines = result.output.splitlines()
     assert lines[0] == "Syncing my_pair"
-    assert lines[1].startswith(
-        "error: my_pair: " 'Storage "my_b" was completely emptied.'
-    )
+    assert lines[1].startswith('error: my_pair: Storage "my_b" was completely emptied.')
     assert result.exception
 
 
@@ -278,27 +278,24 @@ def test_multiple_pairs(tmpdir, runner):
     ],
 )
 def test_create_collections(collections, tmpdir, runner):
-
     runner.write_with_general(
         dedent(
-            """
+            f"""
     [pair foobar]
     a = "foo"
     b = "bar"
-    collections = {colls}
+    collections = {json.dumps(list(collections))}
 
     [storage foo]
     type = "filesystem"
-    path = "{base}/foo/"
+    path = "{tmpdir!s}/foo/"
     fileext = ".txt"
 
     [storage bar]
     type = "filesystem"
-    path = "{base}/bar/"
+    path = "{tmpdir!s}/bar/"
     fileext = ".txt"
-    """.format(
-                base=str(tmpdir), colls=json.dumps(list(collections))
-            )
+    """
         )
     )
 
@@ -316,7 +313,7 @@ def test_create_collections(collections, tmpdir, runner):
 def test_ident_conflict(tmpdir, runner):
     runner.write_with_general(
         dedent(
-            """
+            f"""
     [pair foobar]
     a = "foo"
     b = "bar"
@@ -324,16 +321,14 @@ def test_ident_conflict(tmpdir, runner):
 
     [storage foo]
     type = "filesystem"
-    path = "{base}/foo/"
+    path = "{tmpdir!s}/foo/"
     fileext = ".txt"
 
     [storage bar]
     type = "filesystem"
-    path = "{base}/bar/"
+    path = "{tmpdir!s}/bar/"
     fileext = ".txt"
-    """.format(
-                base=str(tmpdir)
-            )
+    """
         )
     )
 
@@ -363,7 +358,7 @@ def test_ident_conflict(tmpdir, runner):
 
 
 @pytest.mark.parametrize(
-    "existing,missing",
+    ("existing", "missing"),
     [
         ("foo", "bar"),
         ("bar", "foo"),
@@ -372,7 +367,7 @@ def test_ident_conflict(tmpdir, runner):
 def test_unknown_storage(tmpdir, runner, existing, missing):
     runner.write_with_general(
         dedent(
-            """
+            f"""
     [pair foobar]
     a = "foo"
     b = "bar"
@@ -380,11 +375,9 @@ def test_unknown_storage(tmpdir, runner, existing, missing):
 
     [storage {existing}]
     type = "filesystem"
-    path = "{base}/{existing}/"
+    path = "{tmpdir!s}/{existing}/"
     fileext = ".txt"
-    """.format(
-                base=str(tmpdir), existing=existing
-            )
+    """
         )
     )
 
@@ -394,10 +387,8 @@ def test_unknown_storage(tmpdir, runner, existing, missing):
     assert result.exception
 
     assert (
-        "Storage '{missing}' not found. "
-        "These are the configured storages: ['{existing}']".format(
-            missing=missing, existing=existing
-        )
+        f"Storage '{missing}' not found. "
+        f"These are the configured storages: ['{existing}']"
     ) in result.output
 
 
@@ -411,31 +402,29 @@ def test_no_configured_pairs(tmpdir, runner, cmd):
 
 
 @pytest.mark.parametrize(
-    "resolution,expect_foo,expect_bar",
+    ("resolution", "expect_foo", "expect_bar"),
     [(["command", "cp"], "UID:lol\nfööcontent", "UID:lol\nfööcontent")],
 )
 def test_conflict_resolution(tmpdir, runner, resolution, expect_foo, expect_bar):
     runner.write_with_general(
         dedent(
-            """
+            f"""
     [pair foobar]
     a = "foo"
     b = "bar"
     collections = null
-    conflict_resolution = {val}
+    conflict_resolution = {json.dumps(resolution)}
 
     [storage foo]
     type = "filesystem"
     fileext = ".txt"
-    path = "{base}/foo"
+    path = "{tmpdir!s}/foo"
 
     [storage bar]
     type = "filesystem"
     fileext = ".txt"
-    path = "{base}/bar"
-    """.format(
-                base=str(tmpdir), val=json.dumps(resolution)
-            )
+    path = "{tmpdir!s}/bar"
+    """
         )
     )
 
@@ -527,13 +516,11 @@ def test_fetch_only_necessary_params(tmpdir, runner):
     fetch_script = tmpdir.join("fetch_script")
     fetch_script.write(
         dedent(
-            """
+            f"""
     set -e
-    touch "{}"
+    touch "{fetched_file!s}"
     echo ".txt"
-    """.format(
-                str(fetched_file)
-            )
+    """
         )
     )
 
@@ -564,9 +551,7 @@ def test_fetch_only_necessary_params(tmpdir, runner):
     type = "filesystem"
     path = "{path}"
     fileext.fetch = ["command", "sh", "{script}"]
-    """.format(
-                path=str(tmpdir.mkdir("bogus")), script=str(fetch_script)
-            )
+    """.format(path=str(tmpdir.mkdir("bogus")), script=str(fetch_script))
         )
     )
 

tests/system/cli/test_utils.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import pytest
 
 from vdirsyncer import exceptions
@@ -12,7 +14,7 @@ def test_handle_cli_error(capsys):
     except BaseException:
         handle_cli_error()
 
-    out, err = capsys.readouterr()
+    _out, err = capsys.readouterr()
     assert "returned something vdirsyncer doesn't understand" in err
     assert "ayy lmao" in err
 

tests/system/conftest.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import ssl
 
 import pytest

tests/system/utils/test_main.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 
 import aiohttp
@@ -20,30 +22,34 @@ def test_get_storage_init_args():
     from vdirsyncer.storage.memory import MemoryStorage
 
     all, required = utils.get_storage_init_args(MemoryStorage)
-    assert all == {"fileext", "collection", "read_only", "instance_name"}
+    assert all == {"fileext", "collection", "read_only", "instance_name", "no_delete"}
     assert not required
 
 
 @pytest.mark.asyncio
 async def test_request_ssl():
     async with aiohttp.ClientSession() as session:
-        with pytest.raises(aiohttp.ClientConnectorCertificateError) as excinfo:
+        with pytest.raises(
+            aiohttp.ClientConnectorCertificateError,
+            match="certificate verify failed",
+        ):
             await http.request(
                 "GET",
                 "https://self-signed.badssl.com/",
                 session=session,
             )
-        assert "certificate verify failed" in str(excinfo.value)
 
-        # XXX FIXME
 
-        with pytest.raises(Exception):
-            await http.request(
-                "GET",
-                "https://self-signed.badssl.com/",
-                verify=False,
-                session=session,
-            )
+@pytest.mark.xfail(reason="feature not implemented")
+@pytest.mark.asyncio
+async def test_request_unsafe_ssl():
+    async with aiohttp.ClientSession() as session:
+        await http.request(
+            "GET",
+            "https://self-signed.badssl.com/",
+            verify=False,
+            session=session,
+        )
 
 
 def fingerprint_of_cert(cert, hash=hashes.SHA256) -> str:

tests/unit/cli/test_config.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import os
 
 from vdirsyncer.cli.config import _resolve_conflict_via_command

tests/unit/cli/test_discover.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import aiostream
 import pytest
 
@@ -7,7 +9,7 @@ missing = object()
 
 
 @pytest.mark.parametrize(
-    "shortcuts,expected",
+    ("shortcuts", "expected"),
     [
         (
             ["from a"],

tests/unit/cli/test_fetchparams.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from contextlib import contextmanager
 from unittest.mock import patch
 
@@ -106,7 +108,7 @@ def test_failed_strategy(monkeypatch, value_cache):
 
     def strategy(x):
         calls.append(x)
-        raise KeyboardInterrupt()
+        raise KeyboardInterrupt
 
     monkeypatch.setitem(STRATEGIES, "mystrategy", strategy)
 

tests/unit/sync/test_status.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+import contextlib
+
 import hypothesis.strategies as st
 from hypothesis import assume
 from hypothesis import given
@@ -22,13 +26,13 @@ def test_legacy_status(status_dict):
     hrefs_a = {meta_a["href"] for meta_a, meta_b in status_dict.values()}
     hrefs_b = {meta_b["href"] for meta_a, meta_b in status_dict.values()}
     assume(len(hrefs_a) == len(status_dict) == len(hrefs_b))
-    status = SqliteStatus()
-    status.load_legacy_status(status_dict)
-    assert dict(status.to_legacy_status()) == status_dict
+    with contextlib.closing(SqliteStatus()) as status:
+        status.load_legacy_status(status_dict)
+        assert dict(status.to_legacy_status()) == status_dict
 
-    for ident, (meta_a, meta_b) in status_dict.items():
-        ident_a, meta2_a = status.get_by_href_a(meta_a["href"])
-        ident_b, meta2_b = status.get_by_href_b(meta_b["href"])
-        assert meta2_a.to_status() == meta_a
-        assert meta2_b.to_status() == meta_b
-        assert ident_a == ident_b == ident
+        for ident, (meta_a, meta_b) in status_dict.items():
+            ident_a, meta2_a = status.get_by_href_a(meta_a["href"])
+            ident_b, meta2_b = status.get_by_href_b(meta_b["href"])
+            assert meta2_a.to_status() == meta_a
+            assert meta2_b.to_status() == meta_b
+            assert ident_a == ident_b == ident

tests/unit/sync/test_sync.py

@@ -1,4 +1,7 @@
+from __future__ import annotations
+
 import asyncio
+import contextlib
 from copy import deepcopy
 
 import aiostream
@@ -23,13 +26,12 @@ from vdirsyncer.sync.status import SqliteStatus
 from vdirsyncer.vobject import Item
 
 
-async def sync(a, b, status, *args, **kwargs):
-    new_status = SqliteStatus(":memory:")
-    new_status.load_legacy_status(status)
-    rv = await _sync(a, b, new_status, *args, **kwargs)
-    status.clear()
-    status.update(new_status.to_legacy_status())
-    return rv
+async def sync(a, b, status, *args, **kwargs) -> None:
+    with contextlib.closing(SqliteStatus(":memory:")) as new_status:
+        new_status.load_legacy_status(status)
+        await _sync(a, b, new_status, *args, **kwargs)
+        status.clear()
+        status.update(new_status.to_legacy_status())
 
 
 def empty_storage(x):
@@ -96,7 +98,8 @@ async def test_read_only_and_prefetch():
     await sync(a, b, status, force_delete=True)
     await sync(a, b, status, force_delete=True)
 
-    assert not items(a) and not items(b)
+    assert not items(a)
+    assert not items(b)
 
 
 @pytest.mark.asyncio
@@ -224,7 +227,8 @@ async def test_insert_hash():
 
     await a.update(href, Item("UID:1\nHAHA:YES"), etag)
     await sync(a, b, status)
-    assert "hash" in status["1"][0] and "hash" in status["1"][1]
+    assert "hash" in status["1"][0]
+    assert "hash" in status["1"][1]
 
 
 @pytest.mark.asyncio
@@ -344,7 +348,7 @@ async def test_uses_get_multi(monkeypatch):
     a = MemoryStorage()
     b = MemoryStorage()
     item = Item("UID:1")
-    expected_href, etag = await a.upload(item)
+    expected_href, _etag = await a.upload(item)
 
     await sync(a, b, {})
     assert get_multi_calls == [[expected_href]]
@@ -381,7 +385,7 @@ async def test_changed_uids():
     a = MemoryStorage()
     b = MemoryStorage()
     href_a, etag_a = await a.upload(Item("UID:A-ONE"))
-    href_b, etag_b = await b.upload(Item("UID:B-ONE"))
+    _href_b, _etag_b = await b.upload(Item("UID:B-ONE"))
     status = {}
     await sync(a, b, status)
 
@@ -435,7 +439,7 @@ async def test_partial_sync_revert():
     assert items(a) == {"UID:2"}
 
 
-@pytest.mark.parametrize("sync_inbetween", (True, False))
+@pytest.mark.parametrize("sync_inbetween", [True, False])
 @pytest.mark.asyncio
 async def test_ident_conflict(sync_inbetween):
     a = MemoryStorage()
@@ -465,7 +469,7 @@ async def test_moved_href():
     a = MemoryStorage()
     b = MemoryStorage()
     status = {}
-    href, etag = await a.upload(Item("UID:haha"))
+    _href, _etag = await a.upload(Item("UID:haha"))
     await sync(a, b, status)
 
     b.items["lol"] = b.items.pop("haha")
@@ -526,7 +530,7 @@ async def test_unicode_hrefs():
     a = MemoryStorage()
     b = MemoryStorage()
     status = {}
-    href, etag = await a.upload(Item("UID:äää"))
+    _href, _etag = await a.upload(Item("UID:äää"))
     await sync(a, b, status)
 
 
@@ -535,7 +539,7 @@ class ActionIntentionallyFailed(Exception):
 
 
 def action_failure(*a, **kw):
-    raise ActionIntentionallyFailed()
+    raise ActionIntentionallyFailed
 
 
 class SyncMachine(RuleBasedStateMachine):
@@ -549,7 +553,7 @@ class SyncMachine(RuleBasedStateMachine):
         if flaky_etags:
 
             async def get(href):
-                old_etag, item = s.items[href]
+                _old_etag, item = s.items[href]
                 etag = _random_string()
                 s.items[href] = etag, item
                 return item, etag
@@ -640,10 +644,7 @@ class SyncMachine(RuleBasedStateMachine):
 
             errors = []
 
-            if with_error_callback:
-                error_callback = errors.append
-            else:
-                error_callback = None
+            error_callback = errors.append if with_error_callback else None
 
             try:
                 # If one storage is read-only, double-sync because changes don't
@@ -666,7 +667,8 @@ class SyncMachine(RuleBasedStateMachine):
             except ActionIntentionallyFailed:
                 pass
             except BothReadOnly:
-                assert a.read_only and b.read_only
+                assert a.read_only
+                assert b.read_only
                 assume(False)
             except StorageEmpty:
                 if force_delete:

tests/unit/test_exceptions.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from vdirsyncer import exceptions
 
 

tests/unit/test_metasync.py

@@ -1,3 +1,7 @@
+from __future__ import annotations
+
+import asyncio
+
 import hypothesis.strategies as st
 import pytest
 import pytest_asyncio
@@ -31,7 +35,8 @@ async def test_basic(monkeypatch):
 
     await a.set_meta("foo", None)
     await metasync(a, b, status, keys=["foo"])
-    assert await a.get_meta("foo") is None and await b.get_meta("foo") is None
+    assert await a.get_meta("foo") is None
+    assert await b.get_meta("foo") is None
 
     await a.set_meta("foo", "bar")
     await metasync(a, b, status, keys=["foo"])
@@ -50,27 +55,24 @@ async def test_basic(monkeypatch):
 
     await b.set_meta("foo", None)
     await metasync(a, b, status, keys=["foo"])
-    assert not await a.get_meta("foo") and not await b.get_meta("foo")
+    assert not await a.get_meta("foo")
+    assert not await b.get_meta("foo")
 
 
 @pytest_asyncio.fixture
-@pytest.mark.asyncio
-async def conflict_state(request, event_loop):
+async def conflict_state(request):
     a = MemoryStorage()
     b = MemoryStorage()
     status = {}
     await a.set_meta("foo", "bar")
     await b.set_meta("foo", "baz")
 
-    def cleanup():
-        async def do_cleanup():
-            assert await a.get_meta("foo") == "bar"
-            assert await b.get_meta("foo") == "baz"
-            assert not status
+    async def do_cleanup():
+        assert await a.get_meta("foo") == "bar"
+        assert await b.get_meta("foo") == "baz"
+        assert not status
 
-        event_loop.run_until_complete(do_cleanup())
-
-    request.addfinalizer(cleanup)
+    request.addfinalizer(lambda: asyncio.run(do_cleanup()))
 
     return a, b, status
 

tests/unit/test_repair.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import aiostream
 import pytest
 from hypothesis import HealthCheck
@@ -15,7 +17,7 @@ from vdirsyncer.vobject import Item
 
 @given(uid=uid_strategy)
 # Using the random module for UIDs:
-@settings(suppress_health_check=HealthCheck.all())
+@settings(suppress_health_check=list(HealthCheck))
 @pytest.mark.asyncio
 async def test_repair_uids(uid):
     s = MemoryStorage()
@@ -38,12 +40,12 @@ async def test_repair_uids(uid):
 
 @given(uid=uid_strategy.filter(lambda x: not href_safe(x)))
 # Using the random module for UIDs:
-@settings(suppress_health_check=HealthCheck.all())
+@settings(suppress_health_check=list(HealthCheck))
 @pytest.mark.asyncio
 async def test_repair_unsafe_uids(uid):
     s = MemoryStorage()
     item = Item(f"BEGIN:VCARD\nUID:{uid}\nEND:VCARD")
-    href, etag = await s.upload(item)
+    href, _etag = await s.upload(item)
     assert (await s.get(href))[0].uid == uid
     assert not href_safe(uid)
 
@@ -56,7 +58,7 @@ async def test_repair_unsafe_uids(uid):
 
 
 @pytest.mark.parametrize(
-    "uid,href", [("b@dh0mbr3", "perfectly-fine"), ("perfectly-fine", "b@dh0mbr3")]
+    ("uid", "href"), [("b@dh0mbr3", "perfectly-fine"), ("perfectly-fine", "b@dh0mbr3")]
 )
 def test_repair_unsafe_href(uid, href):
     item = Item(f"BEGIN:VCARD\nUID:{uid}\nEND:VCARD")

tests/unit/test_retry.py

@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+import json
+from unittest.mock import AsyncMock
+from unittest.mock import Mock
+
+import aiohttp
+import pytest
+
+from vdirsyncer.http import UsageLimitReached
+from vdirsyncer.http import request
+
+
+async def _create_mock_response(status: int, body: str | dict):
+    raw_body = body
+    text_body = json.dumps(body) if isinstance(body, dict) else body
+
+    mock_response = AsyncMock()
+    mock_response.status = status
+    mock_response.ok = 200 <= status < 300
+    mock_response.reason = "OK" if mock_response.ok else "Forbidden"
+    mock_response.headers = (
+        {"Content-Type": "application/json"}
+        if isinstance(raw_body, dict)
+        else {"Content-Type": "text/plain"}
+    )
+    mock_response.text.return_value = text_body
+    if isinstance(raw_body, dict):
+        mock_response.json.return_value = raw_body
+    else:
+        mock_response.json.side_effect = ValueError("Not JSON")
+    mock_response.raise_for_status = Mock(
+        side_effect=(
+            aiohttp.ClientResponseError(
+                request_info=AsyncMock(),
+                history=(),
+                status=status,
+                message=mock_response.reason,
+                headers=mock_response.headers,
+            )
+            if not mock_response.ok
+            else None
+        )
+    )
+
+    return mock_response
+
+
+@pytest.mark.asyncio
+async def test_request_retry_on_usage_limit():
+    url = "http://example.com/api"
+    max_retries = 5  # As configured in the @retry decorator
+
+    mock_session = AsyncMock()
+
+    # Simulate (max_retries - 1) 403 errors and then a 200 OK
+    mock_session.request.side_effect = [
+        await _create_mock_response(
+            403,
+            {
+                "error": {
+                    "errors": [{"domain": "usageLimits", "reason": "quotaExceeded"}]
+                }
+            },
+        )
+        for _ in range(max_retries - 1)
+    ] + [await _create_mock_response(200, "OK")]
+
+    async with (
+        aiohttp.ClientSession()
+    ):  # Dummy session. Will be replaced by mock_session at call
+        response = await request("GET", url, mock_session)
+
+        assert response.status == 200
+        assert mock_session.request.call_count == max_retries
+
+
+@pytest.mark.asyncio
+async def test_request_retry_exceeds_max_attempts():
+    url = "http://example.com/api"
+    max_retries = 5  # As configured in the @retry decorator
+
+    mock_session = AsyncMock()
+    # Simulate max_retries 403 errors and then a 200 OK
+    mock_session.request.side_effect = [
+        await _create_mock_response(
+            403,
+            {
+                "error": {
+                    "errors": [{"domain": "usageLimits", "reason": "quotaExceeded"}]
+                }
+            },
+        )
+        for _ in range(max_retries)
+    ]
+
+    async with (
+        aiohttp.ClientSession()
+    ):  # Dummy session. Will be replaced by mock_session at call
+        with pytest.raises(UsageLimitReached):
+            await request("GET", url, mock_session)
+        assert mock_session.request.call_count == max_retries
+
+
+@pytest.mark.asyncio
+async def test_request_no_retry_on_generic_403_json():
+    url = "http://example.com/api"
+
+    mock_session = AsyncMock()
+    # Generic non-Google 403 error payload (e.g., GitHub-style)
+    mock_session.request.side_effect = [
+        await _create_mock_response(403, {"message": "API rate limit exceeded"})
+    ]
+
+    async with aiohttp.ClientSession():
+        with pytest.raises(aiohttp.ClientResponseError):
+            await request("GET", url, mock_session)
+        # Should not retry because it's not the Google quotaExceeded shape
+        assert mock_session.request.call_count == 1
+
+
+@pytest.mark.asyncio
+async def test_request_no_retry_on_generic_403_text():
+    url = "http://example.com/api"
+
+    mock_session = AsyncMock()
+    # Plain-text 403 body mentioning rate limits, but not structured as Google error
+    mock_session.request.side_effect = [
+        await _create_mock_response(403, "Rate limit exceeded")
+    ]
+
+    async with aiohttp.ClientSession():
+        with pytest.raises(aiohttp.ClientResponseError):
+            await request("GET", url, mock_session)
+        # Should not retry because the JSON shape is not Google quotaExceeded
+        assert mock_session.request.call_count == 1

tests/unit/utils/test_vobject.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 from textwrap import dedent
 
 import hypothesis.strategies as st
@@ -23,7 +25,7 @@ _simple_split = [
 ]
 
 _simple_joined = "\r\n".join(
-    ["BEGIN:VADDRESSBOOK"] + _simple_split + ["END:VADDRESSBOOK\r\n"]
+    ["BEGIN:VADDRESSBOOK", *_simple_split, "END:VADDRESSBOOK\r\n"]
 )
 
 
@@ -122,7 +124,7 @@ def test_split_collection_timezones():
         "END:VTIMEZONE"
     )
 
-    full = "\r\n".join(["BEGIN:VCALENDAR"] + items + [timezone, "END:VCALENDAR"])
+    full = "\r\n".join(["BEGIN:VCALENDAR", *items, timezone, "END:VCALENDAR"])
 
     given = {normalize_item(item) for item in vobject.split_collection(full)}
     expected = {
@@ -152,7 +154,7 @@ def test_hash_item():
 
 
 def test_multiline_uid(benchmark):
-    a = "BEGIN:FOO\r\n" "UID:123456789abcd\r\n" " efgh\r\n" "END:FOO\r\n"
+    a = "BEGIN:FOO\r\nUID:123456789abcd\r\n efgh\r\nEND:FOO\r\n"
     assert benchmark(lambda: vobject.Item(a).uid) == "123456789abcdefgh"
 
 
@@ -235,6 +237,31 @@ def test_broken_item():
     assert item.parsed is None
 
 
+def test_mismatched_end():
+    with pytest.raises(ValueError) as excinfo:
+        vobject._Component.parse(
+            [
+                "BEGIN:FOO",
+                "END:BAR",
+            ]
+        )
+
+    assert "Got END:BAR, expected END:FOO at line 2" in str(excinfo.value)
+
+
+def test_missing_end():
+    with pytest.raises(ValueError) as excinfo:
+        vobject._Component.parse(
+            [
+                "BEGIN:FOO",
+                "BEGIN:BAR",
+                "END:BAR",
+            ]
+        )
+
+    assert "Missing END for component(s): FOO" in str(excinfo.value)
+
+
 def test_multiple_items():
     with pytest.raises(ValueError) as excinfo:
         vobject._Component.parse(
@@ -272,7 +299,7 @@ def test_input_types():
 
 value_strategy = st.text(
     st.characters(
-        blacklist_categories=("Zs", "Zl", "Zp", "Cc", "Cs"), blacklist_characters=":="
+        exclude_categories=("Zs", "Zl", "Zp", "Cc", "Cs"), exclude_characters=":="
     ),
     min_size=1,
 ).filter(lambda x: x.strip() == x)
@@ -308,7 +335,8 @@ class VobjectMachine(RuleBasedStateMachine):
         assert key in c
         assert c.get(key) == value
         dump = "\r\n".join(c.dump_lines())
-        assert key in dump and value in dump
+        assert key in dump
+        assert value in dump
 
     @rule(
         c=Parsed,
@@ -338,6 +366,16 @@ class VobjectMachine(RuleBasedStateMachine):
 TestVobjectMachine = VobjectMachine.TestCase
 
 
+def test_dupe_consecutive_keys():
+    state = VobjectMachine()
+    unparsed_0 = state.get_unparsed_lines(encoded=False, joined=False)
+    parsed_0 = state.parse(unparsed=unparsed_0)
+    state.add_prop_raw(c=parsed_0, key="0", params=[], value="0")
+    state.add_prop_raw(c=parsed_0, key="0", params=[], value="0")
+    state.add_prop(c=parsed_0, key="0", value="1")
+    state.teardown()
+
+
 def test_component_contains():
     item = vobject._Component.parse(["BEGIN:FOO", "FOO:YES", "END:FOO"])
 
@@ -345,4 +383,4 @@ def test_component_contains():
     assert "BAZ" not in item
 
     with pytest.raises(ValueError):
-        42 in item  # noqa: B015
+        42 in item  # noqa: B015, this check raises.

vdirsyncer/__init__.py

@@ -2,13 +2,14 @@
 Vdirsyncer synchronizes calendars and contacts.
 """
 
+from __future__ import annotations
 
 PROJECT_HOME = "https://github.com/pimutils/vdirsyncer"
 BUGTRACKER_HOME = PROJECT_HOME + "/issues"
 DOCS_HOME = "https://vdirsyncer.pimutils.org/en/stable"
 
 try:
-    from .version import version as __version__  # noqa
+    from .version import version as __version__
 except ImportError:  # pragma: no cover
     raise ImportError(
         "Failed to find (autogenerated) version.py. "
@@ -16,12 +17,14 @@ except ImportError:  # pragma: no cover
         "use the PyPI ones."
     )
 
+__all__ = ["__version__"]
 
-def _check_python_version():  # pragma: no cover
+
+def _check_python_version():
     import sys
 
-    if sys.version_info < (3, 7, 0):
-        print("vdirsyncer requires at least Python 3.7.")
+    if sys.version_info < (3, 9, 0):  # noqa: UP036
+        print("vdirsyncer requires at least Python 3.9.")
         sys.exit(1)
 
 

vdirsyncer/__main__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 if __name__ == "__main__":
     from vdirsyncer.cli import app
 

vdirsyncer/cli/__init__.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import functools
 import json
@@ -8,12 +10,15 @@ import aiohttp
 import click
 import click_log
 
-from .. import BUGTRACKER_HOME
-from .. import __version__
+from vdirsyncer import BUGTRACKER_HOME
+from vdirsyncer import __version__
 
 cli_logger = logging.getLogger(__name__)
 click_log.basic_config("vdirsyncer")
 
+# add short option for the help option
+click_context_settings = {"help_option_names": ["-h", "--help"]}
+
 
 class AppContext:
     def __init__(self):
@@ -39,13 +44,13 @@ def catch_errors(f):
     return inner
 
 
-@click.group()
+@click.group(context_settings=click_context_settings)
 @click_log.simple_verbosity_option("vdirsyncer")
 @click.version_option(version=__version__)
 @click.option("--config", "-c", metavar="FILE", help="Config file to use.")
 @pass_context
 @catch_errors
-def app(ctx, config):
+def app(ctx, config: str):
     """
     Synchronize calendars and contacts
     """
@@ -54,7 +59,7 @@ def app(ctx, config):
         cli_logger.warning(
             "Vdirsyncer currently does not support Windows. "
             "You will likely encounter bugs. "
-            "See {}/535 for more information.".format(BUGTRACKER_HOME)
+            f"See {BUGTRACKER_HOME}/535 for more information."
         )
 
     if not ctx.config:
@@ -63,9 +68,6 @@ def app(ctx, config):
         ctx.config = load_config(config)
 
 
-main = app
-
-
 def collections_arg_callback(ctx, param, value):
     """
     Expand the various CLI shortforms ("pair, pair/collection") to an iterable
@@ -145,7 +147,14 @@ def sync(ctx, collections, force_delete):
                         )
                     )
 
-            await asyncio.gather(*tasks)
+            # `return_exceptions=True` ensures that the event loop lives long enough for
+            # backoffs to be able to finish
+            gathered = await asyncio.gather(*tasks, return_exceptions=True)
+            # but now we need to manually check for and propogate a single failure after
+            # allowing all tasks to finish in order to keep exit status non-zero
+            failures = [e for e in gathered if isinstance(e, BaseException)]
+            if failures:
+                raise failures[0]
 
     asyncio.run(main(collections))
 
@@ -165,7 +174,6 @@ def metasync(ctx, collections):
 
     async def main(collection_names):
         async with aiohttp.TCPConnector(limit_per_host=16) as conn:
-
             for pair_name, collections in collection_names:
                 collections = prepare_pair(
                     pair_name=pair_name,

vdirsyncer/cli/config.py

@@ -3,13 +3,18 @@ from __future__ import annotations
 import json
 import os
 import string
+from collections.abc import Generator
 from configparser import RawConfigParser
+from functools import cached_property
 from itertools import chain
+from typing import IO
+from typing import Any
+
+from vdirsyncer import PROJECT_HOME
+from vdirsyncer import exceptions
+from vdirsyncer.utils import expand_path
+from vdirsyncer.vobject import Item
 
-from .. import PROJECT_HOME
-from .. import exceptions
-from ..utils import cached_property
-from ..utils import expand_path
 from .fetchparams import expand_fetch_params
 from .utils import storage_class_from_config
 
@@ -23,16 +28,16 @@ def validate_section_name(name, section_type):
     if invalid:
         chars_display = "".join(sorted(SECTION_NAME_CHARS))
         raise exceptions.UserError(
-            'The {}-section "{}" contains invalid characters. Only '
+            f'The {section_type}-section "{name}" contains invalid characters. Only '
             "the following characters are allowed for storage and "
-            "pair names:\n{}".format(section_type, name, chars_display)
+            f"pair names:\n{chars_display}"
         )
 
 
-def _validate_general_section(general_config):
+def _validate_general_section(general_config: dict[str, str]):
     invalid = set(general_config) - GENERAL_ALL
     missing = GENERAL_REQUIRED - set(general_config)
-    problems = []
+    problems: list[str] = []
 
     if invalid:
         problems.append(
@@ -47,7 +52,7 @@ def _validate_general_section(general_config):
     if problems:
         raise exceptions.UserError(
             "Invalid general section. Copy the example "
-            "config from the repository and edit it: {}".format(PROJECT_HOME),
+            f"config from the repository and edit it: {PROJECT_HOME}",
             problems=problems,
         )
 
@@ -88,21 +93,31 @@ def _validate_collections_param(collections):
                 raise ValueError("Duplicate value.")
             collection_names.add(collection_name)
         except ValueError as e:
-            raise ValueError(f"`collections` parameter, position {i}: {str(e)}")
+            raise ValueError(f"`collections` parameter, position {i}: {e!s}")
+
+
+def _validate_implicit_param(implicit):
+    if implicit is None:
+        return
+
+    if implicit != "create":
+        raise ValueError("`implicit` parameter must be 'create' or absent.")
 
 
 class _ConfigReader:
-    def __init__(self, f):
-        self._file = f
+    def __init__(self, f: IO[Any]):
+        self._file: IO[Any] = f
         self._parser = c = RawConfigParser()
         c.read_file(f)
-        self._seen_names = set()
+        self._seen_names: set = set()
 
-        self._general = {}
-        self._pairs = {}
-        self._storages = {}
+        self._general: dict[str, str] = {}
+        self._pairs: dict[str, dict[str, str]] = {}
+        self._storages: dict[str, dict[str, str]] = {}
 
-    def _parse_section(self, section_type, name, options):
+    def _parse_section(
+        self, section_type: str, name: str, options: dict[str, Any]
+    ) -> None:
         validate_section_name(name, section_type)
         if name in self._seen_names:
             raise ValueError(f'Name "{name}" already used.')
@@ -119,7 +134,9 @@ class _ConfigReader:
         else:
             raise ValueError("Unknown section type.")
 
-    def parse(self):
+    def parse(
+        self,
+    ) -> tuple[dict[str, str], dict[str, dict[str, str]], dict[str, dict[str, str]]]:
         for section in self._parser.sections():
             if " " in section:
                 section_type, name = section.split(" ", 1)
@@ -133,7 +150,7 @@ class _ConfigReader:
                     dict(_parse_options(self._parser.items(section), section=section)),
                 )
             except ValueError as e:
-                raise exceptions.UserError(f'Section "{section}": {str(e)}')
+                raise exceptions.UserError(f'Section "{section}": {e!s}')
 
         _validate_general_section(self._general)
         if getattr(self._file, "name", None):
@@ -145,7 +162,9 @@ class _ConfigReader:
         return self._general, self._pairs, self._storages
 
 
-def _parse_options(items, section=None):
+def _parse_options(
+    items: list[tuple[str, str]], section: str | None = None
+) -> Generator[tuple[str, dict[str, str]], None, None]:
     for key, value in items:
         try:
             yield key, json.loads(value)
@@ -154,13 +173,18 @@ def _parse_options(items, section=None):
 
 
 class Config:
-    def __init__(self, general, pairs, storages):
+    def __init__(
+        self,
+        general: dict[str, str],
+        pairs: dict[str, dict[str, str]],
+        storages: dict[str, dict[str, str]],
+    ) -> None:
         self.general = general
         self.storages = storages
         for name, options in storages.items():
             options["instance_name"] = name
 
-        self.pairs = {}
+        self.pairs: dict[str, PairConfig] = {}
         for name, options in pairs.items():
             try:
                 self.pairs[name] = PairConfig(self, name, options)
@@ -168,12 +192,12 @@ class Config:
                 raise exceptions.UserError(f"Pair {name}: {e}")
 
     @classmethod
-    def from_fileobject(cls, f):
+    def from_fileobject(cls, f: IO[Any]):
         reader = _ConfigReader(f)
         return cls(*reader.parse())
 
     @classmethod
-    def from_filename_or_environment(cls, fname=None):
+    def from_filename_or_environment(cls, fname: str | None = None):
         if fname is None:
             fname = os.environ.get("VDIRSYNCER_CONFIG", None)
         if fname is None:
@@ -190,15 +214,13 @@ class Config:
         except Exception as e:
             raise exceptions.UserError(f"Error during reading config {fname}: {e}")
 
-    def get_storage_args(self, storage_name):
+    def get_storage_args(self, storage_name: str):
         try:
             args = self.storages[storage_name]
         except KeyError:
             raise exceptions.UserError(
-                "Storage {!r} not found. "
-                "These are the configured storages: {}".format(
-                    storage_name, list(self.storages)
-                )
+                f"Storage {storage_name!r} not found. "
+                f"These are the configured storages: {list(self.storages)}"
             )
         else:
             return expand_fetch_params(args)
@@ -211,14 +233,15 @@ class Config:
 
 
 class PairConfig:
-    def __init__(self, full_config, name, options):
-        self._config = full_config
-        self.name = name
-        self.name_a = options.pop("a")
-        self.name_b = options.pop("b")
+    def __init__(self, full_config: Config, name: str, options: dict[str, str]):
+        self._config: Config = full_config
+        self.name: str = name
+        self.name_a: str = options.pop("a")
+        self.name_b: str = options.pop("b")
+        self.implicit = options.pop("implicit", None)
 
-        self._partial_sync = options.pop("partial_sync", None)
-        self.metadata = options.pop("metadata", None) or ()
+        self._partial_sync: str | None = options.pop("partial_sync", None)
+        self.metadata: str | tuple[()] = options.pop("metadata", ())
 
         self.conflict_resolution = self._process_conflict_resolution_param(
             options.pop("conflict_resolution", None)
@@ -234,14 +257,17 @@ class PairConfig:
             )
         else:
             _validate_collections_param(self.collections)
+            _validate_implicit_param(self.implicit)
 
         if options:
             raise ValueError("Unknown options: {}".format(", ".join(options)))
 
-    def _process_conflict_resolution_param(self, conflict_resolution):
+    def _process_conflict_resolution_param(
+        self, conflict_resolution: str | list[str] | None
+    ):
         if conflict_resolution in (None, "a wins", "b wins"):
             return conflict_resolution
-        elif (
+        if (
             isinstance(conflict_resolution, list)
             and len(conflict_resolution) > 1
             and conflict_resolution[0] == "command"
@@ -255,8 +281,7 @@ class PairConfig:
                 return _resolve_conflict_via_command(a, b, command, a_name, b_name)
 
             return resolve
-        else:
-            raise ValueError("Invalid value for `conflict_resolution`.")
+        raise ValueError("Invalid value for `conflict_resolution`.")
 
     # The following parameters are lazily evaluated because evaluating
     # self.config_a would expand all `x.fetch` parameters. This is costly and
@@ -302,10 +327,10 @@ class PairConfig:
 
 
 class CollectionConfig:
-    def __init__(self, pair, name, config_a, config_b):
+    def __init__(self, pair, name: str, config_a, config_b):
         self.pair = pair
         self._config = pair._config
-        self.name = name
+        self.name: str = name
         self.config_a = config_a
         self.config_b = config_b
 
@@ -314,14 +339,16 @@ class CollectionConfig:
 load_config = Config.from_filename_or_environment
 
 
-def _resolve_conflict_via_command(a, b, command, a_name, b_name, _check_call=None):
+def _resolve_conflict_via_command(
+    a, b, command, a_name, b_name, _check_call=None
+) -> Item:
     import shutil
     import tempfile
 
     if _check_call is None:
         from subprocess import check_call as _check_call
 
-    from ..vobject import Item
+    from vdirsyncer.vobject import Item
 
     dir = tempfile.mkdtemp(prefix="vdirsyncer-conflict.")
     try:
@@ -334,7 +361,7 @@ def _resolve_conflict_via_command(a, b, command, a_name, b_name, _check_call=Non
             f.write(b.raw)
 
         command[0] = expand_path(command[0])
-        _check_call(command + [a_tmp, b_tmp])
+        _check_call([*command, a_tmp, b_tmp])
 
         with open(a_tmp) as f:
             new_a = f.read()
@@ -342,7 +369,7 @@ def _resolve_conflict_via_command(a, b, command, a_name, b_name, _check_call=Non
             new_b = f.read()
 
         if new_a != new_b:
-            raise exceptions.UserError("The two files are not completely " "equal.")
+            raise exceptions.UserError("The two files are not completely equal.")
         return Item(new_a)
     finally:
         shutil.rmtree(dir)

vdirsyncer/cli/discover.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import asyncio
 import hashlib
 import json
@@ -7,7 +9,8 @@ import sys
 import aiohttp
 import aiostream
 
-from .. import exceptions
+from vdirsyncer import exceptions
+
 from .utils import handle_collection_not_found
 from .utils import handle_storage_init_error
 from .utils import load_status
@@ -63,21 +66,19 @@ async def collections_for_pair(
                     rv["collections"], pair.config_a, pair.config_b
                 )
             )
-        elif rv:
+        if rv:
             raise exceptions.UserError(
                 "Detected change in config file, "
-                "please run `vdirsyncer discover {}`.".format(pair.name)
-            )
-        else:
-            raise exceptions.UserError(
-                "Please run `vdirsyncer discover {}` "
-                " before synchronization.".format(pair.name)
+                f"please run `vdirsyncer discover {pair.name}`."
             )
+        raise exceptions.UserError(
+            f"Please run `vdirsyncer discover {pair.name}`  before synchronization."
+        )
 
     logger.info(f"Discovering collections for pair {pair.name}")
 
-    a_discovered = _DiscoverResult(pair.config_a, connector=connector)
-    b_discovered = _DiscoverResult(pair.config_b, connector=connector)
+    a_discovered = DiscoverResult(pair.config_a, connector=connector)
+    b_discovered = DiscoverResult(pair.config_b, connector=connector)
 
     if list_collections:
         # TODO: We should gather data and THEN print, so it can be async.
@@ -92,24 +93,31 @@ async def collections_for_pair(
             connector=connector,
         )
 
+    async def _handle_collection_not_found(
+        config, collection, e=None, implicit_create=False
+    ):
+        return await handle_collection_not_found(
+            config, collection, e=e, implicit_create=pair.implicit == "create"
+        )
+
     # We have to use a list here because the special None/null value would get
     # mangled to string (because JSON objects always have string keys).
-    rv = await aiostream.stream.list(
+    rv = await aiostream.stream.list(  #  type: ignore[assignment]
         expand_collections(
             shortcuts=pair.collections,
             config_a=pair.config_a,
             config_b=pair.config_b,
             get_a_discovered=a_discovered.get_self,
             get_b_discovered=b_discovered.get_self,
-            _handle_collection_not_found=handle_collection_not_found,
+            _handle_collection_not_found=_handle_collection_not_found,
         )
     )
 
     await _sanity_check_collections(rv, connector=connector)
 
     save_status(
-        status_path,
-        pair.name,
+        base_path=status_path,
+        pair=pair.name,
         data_type="collections",
         data={
             "collections": list(
@@ -155,7 +163,7 @@ def _expand_collections_cache(collections, config_a, config_b):
         yield name, (a, b)
 
 
-class _DiscoverResult:
+class DiscoverResult:
     def __init__(self, config, *, connector):
         self._cls, _ = storage_class_from_config(config)
 
@@ -163,6 +171,7 @@ class _DiscoverResult:
             "CardDAVStorage",
             "CalDAVStorage",
             "GoogleCalendarStorage",
+            "GoogleContactsStorage",
         ]:
             assert connector is not None
             config["connector"] = connector
@@ -270,8 +279,8 @@ async def _print_collections(
 
         logger.debug("".join(traceback.format_tb(sys.exc_info()[2])))
         logger.warning(
-            "Failed to discover collections for {}, use `-vdebug` "
-            "to see the full traceback.".format(instance_name)
+            f"Failed to discover collections for {instance_name}, use `-vdebug` "
+            "to see the full traceback."
         )
         return
     logger.info(f"{instance_name}:")

vdirsyncer/cli/fetchparams.py

@@ -1,10 +1,13 @@
+from __future__ import annotations
+
 import logging
 
 import click
 
-from .. import exceptions
-from ..utils import expand_path
-from ..utils import synchronized
+from vdirsyncer import exceptions
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import synchronized
+
 from . import AppContext
 
 SUFFIX = ".fetch"
@@ -37,7 +40,7 @@ def _fetch_value(opts, key):
     try:
         ctx = click.get_current_context().find_object(AppContext)
         if ctx is None:
-            raise RuntimeError()
+            raise RuntimeError
         password_cache = ctx.fetched_params
     except RuntimeError:
         password_cache = {}
@@ -65,8 +68,7 @@ def _fetch_value(opts, key):
     else:
         if not rv:
             raise exceptions.UserError(
-                "Empty value for {}, this most likely "
-                "indicates an error.".format(key)
+                f"Empty value for {key}, this most likely indicates an error."
             )
         password_cache[cache_key] = rv
         return rv
@@ -86,7 +88,7 @@ def _strategy_command(*command: str, shell: bool = False):
         return stdout.strip("\n")
     except OSError as e:
         cmd = " ".join(expanded_command)
-        raise exceptions.UserError(f"Failed to execute command: {cmd}\n{str(e)}")
+        raise exceptions.UserError(f"Failed to execute command: {cmd}\n{e!s}")
 
 
 def _strategy_shell(*command: str):

vdirsyncer/cli/tasks.py

@@ -1,12 +1,15 @@
+from __future__ import annotations
+
 import json
 
 import aiohttp
 
-from .. import exceptions
-from .. import sync
+from vdirsyncer import exceptions
+from vdirsyncer import sync
+
 from .config import CollectionConfig
+from .discover import DiscoverResult
 from .discover import collections_for_pair
-from .discover import storage_class_from_config
 from .discover import storage_instance_from_config
 from .utils import JobFailed
 from .utils import cli_logger
@@ -33,10 +36,8 @@ async def prepare_pair(pair_name, collections, config, *, connector):
             config_a, config_b = all_collections[collection_name]
         except KeyError:
             raise exceptions.UserError(
-                "Pair {}: Collection {} not found. These are the "
-                "configured collections:\n{}".format(
-                    pair_name, json.dumps(collection_name), list(all_collections)
-                )
+                f"Pair {pair_name}: Collection {json.dumps(collection_name)} not found."
+                f"These are the configured collections:\n{list(all_collections)}"
             )
 
         collection = CollectionConfig(pair, collection_name, config_a, config_b)
@@ -80,12 +81,12 @@ async def sync_collection(
             )
 
         if sync_failed:
-            raise JobFailed()
+            raise JobFailed
     except JobFailed:
         raise
     except BaseException:
         handle_cli_error(status_name)
-        raise JobFailed()
+        raise JobFailed
 
 
 async def discover_collections(pair, **kwargs):
@@ -103,26 +104,26 @@ async def repair_collection(
     *,
     connector: aiohttp.TCPConnector,
 ):
-    from ..repair import repair_storage
+    from vdirsyncer.repair import repair_storage
 
     storage_name, collection = collection, None
     if "/" in storage_name:
         storage_name, collection = storage_name.split("/")
 
     config = config.get_storage_args(storage_name)
-    storage_type = config["type"]
+    # If storage type has a slash, ignore it and anything after it.
+    storage_type = config["type"].split("/")[0]
 
     if collection is not None:
         cli_logger.info("Discovering collections (skipping cache).")
-        cls, config = storage_class_from_config(config)
-        async for config in cls.discover(**config):
+        get_discovered = DiscoverResult(config, connector=connector)
+        discovered = await get_discovered.get_self()
+        for config in discovered.values():
             if config["collection"] == collection:
                 break
         else:
             raise exceptions.UserError(
-                "Couldn't find collection {} for storage {}.".format(
-                    collection, storage_name
-                )
+                f"Couldn't find collection {collection} for storage {storage_name}."
             )
 
     config["type"] = storage_type
@@ -134,7 +135,7 @@ async def repair_collection(
 
 
 async def metasync_collection(collection, general, *, connector: aiohttp.TCPConnector):
-    from ..metasync import metasync
+    from vdirsyncer.metasync import metasync
 
     pair = collection.pair
     status_name = get_status_name(pair.name, collection.name)
@@ -142,11 +143,11 @@ async def metasync_collection(collection, general, *, connector: aiohttp.TCPConn
     try:
         cli_logger.info(f"Metasyncing {status_name}")
 
-        status = (
-            load_status(
-                general["status_path"], pair.name, collection.name, data_type="metadata"
-            )
-            or {}
+        status = load_status(
+            general["status_path"],
+            pair.name,
+            collection.name,
+            data_type="metadata",
         )
 
         a = await storage_instance_from_config(collection.config_a, connector=connector)
@@ -161,12 +162,12 @@ async def metasync_collection(collection, general, *, connector: aiohttp.TCPConn
         )
     except BaseException:
         handle_cli_error(status_name)
-        raise JobFailed()
+        raise JobFailed
 
     save_status(
-        general["status_path"],
-        pair.name,
-        collection.name,
+        base_path=general["status_path"],
+        pair=pair.name,
         data_type="metadata",
         data=status,
+        collection=collection.name,
     )

vdirsyncer/cli/utils.py

@@ -1,24 +1,29 @@
+from __future__ import annotations
+
 import contextlib
 import errno
 import importlib
 import json
 import os
 import sys
+from typing import Any
 
 import aiohttp
 import click
-from atomicwrites import atomic_write
 
-from .. import BUGTRACKER_HOME
-from .. import DOCS_HOME
-from .. import exceptions
-from ..sync.exceptions import IdentConflict
-from ..sync.exceptions import PartialSync
-from ..sync.exceptions import StorageEmpty
-from ..sync.exceptions import SyncConflict
-from ..sync.status import SqliteStatus
-from ..utils import expand_path
-from ..utils import get_storage_init_args
+from vdirsyncer import BUGTRACKER_HOME
+from vdirsyncer import DOCS_HOME
+from vdirsyncer import exceptions
+from vdirsyncer.storage.base import Storage
+from vdirsyncer.sync.exceptions import IdentConflict
+from vdirsyncer.sync.exceptions import PartialSync
+from vdirsyncer.sync.exceptions import StorageEmpty
+from vdirsyncer.sync.exceptions import SyncConflict
+from vdirsyncer.sync.status import SqliteStatus
+from vdirsyncer.utils import atomic_write
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import get_storage_init_args
+
 from . import cli_logger
 
 STATUS_PERMISSIONS = 0o600
@@ -26,18 +31,18 @@ STATUS_DIR_PERMISSIONS = 0o700
 
 
 class _StorageIndex:
-    def __init__(self):
-        self._storages = dict(
-            caldav="vdirsyncer.storage.dav.CalDAVStorage",
-            carddav="vdirsyncer.storage.dav.CardDAVStorage",
-            filesystem="vdirsyncer.storage.filesystem.FilesystemStorage",
-            http="vdirsyncer.storage.http.HttpStorage",
-            singlefile="vdirsyncer.storage.singlefile.SingleFileStorage",
-            google_calendar="vdirsyncer.storage.google.GoogleCalendarStorage",
-            google_contacts="vdirsyncer.storage.google.GoogleContactsStorage",
-        )
+    def __init__(self) -> None:
+        self._storages: dict[str, str] = {
+            "caldav": "vdirsyncer.storage.dav.CalDAVStorage",
+            "carddav": "vdirsyncer.storage.dav.CardDAVStorage",
+            "filesystem": "vdirsyncer.storage.filesystem.FilesystemStorage",
+            "http": "vdirsyncer.storage.http.HttpStorage",
+            "singlefile": "vdirsyncer.storage.singlefile.SingleFileStorage",
+            "google_calendar": "vdirsyncer.storage.google.GoogleCalendarStorage",
+            "google_contacts": "vdirsyncer.storage.google.GoogleContactsStorage",
+        }
 
-    def __getitem__(self, name):
+    def __getitem__(self, name: str) -> Storage:
         item = self._storages[name]
         if not isinstance(item, str):
             return item
@@ -74,33 +79,27 @@ def handle_cli_error(status_name=None, e=None):
         cli_logger.critical(e)
     except StorageEmpty as e:
         cli_logger.error(
-            '{status_name}: Storage "{name}" was completely emptied. If you '
-            "want to delete ALL entries on BOTH sides, then use "
-            "`vdirsyncer sync --force-delete {status_name}`. "
-            "Otherwise delete the files for {status_name} in your status "
-            "directory.".format(
-                name=e.empty_storage.instance_name, status_name=status_name
-            )
+            f'{status_name}: Storage "{e.empty_storage.instance_name}" was '
+            "completely emptied. If you want to delete ALL entries on BOTH sides,"
+            f"then use `vdirsyncer sync --force-delete {status_name}`. "
+            f"Otherwise delete the files for {status_name} in your status "
+            "directory."
         )
     except PartialSync as e:
         cli_logger.error(
-            "{status_name}: Attempted change on {storage}, which is read-only"
+            f"{status_name}: Attempted change on {e.storage}, which is read-only"
             ". Set `partial_sync` in your pair section to `ignore` to ignore "
-            "those changes, or `revert` to revert them on the other side.".format(
-                status_name=status_name, storage=e.storage
-            )
+            "those changes, or `revert` to revert them on the other side."
         )
     except SyncConflict as e:
         cli_logger.error(
-            "{status_name}: One item changed on both sides. Resolve this "
+            f"{status_name}: One item changed on both sides. Resolve this "
             "conflict manually, or by setting the `conflict_resolution` "
             "parameter in your config file.\n"
-            "See also {docs}/config.html#pair-section\n"
-            "Item ID: {e.ident}\n"
-            "Item href on side A: {e.href_a}\n"
-            "Item href on side B: {e.href_b}\n".format(
-                status_name=status_name, e=e, docs=DOCS_HOME
-            )
+            f"See also {DOCS_HOME}/config.html#pair-section\n"
+            f"Item ID: {e.ident}\n"
+            f"Item href on side A: {e.href_a}\n"
+            f"Item href on side B: {e.href_b}\n"
         )
     except IdentConflict as e:
         cli_logger.error(
@@ -121,17 +120,17 @@ def handle_cli_error(status_name=None, e=None):
         pass
     except exceptions.PairNotFound as e:
         cli_logger.error(
-            "Pair {pair_name} does not exist. Please check your "
+            f"Pair {e.pair_name} does not exist. Please check your "
             "configuration file and make sure you've typed the pair name "
-            "correctly".format(pair_name=e.pair_name)
+            "correctly"
         )
     except exceptions.InvalidResponse as e:
         cli_logger.error(
             "The server returned something vdirsyncer doesn't understand. "
-            "Error message: {!r}\n"
+            f"Error message: {e!r}\n"
             "While this is most likely a serverside problem, the vdirsyncer "
             "devs are generally interested in such bugs. Please report it in "
-            "the issue tracker at {}".format(e, BUGTRACKER_HOME)
+            f"the issue tracker at {BUGTRACKER_HOME}"
         )
     except exceptions.CollectionRequired:
         cli_logger.error(
@@ -154,13 +153,18 @@ def handle_cli_error(status_name=None, e=None):
         cli_logger.debug("".join(tb))
 
 
-def get_status_name(pair, collection):
+def get_status_name(pair: str, collection: str | None) -> str:
     if collection is None:
         return pair
     return pair + "/" + collection
 
 
-def get_status_path(base_path, pair, collection=None, data_type=None):
+def get_status_path(
+    base_path: str,
+    pair: str,
+    collection: str | None = None,
+    data_type: str | None = None,
+) -> str:
     assert data_type is not None
     status_name = get_status_name(pair, collection)
     path = expand_path(os.path.join(base_path, status_name))
@@ -174,10 +178,15 @@ def get_status_path(base_path, pair, collection=None, data_type=None):
     return path
 
 
-def load_status(base_path, pair, collection=None, data_type=None):
+def load_status(
+    base_path: str,
+    pair: str,
+    collection: str | None = None,
+    data_type: str | None = None,
+) -> dict[str, Any]:
     path = get_status_path(base_path, pair, collection, data_type)
     if not os.path.exists(path):
-        return None
+        return {}
     assert_permissions(path, STATUS_PERMISSIONS)
 
     with open(path) as f:
@@ -189,7 +198,7 @@ def load_status(base_path, pair, collection=None, data_type=None):
     return {}
 
 
-def prepare_status_path(path):
+def prepare_status_path(path: str) -> None:
     dirname = os.path.dirname(path)
 
     try:
@@ -200,7 +209,7 @@ def prepare_status_path(path):
 
 
 @contextlib.contextmanager
-def manage_sync_status(base_path, pair_name, collection_name):
+def manage_sync_status(base_path: str, pair_name: str, collection_name: str):
     path = get_status_path(base_path, pair_name, collection_name, "items")
     status = None
     legacy_status = None
@@ -222,12 +231,17 @@ def manage_sync_status(base_path, pair_name, collection_name):
         prepare_status_path(path)
         status = SqliteStatus(path)
 
-    yield status
+    with contextlib.closing(status):
+        yield status
 
 
-def save_status(base_path, pair, collection=None, data_type=None, data=None):
-    assert data_type is not None
-    assert data is not None
+def save_status(
+    base_path: str,
+    pair: str,
+    data_type: str,
+    data: dict[str, Any],
+    collection: str | None = None,
+) -> None:
     status_name = get_status_name(pair, collection)
     path = expand_path(os.path.join(base_path, status_name)) + "." + data_type
     prepare_status_path(path)
@@ -272,15 +286,14 @@ async def storage_instance_from_config(
     except exceptions.CollectionNotFound as e:
         if create:
             config = await handle_collection_not_found(
-                config, config.get("collection", None), e=str(e)
+                config, config.get("collection", None), e=str(e), implicit_create=True
             )
             return await storage_instance_from_config(
                 config,
                 create=False,
                 connector=connector,
             )
-        else:
-            raise
+        raise
     except Exception:
         return handle_storage_init_error(cls, new_config)
 
@@ -319,18 +332,18 @@ def handle_storage_init_error(cls, config):
     )
 
 
-def assert_permissions(path, wanted):
+def assert_permissions(path: str, wanted: int) -> None:
     permissions = os.stat(path).st_mode & 0o777
     if permissions > wanted:
         cli_logger.warning(
-            "Correcting permissions of {} from {:o} to {:o}".format(
-                path, permissions, wanted
-            )
+            f"Correcting permissions of {path} from {permissions:o} to {wanted:o}"
         )
         os.chmod(path, wanted)
 
 
-async def handle_collection_not_found(config, collection, e=None):
+async def handle_collection_not_found(
+    config, collection, e=None, implicit_create=False
+):
     storage_name = config.get("instance_name", None)
 
     cli_logger.warning(
@@ -339,7 +352,7 @@ async def handle_collection_not_found(config, collection, e=None):
         )
     )
 
-    if click.confirm("Should vdirsyncer attempt to create it?"):
+    if implicit_create or click.confirm("Should vdirsyncer attempt to create it?"):
         storage_type = config["type"]
         cls, config = storage_class_from_config(config)
         config["collection"] = collection
@@ -351,7 +364,7 @@ async def handle_collection_not_found(config, collection, e=None):
             cli_logger.error(e)
 
     raise exceptions.UserError(
-        'Unable to find or create collection "{collection}" for '
-        'storage "{storage}". Please create the collection '
-        "yourself.".format(collection=collection, storage=storage_name)
+        f'Unable to find or create collection "{collection}" for '
+        f'storage "{storage_name}". Please create the collection '
+        "yourself."
     )

vdirsyncer/exceptions.py

@@ -3,6 +3,8 @@ Contains exception classes used by vdirsyncer. Not all exceptions are here,
 only the most commonly used ones.
 """
 
+from __future__ import annotations
+
 
 class Error(Exception):
     """Baseclass for all errors."""

vdirsyncer/http.py

@@ -1,9 +1,25 @@
+from __future__ import annotations
+
+import asyncio
 import logging
+import os
+import platform
+import re
+from abc import ABC
+from abc import abstractmethod
+from base64 import b64encode
 from ssl import create_default_context
 
 import aiohttp
+import requests.auth
+from aiohttp import ServerDisconnectedError
+from aiohttp import ServerTimeoutError
+from requests.utils import parse_dict_header
+from tenacity import retry
+from tenacity import retry_if_exception_type
+from tenacity import stop_after_attempt
+from tenacity import wait_exponential
 
-from . import DOCS_HOME
 from . import __version__
 from . import exceptions
 from .utils import expand_path
@@ -11,54 +27,101 @@ from .utils import expand_path
 logger = logging.getLogger(__name__)
 USERAGENT = f"vdirsyncer/{__version__}"
 
-
-def _detect_faulty_requests():  # pragma: no cover
-    text = (
-        "Error during import: {e}\n\n"
-        "If you have installed vdirsyncer from a distro package, please file "
-        "a bug against that package, not vdirsyncer.\n\n"
-        "Consult {d}/problems.html#requests-related-importerrors"
-        "-based-distributions on how to work around this."
-    )
-
-    try:
-        from requests_toolbelt.auth.guess import GuessAuth  # noqa
-    except ImportError as e:
-        import sys
-
-        print(text.format(e=str(e), d=DOCS_HOME), file=sys.stderr)
-        sys.exit(1)
+# 'hack' to prevent aiohttp from loading the netrc config,
+# but still allow it to read PROXY_* env vars.
+# Otherwise, if our host is defined in the netrc config,
+# aiohttp will overwrite our Authorization header.
+# https://github.com/pimutils/vdirsyncer/issues/1138
+os.environ["NETRC"] = "NUL" if platform.system() == "Windows" else "/dev/null"
 
 
-_detect_faulty_requests()
-del _detect_faulty_requests
+class AuthMethod(ABC):
+    def __init__(self, username, password):
+        self.username = username
+        self.password = password
+
+    @abstractmethod
+    def handle_401(self, response):
+        raise NotImplementedError
+
+    @abstractmethod
+    def get_auth_header(self, method, url):
+        raise NotImplementedError
+
+    def __eq__(self, other):
+        if not isinstance(other, AuthMethod):
+            return False
+        return (
+            self.__class__ == other.__class__
+            and self.username == other.username
+            and self.password == other.password
+        )
+
+
+class BasicAuthMethod(AuthMethod):
+    def handle_401(self, _response):
+        pass
+
+    def get_auth_header(self, _method, _url):
+        auth_str = f"{self.username}:{self.password}"
+        return "Basic " + b64encode(auth_str.encode("utf-8")).decode("utf-8")
+
+
+class DigestAuthMethod(AuthMethod):
+    # make class var to 'cache' the state, which is more efficient because otherwise
+    # each request would first require another 'initialization' request.
+    _auth_helpers: dict[tuple[str, str], requests.auth.HTTPDigestAuth] = {}
+
+    def __init__(self, username: str, password: str):
+        super().__init__(username, password)
+
+        self._auth_helper = self._auth_helpers.get(
+            (username, password), requests.auth.HTTPDigestAuth(username, password)
+        )
+        self._auth_helpers[(username, password)] = self._auth_helper
+
+    @property
+    def auth_helper_vars(self):
+        return self._auth_helper._thread_local
+
+    def handle_401(self, response):
+        s_auth = response.headers.get("www-authenticate", "")
+
+        if "digest" in s_auth.lower():
+            # Original source:
+            # https://github.com/psf/requests/blob/f12ccbef6d6b95564da8d22e280d28c39d53f0e9/src/requests/auth.py#L262-L263
+            pat = re.compile(r"digest ", flags=re.IGNORECASE)
+            self.auth_helper_vars.chal = parse_dict_header(pat.sub("", s_auth, count=1))
+
+    def get_auth_header(self, method, url):
+        self._auth_helper.init_per_thread_state()
+
+        if not self.auth_helper_vars.chal:
+            # Need to do init request first
+            return ""
+
+        return self._auth_helper.build_digest_header(method, url)
 
 
 def prepare_auth(auth, username, password):
     if username and password:
         if auth == "basic" or auth is None:
-            return aiohttp.BasicAuth(username, password)
-        elif auth == "digest":
-            from requests.auth import HTTPDigestAuth
-
-            return HTTPDigestAuth(username, password)
-        elif auth == "guess":
-            try:
-                from requests_toolbelt.auth.guess import GuessAuth
-            except ImportError:
-                raise exceptions.UserError(
-                    "Your version of requests_toolbelt is too "
-                    "old for `guess` authentication. At least "
-                    "version 0.4.0 is required."
-                )
-            else:
-                return GuessAuth(username, password)
+            return BasicAuthMethod(username, password)
+        if auth == "digest":
+            return DigestAuthMethod(username, password)
+        if auth == "guess":
+            raise exceptions.UserError(
+                "'Guess' authentication is not supported in this version of "
+                "vdirsyncer.\n"
+                "Please explicitly specify either 'basic' or 'digest' auth instead. \n"
+                "See the following issue for more information: "
+                "https://github.com/pimutils/vdirsyncer/issues/1015"
+            )
         else:
             raise exceptions.UserError(f"Unknown authentication method: {auth}")
     elif auth:
         raise exceptions.UserError(
-            "You need to specify username and password "
-            "for {} authentication.".format(auth)
+            f"You need to specify username and password for {auth} authentication."
         )
 
     return None
@@ -92,18 +155,79 @@ def prepare_client_cert(cert):
     return cert
 
 
+class TransientNetworkError(exceptions.Error):
+    """Transient network condition that should be retried."""
+
+
+def _is_safe_to_retry_method(method: str) -> bool:
+    """Returns True if the HTTP method is safe/idempotent to retry.
+
+    We consider these safe for our WebDAV usage:
+    - GET, HEAD, OPTIONS: standard safe methods
+    - PROPFIND, REPORT: read-only DAV queries used for listing/fetching
+    """
+    return method.upper() in {"GET", "HEAD", "OPTIONS", "PROPFIND", "REPORT"}
+
+
+class UsageLimitReached(exceptions.Error):
+    pass
+
+
+async def _is_quota_exceeded_google(response: aiohttp.ClientResponse) -> bool:
+    """Return True if the response JSON indicates Google-style `usageLimits` exceeded.
+
+    Expected shape:
+    {"error": {"errors": [{"domain": "usageLimits", ...}], ...}}
+
+    See https://developers.google.com/workspace/calendar/api/guides/errors#403_usage_limits_exceeded
+    """
+    try:
+        data = await response.json(content_type=None)
+    except Exception:
+        return False
+
+    if not isinstance(data, dict):
+        return False
+
+    error = data.get("error")
+    if not isinstance(error, dict):
+        return False
+
+    errors = error.get("errors")
+    if not isinstance(errors, list):
+        return False
+
+    for entry in errors:
+        if isinstance(entry, dict) and entry.get("domain") == "usageLimits":
+            return True
+
+    return False
+
+
+@retry(
+    stop=stop_after_attempt(5),
+    wait=wait_exponential(multiplier=1, min=4, max=10),
+    retry=(
+        retry_if_exception_type(UsageLimitReached)
+        | retry_if_exception_type(TransientNetworkError)
+    ),
+    reraise=True,
+)
 async def request(
     method,
     url,
     session,
+    auth=None,
     latin1_fallback=True,
     **kwargs,
 ):
-    """Wrapper method for requests, to ease logging and mocking.
+    """Wrapper method for requests, to ease logging and mocking as well as to
+    support auth methods currently unsupported by aiohttp.
 
-    Parameters should be the same as for ``aiohttp.request``, as well as:
+    Parameters should be the same as for ``aiohttp.request``, except:
 
     :param session: A requests session object to use.
+    :param auth: The HTTP ``AuthMethod`` to use for authentication.
     :param verify_fingerprint: Optional. SHA256 of the expected server certificate.
     :param latin1_fallback: RFC-2616 specifies the default Content-Type of
         text/* to be latin1, which is not always correct, but exactly what
@@ -122,14 +246,54 @@ async def request(
     logger.debug("=" * 20)
     logger.debug(f"{method} {url}")
     logger.debug(kwargs.get("headers", {}))
-    logger.debug(kwargs.get("data", None))
+    logger.debug(kwargs.get("data"))
     logger.debug("Sending request...")
 
     assert isinstance(kwargs.get("data", b""), bytes)
 
-    kwargs.pop("cert", None)  # TODO XXX FIXME!
+    cert = kwargs.pop("cert", None)
+    if cert is not None:
+        ssl_context = kwargs.pop("ssl", create_default_context())
+        ssl_context.load_cert_chain(*cert)
+        kwargs["ssl"] = ssl_context
 
-    response = await session.request(method, url, **kwargs)
+    headers = kwargs.pop("headers", {})
+    response: aiohttp.ClientResponse | None = None
+    for _attempt in range(2):
+        if auth:
+            headers["Authorization"] = auth.get_auth_header(method, url)
+        try:
+            response = await session.request(method, url, headers=headers, **kwargs)
+        except (
+            ServerDisconnectedError,
+            ServerTimeoutError,
+            asyncio.TimeoutError,
+        ) as e:
+            # Retry only if the method is safe/idempotent for our DAV use
+            if _is_safe_to_retry_method(method):
+                logger.debug(
+                    f"Transient network error on {method} {url}: {e}. Will retry."
+                )
+                raise TransientNetworkError(str(e)) from e
+            raise e from None
+
+        if response is None:
+            raise RuntimeError("No HTTP response obtained")
+
+        if response.ok or not auth:
+            # we don't need to do the 401-loop if we don't do auth in the first place
+            break
+
+        if response.status == 401:
+            auth.handle_401(response)
+            # retry once more after handling the 401 challenge
+            continue
+        else:
+            # some other error, will be handled later on
+            break
+
+    if response is None:
+        raise RuntimeError("No HTTP response obtained")
 
     # See https://github.com/kennethreitz/requests/issues/2042
     content_type = response.headers.get("Content-Type", "")
@@ -145,10 +309,18 @@ async def request(
     logger.debug(response.headers)
     logger.debug(response.content)
 
+    if logger.getEffectiveLevel() <= logging.DEBUG and response.status >= 400:
+        # https://github.com/pimutils/vdirsyncer/issues/1186
+        logger.debug(await response.text())
+
+    if response.status == 403 and await _is_quota_exceeded_google(response):
+        raise UsageLimitReached(response.reason)
     if response.status == 412:
         raise exceptions.PreconditionFailed(response.reason)
     if response.status in (404, 410):
         raise exceptions.NotFoundError(response.reason)
+    if response.status == 429:
+        raise UsageLimitReached(response.reason)
 
     response.raise_for_status()
     return response

vdirsyncer/metasync.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 
 from . import exceptions
@@ -55,7 +57,7 @@ async def metasync(storage_a, storage_b, status, keys, conflict_resolution=None)
         logger.debug(f"B: {b}")
         logger.debug(f"S: {s}")
 
-        if a != s and b != s or storage_a.read_only or storage_b.read_only:
+        if (a != s and b != s) or storage_a.read_only or storage_b.read_only:
             await _resolve_conflict()
         elif a != s and b == s:
             await _a_to_b()

vdirsyncer/repair.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import logging
 from os.path import basename
 
@@ -24,9 +26,9 @@ async def repair_storage(storage, repair_unsafe_uid):
             new_item = repair_item(href, item, seen_uids, repair_unsafe_uid)
         except IrreparableItem:
             logger.error(
-                "Item {!r} is malformed beyond repair. "
+                f"Item {href!r} is malformed beyond repair. "
                 "The PRODID property may indicate which software "
-                "created this item.".format(href)
+                "created this item."
             )
             logger.error(f"Item content: {item.raw!r}")
             continue
@@ -42,7 +44,7 @@ async def repair_storage(storage, repair_unsafe_uid):
 
 def repair_item(href, item, seen_uids, repair_unsafe_uid):
     if item.parsed is None:
-        raise IrreparableItem()
+        raise IrreparableItem
 
     new_item = item
 
@@ -54,14 +56,12 @@ def repair_item(href, item, seen_uids, repair_unsafe_uid):
         new_item = item.with_uid(generate_href())
     elif not href_safe(item.uid) or not href_safe(basename(href)):
         if not repair_unsafe_uid:
-            logger.warning(
-                "UID may cause problems, add " "--repair-unsafe-uid to repair."
-            )
+            logger.warning("UID may cause problems, add --repair-unsafe-uid to repair.")
         else:
             logger.warning("UID or href is unsafe, assigning random UID.")
             new_item = item.with_uid(generate_href())
 
     if not new_item.uid:
-        raise IrreparableItem()
+        raise IrreparableItem
 
     return new_item

vdirsyncer/storage/base.py

@@ -1,16 +1,15 @@
+from __future__ import annotations
+
 import contextlib
 import functools
 from abc import ABCMeta
 from abc import abstractmethod
-from typing import Iterable
-from typing import List
-from typing import Optional
+from collections.abc import Iterable
 
+from vdirsyncer import exceptions
+from vdirsyncer.utils import uniq
 from vdirsyncer.vobject import Item
 
-from .. import exceptions
-from ..utils import uniq
-
 
 def mutating_storage_method(f):
     """Wrap a method and fail if the instance is readonly."""
@@ -34,7 +33,6 @@ class StorageMeta(ABCMeta):
 
 
 class Storage(metaclass=StorageMeta):
-
     """Superclass of all storages, interface that all storages have to
     implement.
 
@@ -67,21 +65,37 @@ class Storage(metaclass=StorageMeta):
     # The machine-readable name of this collection.
     collection = None
 
+    # A value of False means storage does not support delete requests. A
+    # value of True mean the storage supports it.
+    no_delete = False
+
     # A value of True means the storage does not support write-methods such as
     # upload, update and delete.  A value of False means the storage does
     # support those methods.
     read_only = False
 
     # The attribute values to show in the representation of the storage.
-    _repr_attributes: List[str] = []
+    _repr_attributes: tuple[str, ...] = ()
 
-    def __init__(self, instance_name=None, read_only=None, collection=None):
+    def __init__(
+        self,
+        instance_name=None,
+        read_only=None,
+        no_delete=None,
+        collection=None,
+    ):
         if read_only is None:
             read_only = self.read_only
         if self.read_only and not read_only:
             raise exceptions.UserError("This storage can only be read-only.")
         self.read_only = bool(read_only)
 
+        if no_delete is None:
+            no_delete = self.no_delete
+        if self.no_delete and not no_delete:
+            raise exceptions.UserError("Nothing can be deleted in this storage.")
+        self.no_delete = bool(no_delete)
+
         if collection and instance_name:
             instance_name = f"{instance_name}/{collection}"
         self.instance_name = instance_name
@@ -105,7 +119,7 @@ class Storage(metaclass=StorageMeta):
         """
         if False:
             yield  # Needs to be an async generator
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @classmethod
     async def create_collection(cls, collection, **kwargs):
@@ -117,7 +131,7 @@ class Storage(metaclass=StorageMeta):
 
         The returned args should contain the collection name, for UI purposes.
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     def __repr__(self):
         try:
@@ -126,19 +140,17 @@ class Storage(metaclass=StorageMeta):
         except ValueError:
             pass
 
-        return "<{}(**{})>".format(
-            self.__class__.__name__,
-            {x: getattr(self, x) for x in self._repr_attributes},
-        )
+        attrs = {x: getattr(self, x) for x in self._repr_attributes}
+        return f"<{self.__class__.__name__}(**{attrs})>"
 
     @abstractmethod
-    async def list(self) -> List[tuple]:
+    async def list(self) -> list[tuple]:
         """
         :returns: list of (href, etag)
         """
 
     @abstractmethod
-    async def get(self, href: str):
+    async def get(self, href: str) -> tuple[Item, str]:
         """Fetch a single item.
 
         :param href: href to fetch
@@ -184,7 +196,7 @@ class Storage(metaclass=StorageMeta):
 
         :returns: (href, etag)
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     async def update(self, href: str, item: Item, etag):
         """Update an item.
@@ -197,7 +209,7 @@ class Storage(metaclass=StorageMeta):
 
         :returns: etag
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     async def delete(self, href: str, etag: str):
         """Delete an item by href.
@@ -205,7 +217,7 @@ class Storage(metaclass=StorageMeta):
         :raises: :exc:`vdirsyncer.exceptions.PreconditionFailed` when item has
             a different etag or doesn't exist.
         """
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @contextlib.asynccontextmanager
     async def at_once(self):
@@ -227,7 +239,7 @@ class Storage(metaclass=StorageMeta):
         """
         yield
 
-    async def get_meta(self, key: str) -> Optional[str]:
+    async def get_meta(self, key: str) -> str | None:
         """Get metadata value for collection/storage.
 
         See the vdir specification for the keys that *have* to be accepted.
@@ -237,7 +249,7 @@ class Storage(metaclass=StorageMeta):
         """
         raise NotImplementedError("This storage does not support metadata.")
 
-    async def set_meta(self, key: str, value: Optional[str]):
+    async def set_meta(self, key: str, value: str | None):
         """Set metadata value for collection/storage.
 
         :param key: The metadata key.
@@ -246,7 +258,7 @@ class Storage(metaclass=StorageMeta):
         raise NotImplementedError("This storage does not support metadata.")
 
 
-def normalize_meta_value(value) -> Optional[str]:
+def normalize_meta_value(value) -> str | None:
     # `None` is returned by iCloud for empty properties.
     if value is None or value == "None":
         return None

vdirsyncer/storage/dav.py

@@ -1,26 +1,28 @@
+from __future__ import annotations
+
+import contextlib
 import datetime
 import logging
 import urllib.parse as urlparse
 import xml.etree.ElementTree as etree
 from abc import abstractmethod
+from functools import cached_property
 from inspect import getfullargspec
 from inspect import signature
-from typing import Optional
-from typing import Type
 
 import aiohttp
 import aiostream
 
+from vdirsyncer import exceptions
+from vdirsyncer import http
+from vdirsyncer import utils
 from vdirsyncer.exceptions import Error
+from vdirsyncer.http import USERAGENT
+from vdirsyncer.http import prepare_auth
+from vdirsyncer.http import prepare_client_cert
+from vdirsyncer.http import prepare_verify
 from vdirsyncer.vobject import Item
 
-from .. import exceptions
-from .. import http
-from .. import utils
-from ..http import USERAGENT
-from ..http import prepare_auth
-from ..http import prepare_client_cert
-from ..http import prepare_verify
 from .base import Storage
 from .base import normalize_meta_value
 
@@ -92,8 +94,7 @@ def _parse_xml(content):
         return etree.XML(_clean_body(content))
     except etree.ParseError as e:
         raise InvalidXMLResponse(
-            "Invalid XML encountered: {}\n"
-            "Double-check the URLs in your config.".format(e)
+            f"Invalid XML encountered: {e}\nDouble-check the URLs in your config."
         )
 
 
@@ -114,10 +115,8 @@ def _fuzzy_matches_mimetype(strict, weak):
     if strict is None or weak is None:
         return True
 
-    mediatype, subtype = strict.split("/")
-    if subtype in weak:
-        return True
-    return False
+    _mediatype, subtype = strict.split("/")
+    return subtype in weak
 
 
 class Discover:
@@ -128,7 +127,7 @@ class Discover:
 
     @property
     @abstractmethod
-    def _resourcetype(self) -> Optional[str]:
+    def _resourcetype(self) -> str | None:
         pass
 
     @property
@@ -173,7 +172,7 @@ class Discover:
             dav_logger.debug("Trying out well-known URI")
             return await self._find_principal_impl(self._well_known_uri)
 
-    async def _find_principal_impl(self, url):
+    async def _find_principal_impl(self, url) -> str:
         headers = self.session.get_default_headers()
         headers["Depth"] = "0"
         body = b"""
@@ -198,11 +197,9 @@ class Discover:
             # E.g. Synology NAS
             # See https://github.com/pimutils/vdirsyncer/issues/498
             dav_logger.debug(
-                "No current-user-principal returned, re-using URL {}".format(
-                    response.url
-                )
+                f"No current-user-principal returned, re-using URL {response.url}"
             )
-            return response.url
+            return response.url.human_repr()
         return urlparse.urljoin(str(response.url), rv.text).rstrip("/") + "/"
 
     async def find_home(self):
@@ -222,10 +219,8 @@ class Discover:
 
     async def find_collections(self):
         rv = None
-        try:
+        with contextlib.suppress(aiohttp.ClientResponseError, exceptions.Error):
             rv = await aiostream.stream.list(self._find_collections_impl(""))
-        except (aiohttp.ClientResponseError, exceptions.Error):
-            pass
 
         if rv:
             return rv
@@ -240,7 +235,7 @@ class Discover:
             return True
 
         props = _merge_xml(response.findall("{DAV:}propstat/{DAV:}prop"))
-        if props is None or not len(props):
+        if props is None or not props:
             dav_logger.debug("Skipping, missing <prop>: %s", response)
             return False
         if props.find("{DAV:}resourcetype/" + self._resourcetype) is None:
@@ -264,7 +259,7 @@ class Discover:
 
             href = response.find("{DAV:}href")
             if href is None:
-                raise InvalidXMLResponse("Missing href tag for collection " "props.")
+                raise InvalidXMLResponse("Missing href tag for collection props.")
             href = urlparse.urljoin(str(r.url), href.text)
             if href not in done:
                 done.add(href)
@@ -313,9 +308,7 @@ class Discover:
             </mkcol>
         """.format(
             etree.tostring(etree.Element(self._resourcetype), encoding="unicode")
-        ).encode(
-            "utf-8"
-        )
+        ).encode("utf-8")
 
         response = await self.session.request(
             "MKCOL",
@@ -328,7 +321,7 @@ class Discover:
 
 class CalDiscover(Discover):
     _namespace = "urn:ietf:params:xml:ns:caldav"
-    _resourcetype = "{%s}calendar" % _namespace
+    _resourcetype = f"{{{_namespace}}}calendar"
     _homeset_xml = b"""
     <propfind xmlns="DAV:" xmlns:c="urn:ietf:params:xml:ns:caldav">
         <prop>
@@ -336,13 +329,13 @@ class CalDiscover(Discover):
         </prop>
     </propfind>
     """
-    _homeset_tag = "{%s}calendar-home-set" % _namespace
+    _homeset_tag = f"{{{_namespace}}}calendar-home-set"
     _well_known_uri = "/.well-known/caldav"
 
 
 class CardDiscover(Discover):
     _namespace = "urn:ietf:params:xml:ns:carddav"
-    _resourcetype: Optional[str] = "{%s}addressbook" % _namespace
+    _resourcetype: str | None = f"{{{_namespace}}}addressbook"
     _homeset_xml = b"""
     <propfind xmlns="DAV:" xmlns:c="urn:ietf:params:xml:ns:carddav">
         <prop>
@@ -350,7 +343,7 @@ class CardDiscover(Discover):
         </prop>
     </propfind>
     """
-    _homeset_tag = "{%s}addressbook-home-set" % _namespace
+    _homeset_tag = f"{{{_namespace}}}addressbook-home-set"
     _well_known_uri = "/.well-known/carddav"
 
 
@@ -398,7 +391,7 @@ class DAVSession:
         self.url = url.rstrip("/") + "/"
         self.connector = connector
 
-    @utils.cached_property
+    @cached_property
     def parsed_url(self):
         return urlparse.urlparse(self.url)
 
@@ -422,6 +415,7 @@ class DAVSession:
         return aiohttp.ClientSession(
             connector=self.connector,
             connector_owner=False,
+            trust_env=True,
             # TODO use `raise_for_status=true`, though this needs traces first,
         )
 
@@ -450,7 +444,7 @@ class DAVStorage(Storage):
 
     @property
     @abstractmethod
-    def discovery_class(self) -> Type[Discover]:
+    def discovery_class(self) -> type[Discover]:
         """Discover subclass to use."""
 
     # The DAVSession class to use
@@ -458,7 +452,7 @@ class DAVStorage(Storage):
 
     connector: aiohttp.TCPConnector
 
-    _repr_attributes = ["username", "url"]
+    _repr_attributes = ("username", "url")
 
     _property_table = {
         "displayname": ("displayname", "DAV:"),
@@ -503,8 +497,12 @@ class DAVStorage(Storage):
     def _is_item_mimetype(self, mimetype):
         return _fuzzy_matches_mimetype(self.item_mimetype, mimetype)
 
-    async def get(self, href: str):
-        ((actual_href, item, etag),) = await aiostream.stream.list(
+    async def get(self, href: str) -> tuple[Item, str]:
+        actual_href: str
+        item: Item
+        etag: str
+
+        ((actual_href, item, etag),) = await aiostream.stream.list(  # type: ignore[misc]
             self.get_multi([href])
         )
         assert href == actual_href
@@ -630,7 +628,7 @@ class DAVStorage(Storage):
                 continue
 
             props = response.findall("{DAV:}propstat/{DAV:}prop")
-            if props is None or not len(props):
+            if props is None or not props:
                 dav_logger.debug(f"Skipping {href!r}, properties are missing.")
                 continue
             else:
@@ -648,9 +646,7 @@ class DAVStorage(Storage):
             contenttype = getattr(props.find("{DAV:}getcontenttype"), "text", None)
             if not self._is_item_mimetype(contenttype):
                 dav_logger.debug(
-                    "Skipping {!r}, {!r} != {!r}.".format(
-                        href, contenttype, self.item_mimetype
-                    )
+                    f"Skipping {href!r}, {contenttype!r} != {self.item_mimetype!r}."
                 )
                 continue
 
@@ -685,11 +681,11 @@ class DAVStorage(Storage):
         for href, etag, _prop in rv:
             yield href, etag
 
-    async def get_meta(self, key) -> Optional[str]:
+    async def get_meta(self, key) -> str | None:
         try:
             tagname, namespace = self._property_table[key]
         except KeyError:
-            raise exceptions.UnsupportedMetadataError()
+            raise exceptions.UnsupportedMetadataError
 
         xpath = f"{{{namespace}}}{tagname}"
         body = f"""<?xml version="1.0" encoding="utf-8" ?>
@@ -723,7 +719,7 @@ class DAVStorage(Storage):
         try:
             tagname, namespace = self._property_table[key]
         except KeyError:
-            raise exceptions.UnsupportedMetadataError()
+            raise exceptions.UnsupportedMetadataError
 
         lxml_selector = f"{{{namespace}}}{tagname}"
         element = etree.Element(lxml_selector)
@@ -744,9 +740,7 @@ class DAVStorage(Storage):
         """.format(
             etree.tostring(element, encoding="unicode"),
             action=action,
-        ).encode(
-            "utf-8"
-        )
+        ).encode("utf-8")
 
         await self.session.request(
             "PROPPATCH",
@@ -800,7 +794,7 @@ class CalDAVStorage(DAVStorage):
         self.item_types = tuple(item_types)
         if (start_date is None) != (end_date is None):
             raise exceptions.UserError(
-                "If start_date is given, " "end_date has to be given too."
+                "If start_date is given, end_date has to be given too."
             )
         elif start_date is not None and end_date is not None:
             namespace = dict(datetime.__dict__)
@@ -830,9 +824,7 @@ class CalDAVStorage(DAVStorage):
                 start = start.strftime(CALDAV_DT_FORMAT)
                 end = end.strftime(CALDAV_DT_FORMAT)
 
-                timefilter = '<C:time-range start="{start}" end="{end}"/>'.format(
-                    start=start, end=end
-                )
+                timefilter = f'<C:time-range start="{start}" end="{end}"/>'
             else:
                 timefilter = ""
 
@@ -900,14 +892,21 @@ class CardDAVStorage(DAVStorage):
     item_mimetype = "text/vcard"
     discovery_class = CardDiscover
 
-    get_multi_template = """<?xml version="1.0" encoding="utf-8" ?>
+    def __init__(self, *args, use_vcard_4=False, **kwargs):
+        self.use_vcard_4 = use_vcard_4
+        super().__init__(*args, **kwargs)
+
+    @property
+    def get_multi_template(self):
+        ct = 'Content-Type="text/vcard" version="4.0"' if self.use_vcard_4 else ""
+        return f"""<?xml version="1.0" encoding="utf-8" ?>
             <C:addressbook-multiget xmlns="DAV:"
                     xmlns:C="urn:ietf:params:xml:ns:carddav">
                 <prop>
                     <getetag/>
-                    <C:address-data/>
+                    <C:address-data {ct}/>
                 </prop>
-                {hrefs}
+                {{hrefs}}
             </C:addressbook-multiget>"""
 
     get_multi_data_query = "{urn:ietf:params:xml:ns:carddav}address-data"

vdirsyncer/storage/filesystem.py

@@ -1,16 +1,19 @@
+from __future__ import annotations
+
+import contextlib
 import errno
 import logging
 import os
 import subprocess
 
-from atomicwrites import atomic_write
+from vdirsyncer import exceptions
+from vdirsyncer.utils import atomic_write
+from vdirsyncer.utils import checkdir
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import generate_href
+from vdirsyncer.utils import get_etag_from_file
+from vdirsyncer.vobject import Item
 
-from .. import exceptions
-from ..utils import checkdir
-from ..utils import expand_path
-from ..utils import generate_href
-from ..utils import get_etag_from_file
-from ..vobject import Item
 from .base import Storage
 from .base import normalize_meta_value
 
@@ -18,9 +21,8 @@ logger = logging.getLogger(__name__)
 
 
 class FilesystemStorage(Storage):
-
     storage_name = "filesystem"
-    _repr_attributes = ["path"]
+    _repr_attributes = ("path",)
 
     def __init__(
         self,
@@ -28,6 +30,7 @@ class FilesystemStorage(Storage):
         fileext,
         encoding="utf-8",
         post_hook=None,
+        pre_deletion_hook=None,
         fileignoreext=".tmp",
         **kwargs,
     ):
@@ -39,6 +42,7 @@ class FilesystemStorage(Storage):
         self.fileext = fileext
         self.fileignoreext = fileignoreext
         self.post_hook = post_hook
+        self.pre_deletion_hook = pre_deletion_hook
 
     @classmethod
     async def discover(cls, path, **kwargs):
@@ -62,9 +66,7 @@ class FilesystemStorage(Storage):
     def _validate_collection(cls, path):
         if not os.path.isdir(path):
             return False
-        if os.path.basename(path).startswith("."):
-            return False
-        return True
+        return not os.path.basename(path).startswith(".")
 
     @classmethod
     async def create_collection(cls, collection, **kwargs):
@@ -96,7 +98,7 @@ class FilesystemStorage(Storage):
             ):
                 yield fname, get_etag_from_file(fpath)
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         fpath = self._get_filepath(href)
         try:
             with open(fpath, "rb") as f:
@@ -165,6 +167,9 @@ class FilesystemStorage(Storage):
         actual_etag = get_etag_from_file(fpath)
         if etag != actual_etag:
             raise exceptions.WrongEtagError(etag, actual_etag)
+        if self.pre_deletion_hook:
+            self._run_pre_deletion_hook(fpath)
+
         os.remove(fpath)
 
     def _run_post_hook(self, fpath):
@@ -172,7 +177,16 @@ class FilesystemStorage(Storage):
         try:
             subprocess.call([self.post_hook, fpath])
         except OSError as e:
-            logger.warning(f"Error executing external hook: {str(e)}")
+            logger.warning(f"Error executing external hook: {e!s}")
+
+    def _run_pre_deletion_hook(self, fpath):
+        logger.info(
+            f"Calling pre_deletion_hook={self.pre_deletion_hook} with argument={fpath}"
+        )
+        try:
+            subprocess.call([self.pre_deletion_hook, fpath])
+        except OSError as e:
+            logger.warning(f"Error executing external hook: {e!s}")
 
     async def get_meta(self, key):
         fpath = os.path.join(self.path, key)
@@ -190,10 +204,8 @@ class FilesystemStorage(Storage):
 
         fpath = os.path.join(self.path, key)
         if value is None:
-            try:
+            with contextlib.suppress(OSError):
                 os.remove(fpath)
-            except OSError:
-                pass
         else:
             with atomic_write(fpath, mode="wb", overwrite=True) as f:
                 f.write(value.encode(self.encoding))

vdirsyncer/storage/google.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import json
 import logging
 import os
@@ -9,12 +11,13 @@ from threading import Thread
 
 import aiohttp
 import click
-from atomicwrites import atomic_write
 
-from .. import exceptions
-from ..utils import checkdir
-from ..utils import expand_path
-from ..utils import open_graphical_browser
+from vdirsyncer import exceptions
+from vdirsyncer.utils import atomic_write
+from vdirsyncer.utils import checkdir
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import open_graphical_browser
+
 from . import base
 from . import dav
 from .google_helpers import _RedirectWSGIApp
@@ -96,6 +99,7 @@ class GoogleSession(dav.DAVSession):
             token_updater=self._save_token,
             connector=self.connector,
             connector_owner=False,
+            trust_env=True,
         )
 
     async def _init_token(self):
@@ -106,8 +110,8 @@ class GoogleSession(dav.DAVSession):
             pass
         except ValueError as e:
             raise exceptions.UserError(
-                "Failed to load token file {}, try deleting it. "
-                "Original error: {}".format(self._token_file, e)
+                f"Failed to load token file {self._token_file}, try deleting it. "
+                f"Original error: {e}"
             )
 
         if not self._token:
@@ -126,7 +130,7 @@ class GoogleSession(dav.DAVSession):
             async with self._session as session:
                 # Fail fast if the address is occupied
 
-                authorization_url, state = session.authorization_url(
+                authorization_url, _state = session.authorization_url(
                     TOKEN_URL,
                     # access_type and approval_prompt are Google specific
                     # extra parameters.
@@ -190,7 +194,7 @@ class GoogleCalendarStorage(dav.CalDAVStorage):
         **kwargs,
     ):
         if not kwargs.get("collection"):
-            raise exceptions.CollectionRequired()
+            raise exceptions.CollectionRequired
 
         super().__init__(
             token_file=token_file,
@@ -228,7 +232,7 @@ class GoogleContactsStorage(dav.CardDAVStorage):
 
     def __init__(self, token_file, client_id, client_secret, **kwargs):
         if not kwargs.get("collection"):
-            raise exceptions.CollectionRequired()
+            raise exceptions.CollectionRequired
 
         super().__init__(
             token_file=token_file,

vdirsyncer/storage/google_helpers.py

@@ -2,15 +2,14 @@
 #
 # Based on:
 # https://github.com/googleapis/google-auth-library-python-oauthlib/blob/1fb16be1bad9050ee29293541be44e41e82defd7/google_auth_oauthlib/flow.py#L513
+from __future__ import annotations
 
 import logging
 import wsgiref.simple_server
 import wsgiref.util
+from collections.abc import Iterable
 from typing import Any
 from typing import Callable
-from typing import Dict
-from typing import Iterable
-from typing import Optional
 
 logger = logging.getLogger(__name__)
 
@@ -29,7 +28,7 @@ class _RedirectWSGIApp:
     Stores the request URI and displays the given success message.
     """
 
-    last_request_uri: Optional[str]
+    last_request_uri: str | None
 
     def __init__(self, success_message: str):
         """
@@ -41,7 +40,7 @@ class _RedirectWSGIApp:
 
     def __call__(
         self,
-        environ: Dict[str, Any],
+        environ: dict[str, Any],
         start_response: Callable[[str, list], None],
     ) -> Iterable[bytes]:
         """WSGI Callable.

vdirsyncer/storage/http.py

@@ -1,22 +1,29 @@
+from __future__ import annotations
+
+import logging
+import subprocess
 import urllib.parse as urlparse
 
 import aiohttp
 
-from .. import exceptions
-from ..http import USERAGENT
-from ..http import prepare_auth
-from ..http import prepare_client_cert
-from ..http import prepare_verify
-from ..http import request
-from ..vobject import Item
-from ..vobject import split_collection
+from vdirsyncer import exceptions
+from vdirsyncer.http import USERAGENT
+from vdirsyncer.http import prepare_auth
+from vdirsyncer.http import prepare_client_cert
+from vdirsyncer.http import prepare_verify
+from vdirsyncer.http import request
+from vdirsyncer.vobject import Item
+from vdirsyncer.vobject import split_collection
+
 from .base import Storage
 
+logger = logging.getLogger(__name__)
+
 
 class HttpStorage(Storage):
     storage_name = "http"
     read_only = True
-    _repr_attributes = ["username", "url"]
+    _repr_attributes = ("username", "url")
     _items = None
 
     # Required for tests.
@@ -32,9 +39,10 @@ class HttpStorage(Storage):
         useragent=USERAGENT,
         verify_fingerprint=None,
         auth_cert=None,
+        filter_hook=None,
         *,
         connector,
-        **kwargs
+        **kwargs,
     ) -> None:
         super().__init__(**kwargs)
 
@@ -54,6 +62,7 @@ class HttpStorage(Storage):
         self.useragent = useragent
         assert connector is not None
         self.connector = connector
+        self._filter_hook = filter_hook
 
         collection = kwargs.get("collection")
         if collection is not None:
@@ -64,10 +73,24 @@ class HttpStorage(Storage):
     def _default_headers(self):
         return {"User-Agent": self.useragent}
 
+    def _run_filter_hook(self, raw_item):
+        try:
+            result = subprocess.run(
+                [self._filter_hook],
+                input=raw_item,
+                capture_output=True,
+                encoding="utf-8",
+            )
+            return result.stdout
+        except OSError as e:
+            logger.warning(f"Error executing external command: {e!s}")
+            return raw_item
+
     async def list(self):
         async with aiohttp.ClientSession(
             connector=self.connector,
             connector_owner=False,
+            trust_env=True,
             # TODO use `raise_for_status=true`, though this needs traces first,
         ) as session:
             r = await request(
@@ -79,8 +102,13 @@ class HttpStorage(Storage):
             )
         self._items = {}
 
-        for item in split_collection((await r.read()).decode("utf-8")):
-            item = Item(item)
+        for raw_item in split_collection((await r.read()).decode("utf-8")):
+            if self._filter_hook:
+                raw_item = self._run_filter_hook(raw_item)
+            if not raw_item:
+                continue
+
+            item = Item(raw_item)
             if self._ignore_uids:
                 item = item.with_uid(item.hash)
 
@@ -89,11 +117,12 @@ class HttpStorage(Storage):
         for href, (_, etag) in self._items.items():
             yield href, etag
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         if self._items is None:
             async for _ in self.list():
                 pass
 
+        assert self._items is not None  # type assertion
         try:
             return self._items[href]
         except KeyError:

vdirsyncer/storage/memory.py

@@ -1,6 +1,10 @@
+from __future__ import annotations
+
 import random
 
-from .. import exceptions
+from vdirsyncer import exceptions
+from vdirsyncer.vobject import Item
+
 from .base import Storage
 from .base import normalize_meta_value
 
@@ -10,7 +14,6 @@ def _random_string():
 
 
 class MemoryStorage(Storage):
-
     storage_name = "memory"
 
     """
@@ -19,7 +22,7 @@ class MemoryStorage(Storage):
 
     def __init__(self, fileext="", **kwargs):
         if kwargs.get("collection") is not None:
-            raise exceptions.UserError("MemoryStorage does not support " "collections.")
+            raise exceptions.UserError("MemoryStorage does not support collections.")
         self.items = {}  # href => (etag, item)
         self.metadata = {}
         self.fileext = fileext
@@ -32,7 +35,7 @@ class MemoryStorage(Storage):
         for href, (etag, _item) in self.items.items():
             yield href, etag
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         etag, item = self.items[href]
         return item, etag
 

vdirsyncer/storage/singlefile.py

@@ -1,25 +1,36 @@
+from __future__ import annotations
+
 import collections
 import contextlib
 import functools
 import glob
 import logging
 import os
+from collections.abc import Iterable
 
-from atomicwrites import atomic_write
+from vdirsyncer import exceptions
+from vdirsyncer.utils import atomic_write
+from vdirsyncer.utils import checkfile
+from vdirsyncer.utils import expand_path
+from vdirsyncer.utils import get_etag_from_file
+from vdirsyncer.utils import uniq
+from vdirsyncer.vobject import Item
+from vdirsyncer.vobject import join_collection
+from vdirsyncer.vobject import split_collection
 
-from .. import exceptions
-from ..utils import checkfile
-from ..utils import expand_path
-from ..utils import get_etag_from_file
-from ..vobject import Item
-from ..vobject import join_collection
-from ..vobject import split_collection
 from .base import Storage
 
 logger = logging.getLogger(__name__)
 
 
 def _writing_op(f):
+    """Implement at_once for write operations.
+
+    Wrap an operation which writes to the storage, implementing `at_once` if it has been
+    requested. Changes are stored in-memory until the at_once block finishes, at which
+    time they are all written at once.
+    """
+
     @functools.wraps(f)
     async def inner(self, *args, **kwargs):
         if self._items is None or not self._at_once:
@@ -36,7 +47,7 @@ def _writing_op(f):
 
 class SingleFileStorage(Storage):
     storage_name = "singlefile"
-    _repr_attributes = ["path"]
+    _repr_attributes = ("path",)
 
     _write_mode = "wb"
     _append_mode = "ab"
@@ -65,7 +76,7 @@ class SingleFileStorage(Storage):
         except TypeError:
             # If not exactly one '%s' is present, we cannot discover
             # collections because we wouldn't know which name to assign.
-            raise NotImplementedError()
+            raise NotImplementedError
 
         placeholder_pos = path.index("%s")
 
@@ -91,7 +102,7 @@ class SingleFileStorage(Storage):
                 path = path % (collection,)
             except TypeError:
                 raise ValueError(
-                    "Exactly one %s required in path " "if collection is not null."
+                    "Exactly one %s required in path if collection is not null."
                 )
 
         checkfile(path, create=True)
@@ -122,16 +133,23 @@ class SingleFileStorage(Storage):
 
                 yield href, etag
 
-    async def get(self, href):
+    async def get(self, href) -> tuple[Item, str]:
         if self._items is None or not self._at_once:
             async for _ in self.list():
                 pass
 
+        assert self._items is not None  # type assertion
         try:
             return self._items[href]
         except KeyError:
             raise exceptions.NotFoundError(href)
 
+    async def get_multi(self, hrefs: Iterable[str]):
+        async with self.at_once():
+            for href in uniq(hrefs):
+                item, etag = await self.get(href)
+                yield href, item, etag
+
     @_writing_op
     async def upload(self, item):
         href = item.ident
@@ -169,11 +187,9 @@ class SingleFileStorage(Storage):
             self.path
         ):
             raise exceptions.PreconditionFailed(
-                (
-                    "Some other program modified the file {!r}. Re-run the "
-                    "synchronization and make sure absolutely no other program is "
-                    "writing into the same file."
-                ).format(self.path)
+                f"Some other program modified the file {self.path!r}. Re-run the "
+                "synchronization and make sure absolutely no other program is "
+                "writing into the same file."
             )
         text = join_collection(item.raw for item, etag in self._items.values())
         try:
@@ -185,7 +201,8 @@ class SingleFileStorage(Storage):
 
     @contextlib.asynccontextmanager
     async def at_once(self):
-        self.list()
+        async for _ in self.list():
+            pass
         self._at_once = True
         try:
             yield self

vdirsyncer/sync/__init__.py

@@ -9,18 +9,25 @@ Yang: http://blog.ezyang.com/2012/08/how-offlineimap-works/
 Some modifications to it are explained in
 https://unterwaditzer.net/2016/sync-algorithm.html
 """
+
+from __future__ import annotations
+
 import contextlib
 import itertools
 import logging
 
-from ..exceptions import UserError
-from ..utils import uniq
+from vdirsyncer.exceptions import UserError
+from vdirsyncer.storage.base import Storage
+from vdirsyncer.utils import uniq
+from vdirsyncer.vobject import Item
+
 from .exceptions import BothReadOnly
 from .exceptions import IdentAlreadyExists
 from .exceptions import PartialSync
 from .exceptions import StorageEmpty
 from .exceptions import SyncConflict
 from .status import ItemMetadata
+from .status import SqliteStatus
 from .status import SubStatus
 
 sync_logger = logging.getLogger(__name__)
@@ -30,22 +37,22 @@ class _StorageInfo:
     """A wrapper class that holds prefetched items, the status and other
     things."""
 
-    def __init__(self, storage, status):
+    def __init__(self, storage: Storage, status: SubStatus):
         self.storage = storage
         self.status = status
-        self._item_cache = {}
+        self._item_cache = {}  # type: ignore[var-annotated]
 
-    async def prepare_new_status(self):
+    async def prepare_new_status(self) -> bool:
         storage_nonempty = False
         prefetch = []
 
-        def _store_props(ident, props):
+        def _store_props(ident: str, props: ItemMetadata) -> None:
             try:
                 self.status.insert_ident(ident, props)
             except IdentAlreadyExists as e:
                 raise e.to_ident_conflict(self.storage)
 
-        async for href, etag in self.storage.list():
+        async for href, etag in self.storage.list():  # type: ignore[attr-defined]
             storage_nonempty = True
             ident, meta = self.status.get_by_href(href)
 
@@ -68,7 +75,7 @@ class _StorageInfo:
 
         return storage_nonempty
 
-    def is_changed(self, ident):
+    def is_changed(self, ident: str) -> bool:
         old_meta = self.status.get(ident)
         if old_meta is None:  # new item
             return True
@@ -81,30 +88,28 @@ class _StorageInfo:
             and (old_meta.hash is None or new_meta.hash != old_meta.hash)
         )
 
-    def set_item_cache(self, ident, item):
+    def set_item_cache(self, ident, item) -> None:
         actual_hash = self.status.get_new(ident).hash
         assert actual_hash == item.hash
         self._item_cache[ident] = item
 
-    def get_item_cache(self, ident):
+    def get_item_cache(self, ident: str) -> Item:
         return self._item_cache[ident]
 
 
 async def sync(
-    storage_a,
-    storage_b,
-    status,
+    storage_a: Storage,
+    storage_b: Storage,
+    status: SqliteStatus,
     conflict_resolution=None,
     force_delete=False,
     error_callback=None,
     partial_sync="revert",
-):
+) -> None:
     """Synchronizes two storages.
 
     :param storage_a: The first storage
-    :type storage_a: :class:`vdirsyncer.storage.base.Storage`
     :param storage_b: The second storage
-    :type storage_b: :class:`vdirsyncer.storage.base.Storage`
     :param status: {ident: (href_a, etag_a, href_b, etag_b)}
         metadata about the two storages for detection of changes. Will be
         modified by the function and should be passed to it at the next sync.
@@ -128,12 +133,16 @@ async def sync(
         - ``revert`` (default): Revert changes on other side.
     """
     if storage_a.read_only and storage_b.read_only:
-        raise BothReadOnly()
+        raise BothReadOnly
 
     if conflict_resolution == "a wins":
-        conflict_resolution = lambda a, b: a  # noqa: E731
+
+        def conflict_resolution(a, b):
+            return a
     elif conflict_resolution == "b wins":
-        conflict_resolution = lambda a, b: b  # noqa: E731
+
+        def conflict_resolution(a, b):
+            return b
 
     status_nonempty = bool(next(status.iter_old(), None))
 
@@ -165,7 +174,7 @@ async def sync(
 
 class Action:
     async def _run_impl(self, a, b):  # pragma: no cover
-        raise NotImplementedError()
+        raise NotImplementedError
 
     async def run(self, a, b, conflict_resolution, partial_sync):
         with self.auto_rollback(a, b):
@@ -199,14 +208,11 @@ class Upload(Action):
         self.dest = dest
 
     async def _run_impl(self, a, b):
-
         if self.dest.storage.read_only:
             href = etag = None
         else:
             sync_logger.info(
-                "Copying (uploading) item {} to {}".format(
-                    self.ident, self.dest.storage
-                )
+                f"Copying (uploading) item {self.ident} to {self.dest.storage}"
             )
             href, etag = await self.dest.storage.upload(self.item)
             assert href is not None
@@ -242,7 +248,11 @@ class Delete(Action):
 
     async def _run_impl(self, a, b):
         meta = self.dest.status.get_new(self.ident)
-        if not self.dest.storage.read_only:
+        if self.dest.storage.read_only or self.dest.storage.no_delete:
+            sync_logger.debug(
+                f"Skipping deletion of item {self.ident} from {self.dest.storage}"
+            )
+        else:
             sync_logger.info(f"Deleting item {self.ident} from {self.dest.storage}")
             await self.dest.storage.delete(meta.href, meta.etag)
 
@@ -290,7 +300,7 @@ class ResolveConflict(Action):
                 )
 
 
-def _get_actions(a_info, b_info):
+def _get_actions(a_info: _StorageInfo, b_info: _StorageInfo):
     for ident in uniq(
         itertools.chain(
             a_info.status.parent.iter_new(), a_info.status.parent.iter_old()

vdirsyncer/sync/exceptions.py

@@ -1,4 +1,6 @@
-from .. import exceptions
+from __future__ import annotations
+
+from vdirsyncer import exceptions
 
 
 class SyncError(exceptions.Error):

vdirsyncer/sync/status.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import abc
 import contextlib
 import sqlite3
@@ -47,63 +49,63 @@ class _StatusBase(metaclass=abc.ABCMeta):
 
     @abc.abstractmethod
     def transaction(self):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def insert_ident_a(self, ident, props):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def insert_ident_b(self, ident, props):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def update_ident_a(self, ident, props):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def update_ident_b(self, ident, props):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def remove_ident(self, ident):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def get_a(self, ident):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def get_b(self, ident):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def get_new_a(self, ident):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def get_new_b(self, ident):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def iter_old(self):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def iter_new(self):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def get_by_href_a(self, href, default=(None, None)):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def get_by_href_b(self, href, default=(None, None)):
-        raise NotImplementedError()
+        raise NotImplementedError
 
     @abc.abstractmethod
     def rollback(self, ident):
-        raise NotImplementedError()
+        raise NotImplementedError
 
 
 class SqliteStatus(_StatusBase):
@@ -167,6 +169,11 @@ class SqliteStatus(_StatusBase):
             ); """
             )
 
+    def close(self):
+        if self._c:
+            self._c.close()
+            self._c = None
+
     def _is_latest_version(self):
         try:
             return bool(
@@ -185,7 +192,7 @@ class SqliteStatus(_StatusBase):
                 self._c = new_c
                 yield
                 self._c.execute("DELETE FROM status")
-                self._c.execute("INSERT INTO status " "SELECT * FROM new_status")
+                self._c.execute("INSERT INTO status SELECT * FROM new_status")
                 self._c.execute("DELETE FROM new_status")
         finally:
             self._c = old_c
@@ -197,7 +204,7 @@ class SqliteStatus(_StatusBase):
             raise IdentAlreadyExists(old_href=old_props.href, new_href=a_props.href)
         b_props = self.get_new_b(ident) or ItemMetadata()
         self._c.execute(
-            "INSERT OR REPLACE INTO new_status " "VALUES(?, ?, ?, ?, ?, ?, ?)",
+            "INSERT OR REPLACE INTO new_status VALUES(?, ?, ?, ?, ?, ?, ?)",
             (
                 ident,
                 a_props.href,
@@ -216,7 +223,7 @@ class SqliteStatus(_StatusBase):
             raise IdentAlreadyExists(old_href=old_props.href, new_href=b_props.href)
         a_props = self.get_new_a(ident) or ItemMetadata()
         self._c.execute(
-            "INSERT OR REPLACE INTO new_status " "VALUES(?, ?, ?, ?, ?, ?, ?)",
+            "INSERT OR REPLACE INTO new_status VALUES(?, ?, ?, ?, ?, ?, ?)",
             (
                 ident,
                 a_props.href,
@@ -230,14 +237,14 @@ class SqliteStatus(_StatusBase):
 
     def update_ident_a(self, ident, props):
         self._c.execute(
-            "UPDATE new_status" " SET href_a=?, hash_a=?, etag_a=?" " WHERE ident=?",
+            "UPDATE new_status SET href_a=?, hash_a=?, etag_a=? WHERE ident=?",
             (props.href, props.hash, props.etag, ident),
         )
         assert self._c.rowcount > 0
 
     def update_ident_b(self, ident, props):
         self._c.execute(
-            "UPDATE new_status" " SET href_b=?, hash_b=?, etag_b=?" " WHERE ident=?",
+            "UPDATE new_status SET href_b=?, hash_b=?, etag_b=? WHERE ident=?",
             (props.href, props.hash, props.etag, ident),
         )
         assert self._c.rowcount > 0
@@ -247,10 +254,10 @@ class SqliteStatus(_StatusBase):
 
     def _get_impl(self, ident, side, table):
         res = self._c.execute(
-            "SELECT href_{side} AS href,"
-            "       hash_{side} AS hash,"
-            "       etag_{side} AS etag "
-            "FROM {table} WHERE ident=?".format(side=side, table=table),
+            f"SELECT href_{side} AS href,"
+            f"       hash_{side} AS hash,"
+            f"       etag_{side} AS etag "
+            f"FROM {table} WHERE ident=?",
             (ident,),
         ).fetchone()
         if res is None:
@@ -298,14 +305,14 @@ class SqliteStatus(_StatusBase):
             return
 
         self._c.execute(
-            "INSERT OR REPLACE INTO new_status" " VALUES (?, ?, ?, ?, ?, ?, ?)",
+            "INSERT OR REPLACE INTO new_status VALUES (?, ?, ?, ?, ?, ?, ?)",
             (ident, a.href, b.href, a.hash, b.hash, a.etag, b.etag),
         )
 
     def _get_by_href_impl(self, href, default=(None, None), side=None):
         res = self._c.execute(
-            "SELECT ident, hash_{side} AS hash, etag_{side} AS etag "
-            "FROM status WHERE href_{side}=?".format(side=side),
+            f"SELECT ident, hash_{side} AS hash, etag_{side} AS etag "
+            f"FROM status WHERE href_{side}=?",
             (href,),
         ).fetchone()
         if not res:
@@ -326,7 +333,7 @@ class SqliteStatus(_StatusBase):
 
 
 class SubStatus:
-    def __init__(self, parent, side):
+    def __init__(self, parent: SqliteStatus, side: str):
         self.parent = parent
         assert side in "ab"
 

vdirsyncer/utils.py

@@ -1,6 +1,10 @@
+from __future__ import annotations
+
+import contextlib
 import functools
 import os
 import sys
+import tempfile
 import uuid
 from inspect import getfullargspec
 from typing import Callable
@@ -20,8 +24,7 @@ _missing = object()
 def expand_path(p: str) -> str:
     """Expand $HOME in a path and normalise slashes."""
     p = os.path.expanduser(p)
-    p = os.path.normpath(p)
-    return p
+    return os.path.normpath(p)
 
 
 def split_dict(d: dict, f: Callable):
@@ -74,7 +77,7 @@ def get_storage_init_specs(cls, stop_at=object):
     spec = getfullargspec(cls.__init__)
     traverse_superclass = getattr(cls.__init__, "_traverse_superclass", True)
     if traverse_superclass:
-        if traverse_superclass is True:  # noqa
+        if traverse_superclass is True:
             supercls = next(
                 getattr(x.__init__, "__objclass__", x) for x in cls.__mro__[1:]
             )
@@ -84,7 +87,7 @@ def get_storage_init_specs(cls, stop_at=object):
     else:
         superspecs = ()
 
-    return (spec,) + superspecs
+    return (spec, *superspecs)
 
 
 def get_storage_init_args(cls, stop_at=object):
@@ -123,12 +126,13 @@ def checkdir(path: str, create: bool = False, mode: int = 0o750) -> None:
             raise exceptions.CollectionNotFound(f"Directory {path} does not exist.")
 
 
-def checkfile(path, create=False):
-    """
-    Check whether ``path`` is a file.
+def checkfile(path, create=False) -> None:
+    """Check whether ``path`` is a file.
 
     :param create: Whether to create the file's parent directories if they do
         not exist.
+    :raises CollectionNotFound: if path does not exist.
+    :raises OSError: if path exists but is not a file.
     """
     checkdir(os.path.dirname(path), create=create)
     if not os.path.isfile(path):
@@ -141,24 +145,6 @@ def checkfile(path, create=False):
             raise exceptions.CollectionNotFound(f"File {path} does not exist.")
 
 
-class cached_property:
-    """A read-only @property that is only evaluated once. Only usable on class
-    instances' methods.
-    """
-
-    def __init__(self, fget, doc=None):
-        self.__name__ = fget.__name__
-        self.__module__ = fget.__module__
-        self.__doc__ = doc or fget.__doc__
-        self.fget = fget
-
-    def __get__(self, obj, cls):
-        if obj is None:  # pragma: no cover
-            return self
-        obj.__dict__[self.__name__] = result = self.fget(obj)
-        return result
-
-
 def href_safe(ident, safe=SAFE_UID_CHARS):
     return not bool(set(ident) - set(safe))
 
@@ -172,8 +158,7 @@ def generate_href(ident=None, safe=SAFE_UID_CHARS):
     """
     if not ident or not href_safe(ident, safe):
         return str(uuid.uuid4())
-    else:
-        return ident
+    return ident
 
 
 def synchronized(lock=None):
@@ -206,7 +191,7 @@ def open_graphical_browser(url, new=0, autoraise=True):
 
     cli_names = {"www-browser", "links", "links2", "elinks", "lynx", "w3m"}
 
-    if webbrowser._tryorder is None:  # Python 3.7
+    if webbrowser._tryorder is None:  # Python 3.8
         webbrowser.register_standard_browsers()
 
     for name in webbrowser._tryorder:
@@ -217,4 +202,28 @@ def open_graphical_browser(url, new=0, autoraise=True):
         if browser.open(url, new, autoraise):
             return
 
-    raise RuntimeError("No graphical browser found. Please open the URL " "manually.")
+    raise RuntimeError("No graphical browser found. Please open the URL manually.")
+
+
+@contextlib.contextmanager
+def atomic_write(dest, mode="wb", overwrite=False):
+    if "w" not in mode:
+        raise RuntimeError("`atomic_write` requires write access")
+
+    fd, src = tempfile.mkstemp(prefix=os.path.basename(dest), dir=os.path.dirname(dest))
+    file = os.fdopen(fd, mode=mode)
+
+    try:
+        yield file
+    except Exception:
+        os.unlink(src)
+        raise
+    else:
+        file.flush()
+        file.close()
+
+        if overwrite:
+            os.rename(src, dest)
+        else:
+            os.link(src, dest)
+            os.unlink(src)

vdirsyncer/vobject.py

@@ -1,8 +1,10 @@
+from __future__ import annotations
+
 import hashlib
+from functools import cached_property
 from itertools import chain
 from itertools import tee
 
-from .utils import cached_property
 from .utils import uniq
 
 IGNORE_PROPS = (
@@ -34,7 +36,6 @@ IGNORE_PROPS = (
 
 
 class Item:
-
     """Immutable wrapper class for VCALENDAR (VEVENT, VTODO) and
     VCARD"""
 
@@ -187,7 +188,7 @@ def join_collection(items, wrappers=_default_join_wrappers):
     """
 
     items1, items2 = tee((_Component.parse(x) for x in items), 2)
-    item_type, wrapper_type = _get_item_type(items1, wrappers)
+    _item_type, wrapper_type = _get_item_type(items1, wrappers)
     wrapper_props = []
 
     def _get_item_components(x):
@@ -230,8 +231,7 @@ def _get_item_type(components, wrappers):
 
     if not i:
         return None, None
-    else:
-        raise ValueError("Not sure how to join components.")
+    raise ValueError("Not sure how to join components.")
 
 
 class _Component:
@@ -279,6 +279,12 @@ class _Component:
                     stack.append(cls(c_name, [], []))
                 elif line.startswith("END:"):
                     component = stack.pop()
+                    c_name = line[len("END:") :].strip().upper()
+                    if c_name != component.name:
+                        raise ValueError(
+                            f"Got END:{c_name}, expected END:{component.name}"
+                            + f" at line {_i + 1}"
+                        )
                     if stack:
                         stack[-1].subcomponents.append(component)
                     else:
@@ -289,12 +295,16 @@ class _Component:
         except IndexError:
             raise ValueError(f"Parsing error at line {_i + 1}")
 
+        if len(stack) > 0:
+            raise ValueError(
+                f"Missing END for component(s): {', '.join(c.name for c in stack)}"
+            )
+
         if multiple:
             return rv
-        elif len(rv) != 1:
+        if len(rv) != 1:
             raise ValueError(f"Found {len(rv)} components, expected one.")
-        else:
-            return rv[0]
+        return rv[0]
 
     def dump_lines(self):
         yield f"BEGIN:{self.name}"
@@ -311,13 +321,12 @@ class _Component:
             for line in lineiter:
                 if line.startswith(prefix):
                     break
-                else:
-                    new_lines.append(line)
+                new_lines.append(line)
             else:
                 break
 
             for line in lineiter:
-                if not line.startswith((" ", "\t")):
+                if not line.startswith((" ", "\t", *prefix)):
                     new_lines.append(line)
                     break
 
@@ -335,10 +344,9 @@ class _Component:
             return obj not in self.subcomponents and not any(
                 obj in x for x in self.subcomponents
             )
-        elif isinstance(obj, str):
+        if isinstance(obj, str):
             return self.get(obj, None) is not None
-        else:
-            raise ValueError(obj)
+        raise ValueError(obj)
 
     def __getitem__(self, key):
         prefix_without_params = f"{key}:"
@@ -348,11 +356,11 @@ class _Component:
             if line.startswith(prefix_without_params):
                 rv = line[len(prefix_without_params) :]
                 break
-            elif line.startswith(prefix_with_params):
+            if line.startswith(prefix_with_params):
                 rv = line[len(prefix_with_params) :].split(":", 1)[-1]
                 break
         else:
-            raise KeyError()
+            raise KeyError
 
         for line in iterlines:
             if line.startswith((" ", "\t")):