From 85ae33955fd766f690b07a65d31d6aca1017c6a4 Mon Sep 17 00:00:00 2001 From: Enrico Guiraud Date: Thu, 9 Mar 2023 17:59:20 -0600 Subject: [PATCH] Prevent single file storage from performing unnecessary N^2 loop For single file storage we wrap the logic of get_multi with the at_once context manager so that `self.list()` (which is called by `self.get()`) actually caches the items rather than re-computing them at every call. This should largely mitigate the performance issue describe at https://github.com/pimutils/vdirsyncer/issues/818 . The issue discussion also contains more background about this patch. --- vdirsyncer/storage/singlefile.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vdirsyncer/storage/singlefile.py b/vdirsyncer/storage/singlefile.py index 2297c02..04f41e9 100644 --- a/vdirsyncer/storage/singlefile.py +++ b/vdirsyncer/storage/singlefile.py @@ -4,6 +4,7 @@ import functools import glob import logging import os +from typing import Iterable from atomicwrites import atomic_write @@ -11,6 +12,7 @@ from .. import exceptions from ..utils import checkfile from ..utils import expand_path from ..utils import get_etag_from_file +from ..utils import uniq from ..vobject import Item from ..vobject import join_collection from ..vobject import split_collection @@ -132,6 +134,12 @@ class SingleFileStorage(Storage): except KeyError: raise exceptions.NotFoundError(href) + async def get_multi(self, hrefs: Iterable[str]): + async with self.at_once(): + for href in uniq(hrefs): + item, etag = await self.get(href) + yield href, item, etag + @_writing_op async def upload(self, item): href = item.ident