From 804b9f04293b2ad5b696fcba0f417500ec97d735 Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Wed, 19 May 2021 17:40:04 +0200 Subject: [PATCH 1/6] Add tests to filesystem storage for file ignorance that ignore .tmp files even when fileext is empty. Prepares to make the filesystem storage more universal as part of #881 . --- tests/storage/test_filesystem.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tests/storage/test_filesystem.py b/tests/storage/test_filesystem.py index 94e5d17..20bbf3a 100644 --- a/tests/storage/test_filesystem.py +++ b/tests/storage/test_filesystem.py @@ -44,6 +44,25 @@ class TestFilesystemStorage(StorageTests): (item_file,) = tmpdir.listdir() assert "/" not in item_file.basename and item_file.isfile() + def test_ignore_tmp_files(self, tmpdir): + """Test that files with .tmp suffix beside .ics files are ignored.""" + s = self.storage_class(str(tmpdir), '.ics') + s.upload(Item('UID:xyzxyz')) + item_file, = tmpdir.listdir() + item_file.copy(item_file.new(ext='tmp')) + assert len(tmpdir.listdir()) == 2 + assert len(list(s.list())) == 1 + + def test_ignore_tmp_files_empty_fileext(self, tmpdir): + """Test that files with .tmp suffix are ignored with empty fileext.""" + s = self.storage_class(str(tmpdir), '') + s.upload(Item('UID:xyzxyz')) + item_file, = tmpdir.listdir() + item_file.copy(item_file.new(ext='tmp')) + assert len(tmpdir.listdir()) == 2 + # assert False, tmpdir.listdir() # enable to see the created filename + assert len(list(s.list())) == 1 + def test_too_long_uid(self, tmpdir): s = self.storage_class(str(tmpdir), ".txt") item = Item("UID:" + "hue" * 600) From 439e63f8eab65aeca68f398ca2898e5de09e0de5 Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Wed, 19 May 2021 17:51:57 +0200 Subject: [PATCH 2/6] Make /storage/filesystem ignore `.tmp` files Hardcode to ignore files with `.tmp` suffix as this is mentioned in the vdir specification. --- vdirsyncer/storage/filesystem.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vdirsyncer/storage/filesystem.py b/vdirsyncer/storage/filesystem.py index 352e59c..1e47ba1 100644 --- a/vdirsyncer/storage/filesystem.py +++ b/vdirsyncer/storage/filesystem.py @@ -80,7 +80,8 @@ class FilesystemStorage(Storage): def list(self): for fname in os.listdir(self.path): fpath = os.path.join(self.path, fname) - if os.path.isfile(fpath) and fname.endswith(self.fileext): + if os.path.isfile(fpath) and fname.endswith(self.fileext) and ( + not fname.endswith('.tmp')): yield fname, get_etag_from_file(fpath) def get(self, href): From 81895c291e68395512379b98b44981c0e643e705 Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Wed, 19 May 2021 18:00:09 +0200 Subject: [PATCH 3/6] Make /storage/filesystem more flexible by adding the optional fileignoreext parameter. --- docs/config.rst | 3 +++ vdirsyncer/storage/filesystem.py | 6 ++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 0ec8695..454074a 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -408,6 +408,7 @@ Local fileext = "..." #encoding = "utf-8" #post_hook = null + #fileextignore = ".tmp" Can be used with `khal `_. See :doc:`vdir` for a more formal description of the format. @@ -426,6 +427,8 @@ Local :param post_hook: A command to call for each item creation and modification. The command will be called with the path of the new/updated file. + :param fileextignore: The file extention to ignore, + the default is ``.tmp``. .. storage:: singlefile diff --git a/vdirsyncer/storage/filesystem.py b/vdirsyncer/storage/filesystem.py index 1e47ba1..1fd1ac6 100644 --- a/vdirsyncer/storage/filesystem.py +++ b/vdirsyncer/storage/filesystem.py @@ -22,13 +22,15 @@ class FilesystemStorage(Storage): storage_name = "filesystem" _repr_attributes = ("path",) - def __init__(self, path, fileext, encoding="utf-8", post_hook=None, **kwargs): + def __init__(self, path, fileext, + encoding="utf-8", post_hook=None, fileignoreext=".tmp", **kwargs): super().__init__(**kwargs) path = expand_path(path) checkdir(path, create=False) self.path = path self.encoding = encoding self.fileext = fileext + self.fileignoreext = fileignoreext self.post_hook = post_hook @classmethod @@ -81,7 +83,7 @@ class FilesystemStorage(Storage): for fname in os.listdir(self.path): fpath = os.path.join(self.path, fname) if os.path.isfile(fpath) and fname.endswith(self.fileext) and ( - not fname.endswith('.tmp')): + not fname.endswith(self.fileignoreext)): yield fname, get_etag_from_file(fpath) def get(self, href): From 9b5e01ab3870352299a3c0817093a2c9196f5203 Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Thu, 20 May 2021 12:50:05 +0200 Subject: [PATCH 4/6] Improve storage/test_filesystem with one additional test and fixing of the documentation. --- docs/config.rst | 4 ++-- tests/storage/test_filesystem.py | 11 +++++++++++ 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 454074a..6c35476 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -408,7 +408,7 @@ Local fileext = "..." #encoding = "utf-8" #post_hook = null - #fileextignore = ".tmp" + #fileignoreext = ".tmp" Can be used with `khal `_. See :doc:`vdir` for a more formal description of the format. @@ -427,7 +427,7 @@ Local :param post_hook: A command to call for each item creation and modification. The command will be called with the path of the new/updated file. - :param fileextignore: The file extention to ignore, + :param fileeignoreext: The file extention to ignore, the default is ``.tmp``. .. storage:: singlefile diff --git a/tests/storage/test_filesystem.py b/tests/storage/test_filesystem.py index 20bbf3a..f85c1f3 100644 --- a/tests/storage/test_filesystem.py +++ b/tests/storage/test_filesystem.py @@ -63,6 +63,17 @@ class TestFilesystemStorage(StorageTests): # assert False, tmpdir.listdir() # enable to see the created filename assert len(list(s.list())) == 1 + def test_ignore_files_typical_backup(self, tmpdir): + """Test file-name ignorance with typical backup ending ~.""" + ignorext = "~" # without dot + s = self.storage_class(str(tmpdir), '', fileignoreext="~") + s.upload(Item('UID:xyzxyz')) + item_file, = tmpdir.listdir() + item_file.copy(item_file.new(basename=item_file.basename+'~')) + assert len(tmpdir.listdir()) == 2 + #assert False, tmpdir.listdir() # enable to see the created filename + assert len(list(s.list())) == 1 + def test_too_long_uid(self, tmpdir): s = self.storage_class(str(tmpdir), ".txt") item = Item("UID:" + "hue" * 600) From 9a1582cc0faec365bab4d952c3b7f14480d16a1d Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Fri, 21 May 2021 14:22:30 +0200 Subject: [PATCH 5/6] Improve storage/filesystem docs add hints about how to use the `fileext` and `fileignoreext` parameters. --- docs/config.rst | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 6c35476..4b46f97 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -422,13 +422,15 @@ Local :param fileext: The file extension to use (e.g. ``.txt``). Contained in the href, so if you change the file extension after a sync, this will trigger a re-download of everything (but *should* not cause data-loss - of any kind). + of any kind). To be compatible with the ``vset to the empty string`` format you have + to either use ``.vcf`` or ``.ics``. Note that metasync won't work + if you use an empty string here. :param encoding: File encoding for items, both content and filename. :param post_hook: A command to call for each item creation and modification. The command will be called with the path of the new/updated file. - :param fileeignoreext: The file extention to ignore, - the default is ``.tmp``. + :param fileeignoreext: The file extention to ignore. It is only useful + if fileext is set to the empty string. The default is ``.tmp``. .. storage:: singlefile From b9f5d88af9f51421ca6230ce2ae53cfb12c9a995 Mon Sep 17 00:00:00 2001 From: Bernhard Reiter Date: Wed, 2 Jun 2021 13:52:10 +0200 Subject: [PATCH 6/6] Fixing docs/config.rst and code formatting --- docs/config.rst | 2 +- vdirsyncer/storage/filesystem.py | 18 ++++++++++++++---- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/docs/config.rst b/docs/config.rst index 4b46f97..d8022d9 100644 --- a/docs/config.rst +++ b/docs/config.rst @@ -422,7 +422,7 @@ Local :param fileext: The file extension to use (e.g. ``.txt``). Contained in the href, so if you change the file extension after a sync, this will trigger a re-download of everything (but *should* not cause data-loss - of any kind). To be compatible with the ``vset to the empty string`` format you have + of any kind). To be compatible with the ``vset`` format you have to either use ``.vcf`` or ``.ics``. Note that metasync won't work if you use an empty string here. :param encoding: File encoding for items, both content and filename. diff --git a/vdirsyncer/storage/filesystem.py b/vdirsyncer/storage/filesystem.py index 1fd1ac6..98159ec 100644 --- a/vdirsyncer/storage/filesystem.py +++ b/vdirsyncer/storage/filesystem.py @@ -22,8 +22,15 @@ class FilesystemStorage(Storage): storage_name = "filesystem" _repr_attributes = ("path",) - def __init__(self, path, fileext, - encoding="utf-8", post_hook=None, fileignoreext=".tmp", **kwargs): + def __init__( + self, + path, + fileext, + encoding="utf-8", + post_hook=None, + fileignoreext=".tmp", + **kwargs + ): super().__init__(**kwargs) path = expand_path(path) checkdir(path, create=False) @@ -82,8 +89,11 @@ class FilesystemStorage(Storage): def list(self): for fname in os.listdir(self.path): fpath = os.path.join(self.path, fname) - if os.path.isfile(fpath) and fname.endswith(self.fileext) and ( - not fname.endswith(self.fileignoreext)): + if ( + os.path.isfile(fpath) + and fname.endswith(self.fileext) + and (not fname.endswith(self.fileignoreext)) + ): yield fname, get_etag_from_file(fpath) def get(self, href):