From c7e6ca20e461c8461f22dca151b543a128e9d8fd Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 2 Mar 2016 23:27:48 +0100 Subject: [PATCH 01/11] Add packaging guidelines --- MANIFEST.in | 2 ++ Makefile | 15 +++-------- docs-requirements.txt | 2 ++ docs/index.rst | 1 + docs/packaging.rst | 60 +++++++++++++++++++++++++++++++++++++++++++ setup.py | 2 +- test-requirements.txt | 5 ++++ 7 files changed, 74 insertions(+), 13 deletions(-) create mode 100644 docs-requirements.txt create mode 100644 docs/packaging.rst create mode 100644 test-requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 24ea06d..4faef4e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,8 @@ include CHANGELOG.rst include LICENSE include config.example include Makefile +include test-requirements.txt +include docs-requirements.txt recursive-include docs * recursive-include tests * diff --git a/Makefile b/Makefile index 2f2ea0d..4da8219 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,4 @@ -# Packagers who want to run the testsuite against an installed vdirsyncer: -# -# - Create a virtualenv -# - Somehow link your installation of vdirsyncer into the virtualenv, e.g. by -# using --system-site-packages when creating the virtualenv -# - Inside the virtualenv: `make install-test test` -# -# The `install-test` target requires internet access. +# See the documentation on how to run the tests. export DAV_SERVER := skip export REMOTESTORAGE_SERVER := skip @@ -28,14 +21,12 @@ install-servers: install-test: install-servers (python --version | grep -vq 'Python 3.3') || pip install enum34 + pip install -r test-requirements.txt set -xe && if [ "$$REQUIREMENTS" = "devel" ]; then \ pip install -U --force-reinstall \ git+https://github.com/DRMacIver/hypothesis \ git+https://github.com/pytest-dev/pytest; \ - else \ - pip install pytest hypothesis; \ fi - pip install pytest-xprocess pytest-localserver pytest-subtesthack [ $(TRAVIS) != "true" ] || pip install coverage codecov test: @@ -62,7 +53,7 @@ travis-conf: python3 scripts/make_travisconf.py > .travis.yml install-docs: - pip install sphinx sphinx_rtd_theme + pip install -r docs-requirements.txt docs: cd docs && make html diff --git a/docs-requirements.txt b/docs-requirements.txt new file mode 100644 index 0000000..8213302 --- /dev/null +++ b/docs-requirements.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme diff --git a/docs/index.rst b/docs/index.rst index 5f72d14..50aa337 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -29,6 +29,7 @@ Table of Contents problems vdir contributing + packaging contact changelog license diff --git a/docs/packaging.rst b/docs/packaging.rst new file mode 100644 index 0000000..77663ca --- /dev/null +++ b/docs/packaging.rst @@ -0,0 +1,60 @@ +==================== +Packaging guidelines +==================== + +Thank you very much for packaging vdirsyncer! The following guidelines should +help you to avoid some common pitfalls. + +While they are called guidelines and therefore theoretically not mandatory, if +you consider going a different direction, please first open an issue or contact +me otherwise instead of just going ahead. These guidelines exist for my own +convenience too. + +Obtaining the source code +========================= + +The main distribution channel is `PyPI +`_, and source tarballs can be +obtained there. Do not use the ones from GitHub: Their tarballs contain useless +junk and are more of a distraction than anything else. + +I give each release a tag in the git repo. If you want to get notified of new +releases, `GitHub's feed +`_ is a good way. + +Dependency versions +=================== + +It is strongly discouraged to package vdirsyncer as a Python 2 application. +Future releases will only work on Python 3.3 and newer versions. + +As with most Python packages, ``setup.py`` denotes the runtime dependencies of +vdirsyncer. It also contains lower-bound versions of each dependency. Older +versions will be rejected by the testsuite. + +Testing +======= + +Everything testing-related goes through the ``Makefile`` in the root of the +repository or PyPI package. Trying to e.g. run ``py.test`` directly will +require a lot of environment variables to be set (for configuration) and you +probably don't want to deal with that. + +You can install the testing dependencies with ``make test-install``. You +probably don't want this since it will use pip to download the dependencies. +Alternatively you can find the testing dependencies in +``test-requirements.txt``, again with lower-bound version requirements. + +You also have to have vdirsyncer fully installed at this point. Merely +``cd``-ing into the tarball will not be sufficient. + +Running the tests happens with ``make test``. + +Documentation +============= + +You can find a list of dependencies in ``docs-requirements.txt``. + +Change into the ``docs/`` directory and build whatever format you want. That +said, I only take care of the HTML docs' formatting -- other targets (such as +the generated manpage) may look like garbage. diff --git a/setup.py b/setup.py index f721e18..6831bad 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ setup( ), # https://github.com/sigmavirus24/requests-toolbelt/pull/28 'requests_toolbelt >=0.5.0', - 'atomicwrites' + 'atomicwrites>=0.1.6' ], extras_require={ 'remotestorage': ['requests-oauthlib'] diff --git a/test-requirements.txt b/test-requirements.txt new file mode 100644 index 0000000..85cb644 --- /dev/null +++ b/test-requirements.txt @@ -0,0 +1,5 @@ +hypothesis>=3 +pytest +pytest-localserver +pytest-subtesthack +pytest-xprocess From cd07d7fc686bb353ab0831c7dff8cfaa102aee48 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Thu, 3 Mar 2016 13:05:00 +0100 Subject: [PATCH 02/11] Add test for proper dependencies --- tests/cli/test_main.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/cli/test_main.py b/tests/cli/test_main.py index bb89c76..1cd8e2b 100644 --- a/tests/cli/test_main.py +++ b/tests/cli/test_main.py @@ -8,12 +8,22 @@ from click.testing import CliRunner from hypothesis import example, given import hypothesis.strategies as st +from pkg_resources import load_entry_point + import pytest import vdirsyncer.cli as cli from vdirsyncer.utils.compat import PY2, to_native +def test_entry_points(monkeypatch, capsys): + monkeypatch.setattr('sys.argv', ['--help']) + with pytest.raises(SystemExit) as excinfo: + load_entry_point('vdirsyncer', 'console_scripts', 'vdirsyncer')() + + assert excinfo.value.code == 0 + + def test_simple_run(tmpdir, runner): runner.write_with_general(dedent(''' [pair my_pair] From 48a649ee8c367ce89b87501b49a221932c90648b Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 08:23:32 +0100 Subject: [PATCH 03/11] Fix up requirements=minimal --- Makefile | 10 +++++++++- setup.py | 11 ++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 4da8219..8dcbbc3 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,15 @@ install-dev: set -xe && if [ "$$REQUIREMENTS" = "devel" ]; then \ pip install -U --force-reinstall git+https://github.com/kennethreitz/requests; \ elif [ "$$REQUIREMENTS" = "minimal" ]; then \ - pip install -U --force-reinstall lxml==3.1 requests==2.4.1 requests_toolbelt==0.4.0 click==5.0; \ + # FIXME: Looking for a tool that parses setup.py requirements and installs minimum viable version + pip install -U --force-reinstall \ + lxml==3.1 \ + requests==2.0.1 \ + requests_toolbelt==0.4.0 \ + click-threading==0.1.2 \ + click-log==0.1.3 \ + atomicwrites==0.1.7 \ + click==5.0; \ fi .PHONY: docs diff --git a/setup.py b/setup.py index 6831bad..4e3dab9 100644 --- a/setup.py +++ b/setup.py @@ -35,10 +35,10 @@ setup( install_requires=[ # https://github.com/mitsuhiko/click/issues/200 'click>=5.0', - 'click-log', - 'click-threading', + 'click-log>=0.1.3', + 'click-threading>=0.1.2', # https://github.com/kennethreitz/requests/issues/2930 - 'requests !=2.9.0', + 'requests >=2.0.1, !=2.9.0', 'lxml >=3.1' + ( # See https://github.com/untitaker/vdirsyncer/issues/298 # We pin some LXML version that is known to work with PyPy @@ -49,8 +49,9 @@ setup( else '' ), # https://github.com/sigmavirus24/requests-toolbelt/pull/28 - 'requests_toolbelt >=0.5.0', - 'atomicwrites>=0.1.6' + 'requests_toolbelt >=0.3.0', + # https://github.com/untitaker/python-atomicwrites/commit/4d12f23227b6a944ab1d99c507a69fdbc7c9ed6d # noqa + 'atomicwrites>=0.1.7' ], extras_require={ 'remotestorage': ['requests-oauthlib'] From 8a7ac52ee773b25dd5f479bd29a0ff8562706bfe Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Fri, 4 Mar 2016 06:15:25 -0300 Subject: [PATCH 04/11] Implement command to print minimal requirements --- setup.py | 69 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 22 deletions(-) diff --git a/setup.py b/setup.py index 4e3dab9..cf1ec45 100644 --- a/setup.py +++ b/setup.py @@ -11,10 +11,52 @@ Vdirsyncer is a synchronization tool for vdir. See the README for more details. import platform +import re -from setuptools import find_packages, setup +from setuptools import find_packages, setup, Command +requirements = [ + # https://github.com/mitsuhiko/click/issues/200 + 'click>=5.0', + 'click-log>=0.1.3', + 'click-threading>=0.1.2', + # https://github.com/kennethreitz/requests/issues/2930 + 'requests >=2.0.1, !=2.9.0', + 'lxml >=3.1' + ( + # See https://github.com/untitaker/vdirsyncer/issues/298 + # We pin some LXML version that is known to work with PyPy + # I assume nobody actually uses PyPy with vdirsyncer, so this is + # moot + ', <=3.4.4' + if platform.python_implementation() == 'PyPy' + else '' + ), + # https://github.com/sigmavirus24/requests-toolbelt/pull/28 + 'requests_toolbelt >=0.3.0', + # https://github.com/untitaker/python-atomicwrites/commit/4d12f23227b6a944ab1d99c507a69fdbc7c9ed6d # noqa + 'atomicwrites>=0.1.7' +] + + +class PrintRequirements(Command): + + description = 'Prints minimal requirements' + + user_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + [ + print(requirement.replace(">", "=").replace(" ", "")) + for requirement in requirements + ] + setup( name='vdirsyncer', use_scm_version={ @@ -32,28 +74,11 @@ setup( entry_points={ 'console_scripts': ['vdirsyncer = vdirsyncer.cli:main'] }, - install_requires=[ - # https://github.com/mitsuhiko/click/issues/200 - 'click>=5.0', - 'click-log>=0.1.3', - 'click-threading>=0.1.2', - # https://github.com/kennethreitz/requests/issues/2930 - 'requests >=2.0.1, !=2.9.0', - 'lxml >=3.1' + ( - # See https://github.com/untitaker/vdirsyncer/issues/298 - # We pin some LXML version that is known to work with PyPy - # I assume nobody actually uses PyPy with vdirsyncer, so this is - # moot - ', <=3.4.4' - if platform.python_implementation() == 'PyPy' - else '' - ), - # https://github.com/sigmavirus24/requests-toolbelt/pull/28 - 'requests_toolbelt >=0.3.0', - # https://github.com/untitaker/python-atomicwrites/commit/4d12f23227b6a944ab1d99c507a69fdbc7c9ed6d # noqa - 'atomicwrites>=0.1.7' - ], + install_requires=requirements, extras_require={ 'remotestorage': ['requests-oauthlib'] + }, + cmdclass={ + 'requirements': PrintRequirements } ) From dd48ac05c91994d69cd71b0f866b4bdf3bc1c3c9 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 10:32:43 +0100 Subject: [PATCH 05/11] Rename setup.py requirements cmd --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cf1ec45..6f112ff 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,6 @@ setup( 'remotestorage': ['requests-oauthlib'] }, cmdclass={ - 'requirements': PrintRequirements + 'minimal_requirements': PrintRequirements } ) From fe5ac51231bb1452dfc14c2b106fd979290ac44f Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 10:34:40 +0100 Subject: [PATCH 06/11] Use new minimal_requirements command in Makefile --- Makefile | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 8dcbbc3..9fbf8e3 100644 --- a/Makefile +++ b/Makefile @@ -79,15 +79,7 @@ install-dev: set -xe && if [ "$$REQUIREMENTS" = "devel" ]; then \ pip install -U --force-reinstall git+https://github.com/kennethreitz/requests; \ elif [ "$$REQUIREMENTS" = "minimal" ]; then \ - # FIXME: Looking for a tool that parses setup.py requirements and installs minimum viable version - pip install -U --force-reinstall \ - lxml==3.1 \ - requests==2.0.1 \ - requests_toolbelt==0.4.0 \ - click-threading==0.1.2 \ - click-log==0.1.3 \ - atomicwrites==0.1.7 \ - click==5.0; \ + pip install -U --force-reinstall $$(python setup.py --quiet minimal_requirements); \ fi .PHONY: docs From c8c4409321ab97b3afdac9989d2203ecd5270f8b Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 10:37:50 +0100 Subject: [PATCH 07/11] Just use a normal loop --- setup.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 6f112ff..8f79719 100644 --- a/setup.py +++ b/setup.py @@ -52,10 +52,8 @@ class PrintRequirements(Command): pass def run(self): - [ + for requirement in requirements: print(requirement.replace(">", "=").replace(" ", "")) - for requirement in requirements - ] setup( name='vdirsyncer', From 1ede6884b78b9cca793b651587bebe18606c93b2 Mon Sep 17 00:00:00 2001 From: Hugo Osvaldo Barrera Date: Fri, 4 Mar 2016 05:11:40 -0300 Subject: [PATCH 08/11] Allow running deterministic tests --- Makefile | 2 ++ tests/conftest.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 9fbf8e3..6692248 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ export RADICALE_BACKEND := filesystem export REQUIREMENTS := release export TESTSERVER_BASE := ./tests/storage/servers/ export TRAVIS := false +export CI := false +export DETERMINISTIC_TESTS := false install-servers: set -ex; \ diff --git a/tests/conftest.py b/tests/conftest.py index f233759..dd2fe16 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -3,10 +3,11 @@ General-purpose fixtures for vdirsyncer's testsuite. ''' import logging +import os import click_log - import pytest +from hypothesis import settings, Verbosity @pytest.fixture(autouse=True) @@ -27,3 +28,16 @@ except ImportError: return lambda x: x() else: del pytest_benchmark + +settings.register_profile("ci", settings( + max_examples=1000, + verbosity=Verbosity.verbose, +)) +settings.register_profile("deterministic", settings( + derandomize=True, +)) + +if os.getenv('DETERMINISTIC_TESTS').lower == 'true': + settings.load_profile("deterministic") +elif os.getenv('CI').lower == 'true': + settings.load_profile("ci") From 7ad2af7063b467727b0a1373048937a60c30bc1a Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 10:43:30 +0100 Subject: [PATCH 09/11] packaging.rst: Add note about DETERMINISTIC_TESTS --- docs/packaging.rst | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/packaging.rst b/docs/packaging.rst index 77663ca..a86b417 100644 --- a/docs/packaging.rst +++ b/docs/packaging.rst @@ -50,6 +50,11 @@ You also have to have vdirsyncer fully installed at this point. Merely Running the tests happens with ``make test``. +Hypothesis will randomly generate test input. If you care about deterministic +tests, set the ``DETERMINISTIC_TESTS`` variable to ``"true"``:: + + make DETERMINISTIC_TESTS=true test + Documentation ============= From e244eecc520835c45a6780ec393f23811b117fc3 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 11:06:45 +0100 Subject: [PATCH 10/11] Stylefixes --- setup.py | 3 +-- tests/conftest.py | 4 +++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 8f79719..5b8ffde 100644 --- a/setup.py +++ b/setup.py @@ -11,9 +11,8 @@ Vdirsyncer is a synchronization tool for vdir. See the README for more details. import platform -import re -from setuptools import find_packages, setup, Command +from setuptools import Command, find_packages, setup requirements = [ diff --git a/tests/conftest.py b/tests/conftest.py index dd2fe16..e9de77a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,8 +6,10 @@ import logging import os import click_log + +from hypothesis import Verbosity, settings + import pytest -from hypothesis import settings, Verbosity @pytest.fixture(autouse=True) From 13af5ffbafe159adedfb867a5cb97dfa77a55c2d Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 4 Mar 2016 14:15:23 +0100 Subject: [PATCH 11/11] Replace CI with TRAVIS --- Makefile | 1 - tests/conftest.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 6692248..4596864 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ export RADICALE_BACKEND := filesystem export REQUIREMENTS := release export TESTSERVER_BASE := ./tests/storage/servers/ export TRAVIS := false -export CI := false export DETERMINISTIC_TESTS := false install-servers: diff --git a/tests/conftest.py b/tests/conftest.py index e9de77a..2268df9 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -41,5 +41,5 @@ settings.register_profile("deterministic", settings( if os.getenv('DETERMINISTIC_TESTS').lower == 'true': settings.load_profile("deterministic") -elif os.getenv('CI').lower == 'true': +elif os.getenv('TRAVIS').lower == 'true': settings.load_profile("ci")