diff --git a/Makefile b/Makefile index 9579a30..52d70cb 100644 --- a/Makefile +++ b/Makefile @@ -93,12 +93,18 @@ install-test: install-servers install-style: install-docs pip install -U flake8 flake8-import-order 'flake8-bugbear>=17.3.0' + which cargo-install-update || cargo +nightly install cargo-update + cargo +nightly install-update -i clippy + cargo +nightly install-update -i rustfmt-nightly + cargo +nightly install-update -i cargo-update style: flake8 ! git grep -i syncroniz */* ! git grep -i 'text/icalendar' */* sphinx-build -W -b html ./docs/ ./docs/_build/html/ + cd rust/ && cargo +nightly clippy + cd rust/ && cargo fmt install-docs: pip install -Ur docs-requirements.txt @@ -139,7 +145,7 @@ ssh-submodule-urls: git remote get-url origin" install-rust: - curl https://sh.rustup.rs -sSf | sh -s -- -y + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain nightly rust-ext: [ "$$READTHEDOCS" != "True" ] || $(MAKE) install-rust diff --git a/rust/.gitignore b/rust/.gitignore index 2f7896d..ac3376e 100644 --- a/rust/.gitignore +++ b/rust/.gitignore @@ -1 +1,2 @@ target/ +src/storage/exports.rs diff --git a/rust/Cargo.lock b/rust/Cargo.lock index fb14d83..35d50a1 100644 --- a/rust/Cargo.lock +++ b/rust/Cargo.lock @@ -1,40 +1,39 @@ -[root] -name = "vdirsyncer_rustext" -version = "0.1.0" -dependencies = [ - "cbindgen 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)", - "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", - "vobject 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "ansi_term" -version = "0.9.0" +version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] -name = "atty" -version = "0.2.3" +name = "atomicwrites" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "nix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "atty" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", + "termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "backtrace" -version = "0.3.3" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", - "dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", "rustc-demangle 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -42,38 +41,37 @@ name = "backtrace-sys" version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "cc 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "bitflags" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" - [[package]] name = "bitflags" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "bitflags" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "cbindgen" -version = "0.1.29" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)", - "log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)", + "clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)", + "log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", "tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", - "toml 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "cc" -version = "1.0.3" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -83,12 +81,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "clap" -version = "2.27.1" +version = "2.29.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", - "atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)", - "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)", + "atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", "strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)", "textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", "unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", @@ -100,19 +98,10 @@ name = "coco" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "either 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", + "either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)", "scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] -[[package]] -name = "dbghelp-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", - "winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", -] - [[package]] name = "dtoa" version = "0.4.2" @@ -120,7 +109,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "either" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -128,28 +117,21 @@ name = "error-chain" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "backtrace 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "backtrace 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "fuchsia-zircon" -version = "0.2.1" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "fuchsia-zircon-sys" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -dependencies = [ - "bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)", -] - -[[package]] -name = "futures" -version = "0.1.17" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -173,30 +155,52 @@ dependencies = [ [[package]] name = "lazy_static" -version = "0.2.9" +version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "libc" -version = "0.2.33" +version = "0.2.35" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "log" -version = "0.3.8" +version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "log" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "nix" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", + "void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)", +] [[package]] name = "num-traits" -version = "0.1.40" +version = "0.1.41" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "num_cpus" -version = "1.7.0" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -206,11 +210,11 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "rand" -version = "0.3.17" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -218,25 +222,24 @@ name = "rayon" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "rayon-core" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", - "futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)", - "rand 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", + "num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "redox_syscall" -version = "0.1.31" +version = "0.1.37" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -244,7 +247,7 @@ name = "redox_termios" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -253,8 +256,8 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)", - "lazy_static 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", + "lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", "rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -271,36 +274,40 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "serde" -version = "0.9.15" -source = "registry+https://github.com/rust-lang/crates.io-index" - -[[package]] -name = "serde_codegen_internals" -version = "0.14.2" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "serde_derive" -version = "0.9.15" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)", - "serde_codegen_internals 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)", + "serde_derive_internals 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)", "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "serde_derive_internals" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)", + "synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "serde_json" -version = "0.9.10" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", "itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)", - "num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)", - "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -331,7 +338,7 @@ name = "tempdir" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "rand 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)", + "rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -339,8 +346,8 @@ name = "termion" version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)", - "redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)", + "redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)", "redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -354,10 +361,10 @@ dependencies = [ [[package]] name = "toml" -version = "0.3.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)", + "serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] @@ -375,6 +382,17 @@ name = "untrusted" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "vdirsyncer_rustext" +version = "0.1.0" +dependencies = [ + "atomicwrites 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", + "cbindgen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)", + "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)", + "vobject 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "vec_map" version = "0.8.0" @@ -382,73 +400,102 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "vobject" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [[package]] name = "winapi" version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "winapi" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "winapi-build" version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" + [metadata] -"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" -"checksum atty 0.2.3 (registry+https://github.com/rust-lang/crates.io-index)" = "21e50800ec991574876040fff8ee46b136a53e985286fbe6a3bdfe6421b78860" -"checksum backtrace 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "99f2ce94e22b8e664d95c57fff45b98a966c2252b60691d0b7aeeccd88d70983" +"checksum ansi_term 0.10.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6b3568b48b7cefa6b8ce125f9bb4989e52fbcc29ebea88df04cc7c5f12f70455" +"checksum atomicwrites 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "4560dd4eadad8c80a88e25426f96a74ad62c95d4ee424226803013c0ba94f1cf" +"checksum atty 0.2.6 (registry+https://github.com/rust-lang/crates.io-index)" = "8352656fd42c30a0c3c89d26dea01e3b77c0ab2af18230835c15e2e13cd51859" +"checksum backtrace 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "ebbbf59b1c43eefa8c3ede390fcc36820b4999f7914104015be25025e0d62af2" "checksum backtrace-sys 0.1.16 (registry+https://github.com/rust-lang/crates.io-index)" = "44585761d6161b0f57afc49482ab6bd067e4edef48c12a152c237eb0203f7661" -"checksum bitflags 0.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "aad18937a628ec6abcd26d1489012cc0e18c21798210f491af69ded9b881106d" "checksum bitflags 0.9.1 (registry+https://github.com/rust-lang/crates.io-index)" = "4efd02e230a02e18f92fc2735f44597385ed02ad8f831e7c1c1156ee5e1ab3a5" -"checksum cbindgen 0.1.29 (registry+https://github.com/rust-lang/crates.io-index)" = "a4f329332ea243c7bf4ca3fc8a0c35884ab6de60b4c4a2bb89238ec6947a75cb" -"checksum cc 1.0.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a9b13a57efd6b30ecd6598ebdb302cca617930b5470647570468a65d12ef9719" +"checksum bitflags 1.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "b3c30d3802dfb7281680d6285f2ccdaa8c2d8fee41f93805dba5c4cf50dc23cf" +"checksum cbindgen 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "370c18a61741bd716377aba3fc42d78788df5d1af5e4bfbe22926013bd91d50a" +"checksum cc 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "deaf9ec656256bb25b404c51ef50097207b9cbb29c933d31f92cae5a8a0ffee0" "checksum cfg-if 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "d4c819a1287eb618df47cc647173c5c4c66ba19d888a6e50d605672aed3140de" -"checksum clap 2.27.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1b8c532887f1a292d17de05ae858a8fe50a301e196f9ef0ddb7ccd0d1d00f180" +"checksum clap 2.29.1 (registry+https://github.com/rust-lang/crates.io-index)" = "8f4a2b3bb7ef3c672d7c13d15613211d5a6976b6892c598b0fcb5d40765f19c2" "checksum coco 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "c06169f5beb7e31c7c67ebf5540b8b472d23e3eade3b2ec7d1f5b504a85f91bd" -"checksum dbghelp-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "97590ba53bcb8ac28279161ca943a924d1fd4a8fb3fa63302591647c4fc5b850" "checksum dtoa 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "09c3753c3db574d215cba4ea76018483895d7bff25a31b49ba45db21c48e50ab" -"checksum either 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e311a7479512fbdf858fb54d91ec59f3b9f85bc0113659f46bba12b199d273ce" +"checksum either 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "740178ddf48b1a9e878e6d6509a1442a2d42fd2928aae8e7a6f8a36fb01981b3" "checksum error-chain 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ff511d5dc435d703f4971bc399647c9bc38e20cb41452e3b9feb4765419ed3f3" -"checksum fuchsia-zircon 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f6c0581a4e363262e52b87f59ee2afe3415361c6ec35e665924eb08afe8ff159" -"checksum fuchsia-zircon-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "43f3795b4bae048dc6123a6b972cadde2e676f9ded08aef6bb77f5f157684a82" -"checksum futures 0.1.17 (registry+https://github.com/rust-lang/crates.io-index)" = "118b49cac82e04121117cbd3121ede3147e885627d82c4546b87c702debb90c1" +"checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +"checksum fuchsia-zircon-sys 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" "checksum gcc 0.3.54 (registry+https://github.com/rust-lang/crates.io-index)" = "5e33ec290da0d127825013597dbdfc28bee4964690c7ce1166cbc2a7bd08b1bb" "checksum itoa 0.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8324a32baf01e2ae060e9de58ed0bc2320c9a2833491ee36cd3b4c414de4db8c" "checksum kernel32-sys 0.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -"checksum lazy_static 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c9e5e58fa1a4c3b915a561a78a22ee0cac6ab97dca2504428bc1cb074375f8d5" -"checksum libc 0.2.33 (registry+https://github.com/rust-lang/crates.io-index)" = "5ba3df4dcb460b9dfbd070d41c94c19209620c191b0340b929ce748a2bcd42d2" -"checksum log 0.3.8 (registry+https://github.com/rust-lang/crates.io-index)" = "880f77541efa6e5cc74e76910c9884d9859683118839d6a1dc3b11e63512565b" -"checksum num-traits 0.1.40 (registry+https://github.com/rust-lang/crates.io-index)" = "99843c856d68d8b4313b03a17e33c4bb42ae8f6610ea81b28abe076ac721b9b0" -"checksum num_cpus 1.7.0 (registry+https://github.com/rust-lang/crates.io-index)" = "514f0d73e64be53ff320680ca671b64fe3fb91da01e1ae2ddc99eb51d453b20d" +"checksum lazy_static 0.2.11 (registry+https://github.com/rust-lang/crates.io-index)" = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" +"checksum libc 0.2.35 (registry+https://github.com/rust-lang/crates.io-index)" = "96264e9b293e95d25bfcbbf8a88ffd1aedc85b754eba8b7d78012f638ba220eb" +"checksum log 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "e19e8d5c34a3e0e2223db8e060f9e8264aeeb5c5fc64a4ee9965c062211c024b" +"checksum log 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "89f010e843f2b1a31dbd316b3b8d443758bc634bed37aabade59c686d644e0a2" +"checksum nix 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "a2c5afeb0198ec7be8569d666644b574345aad2e95a53baf3a532da3e0f3fb32" +"checksum num-traits 0.1.41 (registry+https://github.com/rust-lang/crates.io-index)" = "cacfcab5eb48250ee7d0c7896b51a2c5eec99c1feea5f32025635f5ae4b00070" +"checksum num_cpus 1.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c51a3322e4bca9d212ad9a158a02abc6934d005490c054a2778df73a70aa0a30" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" -"checksum rand 0.3.17 (registry+https://github.com/rust-lang/crates.io-index)" = "61efcbcd9fa8d8fbb07c84e34a8af18a1ff177b449689ad38a6e9457ecc7b2ae" +"checksum rand 0.3.20 (registry+https://github.com/rust-lang/crates.io-index)" = "512870020642bb8c221bf68baa1b2573da814f6ccfe5c9699b1c303047abe9b1" "checksum rayon 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)" = "b614fe08b6665cb9a231d07ac1364b0ef3cb3698f1239ee0c4c3a88a524f54c8" -"checksum rayon-core 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7febc28567082c345f10cddc3612c6ea020fc3297a1977d472cf9fdb73e6e493" -"checksum redox_syscall 0.1.31 (registry+https://github.com/rust-lang/crates.io-index)" = "8dde11f18c108289bef24469638a04dce49da56084f2d50618b226e47eb04509" +"checksum rayon-core 1.3.0 (registry+https://github.com/rust-lang/crates.io-index)" = "e64b609139d83da75902f88fd6c01820046840a18471e4dfcd5ac7c0f46bea53" +"checksum redox_syscall 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "0d92eecebad22b767915e4d529f89f28ee96dbbf5a4810d2b844373f136417fd" "checksum redox_termios 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "7e891cfe48e9100a70a3b6eb652fef28920c117d366339687bd5576160db0f76" "checksum ring 0.12.1 (registry+https://github.com/rust-lang/crates.io-index)" = "6f7d28b30a72c01b458428e0ae988d4149c20d902346902be881e3edc4bb325c" "checksum rustc-demangle 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "aee45432acc62f7b9a108cc054142dac51f979e69e71ddce7d6fc7adf29e817e" "checksum scopeguard 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "94258f53601af11e6a49f722422f6e3425c52b06245a5cf9bc09908b174f5e27" -"checksum serde 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "34b623917345a631dc9608d5194cc206b3fe6c3554cd1c75b937e55e285254af" -"checksum serde_codegen_internals 0.14.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bc888bd283bd2420b16ad0d860e35ad8acb21941180a83a189bb2046f9d00400" -"checksum serde_derive 0.9.15 (registry+https://github.com/rust-lang/crates.io-index)" = "978fd866f4d4872084a81ccc35e275158351d3b9fe620074e7d7504b816b74ba" -"checksum serde_json 0.9.10 (registry+https://github.com/rust-lang/crates.io-index)" = "ad8bcf487be7d2e15d3d543f04312de991d631cfe1b43ea0ade69e6a8a5b16a1" +"checksum serde 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "db99f3919e20faa51bb2996057f5031d8685019b5a06139b1ce761da671b8526" +"checksum serde_derive 1.0.27 (registry+https://github.com/rust-lang/crates.io-index)" = "f4ba7591cfe93755e89eeecdbcc668885624829b020050e6aec99c2a03bd3fd0" +"checksum serde_derive_internals 0.19.0 (registry+https://github.com/rust-lang/crates.io-index)" = "6e03f1c9530c3fb0a0a5c9b826bdd9246a5921ae995d75f512ac917fc4dd55b5" +"checksum serde_json 1.0.9 (registry+https://github.com/rust-lang/crates.io-index)" = "c9db7266c7d63a4c4b7fe8719656ccdd51acf1bed6124b174f933b009fb10bcb" "checksum strsim 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b4d15c810519a91cf877e7e36e63fe068815c678181439f2f29e2562147c3694" "checksum syn 0.11.11 (registry+https://github.com/rust-lang/crates.io-index)" = "d3b891b9015c88c576343b9b3e41c2c11a51c219ef067b264bd9c8aa9b441dad" "checksum synom 0.11.3 (registry+https://github.com/rust-lang/crates.io-index)" = "a393066ed9010ebaed60b9eafa373d4b1baac186dd7e008555b0f702b51945b6" "checksum tempdir 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "87974a6f5c1dfb344d733055601650059a3363de2a6104819293baff662132d6" "checksum termion 1.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "689a3bdfaab439fd92bc87df5c4c78417d3cbe537487274e9b0b2dce76e92096" "checksum textwrap 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "c0b59b6b4b44d867f1370ef1bd91bfb262bf07bf0ae65c202ea2fbc16153b693" -"checksum toml 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "bd86ad9ebee246fdedd610e0f6d0587b754a3d81438db930a244d0480ed7878f" +"checksum toml 0.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "a7540f4ffc193e0d3c94121edb19b055670d369f77d5804db11ae053a45b6e7e" "checksum unicode-width 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "bf3a113775714a22dcb774d8ea3655c53a32debae63a063acc00a91cc586245f" "checksum unicode-xid 0.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "8c1f860d7d29cf02cb2f3f359fd35991af3d30bac52c57d265a3c461074cb4dc" "checksum untrusted 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f392d7819dbe58833e26872f5f6f0d68b7bbbe90fc3667e98731c4a15ad9a7ae" "checksum vec_map 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)" = "887b5b631c2ad01628bbbaa7dd4c869f80d3186688f8d0b6f58774fbe324988c" -"checksum vobject 0.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "9ce763ee7201cb2915eb28450a2a1361e6fdc1650ff2ed806aa3d3216df92141" +"checksum vobject 0.4.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6041995691036270fabeb41975ca858f3b5113b82eea19a4f276bfb8b32e9ae4" +"checksum void 1.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" "checksum winapi 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)" = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +"checksum winapi 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "b09fb3b6f248ea4cd42c9a65113a847d612e17505d6ebd1f7357ad68a8bf8693" "checksum winapi-build 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" +"checksum winapi-i686-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "ec6667f60c23eca65c561e63a13d81b44234c2e38a6b6c959025ee907ec614cc" +"checksum winapi-x86_64-pc-windows-gnu 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "98f12c52b2630cd05d2c3ffd8e008f7f48252c042b4871c72aed9dc733b96668" diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 439fb97..fbfe57a 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -9,8 +9,10 @@ name = "vdirsyncer_rustext" crate-type = ["cdylib"] [dependencies] -vobject = "0.4.0" +vobject = "0.4.2" ring = "0.12.1" +error-chain = "0.11.0" +atomicwrites = "0.1.4" [build-dependencies] -cbindgen = "0.1" +cbindgen = "0.4" diff --git a/rust/build.rs b/rust/build.rs index 1c00f4c..143c8d1 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -1,12 +1,161 @@ extern crate cbindgen; use std::env; +use std::fs::{remove_file, File}; +use std::io::Write; +use std::path::Path; + +const TEMPLATE_EACH: &'static str = r#" +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_list( + storage: *mut {path}, + err: *mut VdirsyncerError +) -> *mut VdirsyncerStorageListing { + match (*storage).list() { + Ok(x) => Box::into_raw(Box::new(VdirsyncerStorageListing { + iterator: x, + href: None, + etag: None + })), + Err(e) => { + e.fill_c_err(err); + mem::zeroed() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_get( + storage: *mut {path}, + c_href: *const c_char, + err: *mut VdirsyncerError +) -> *mut VdirsyncerStorageGetResult { + let href = CStr::from_ptr(c_href); + match (*storage).get(href.to_str().unwrap()) { + Ok((item, href)) => { + Box::into_raw(Box::new(VdirsyncerStorageGetResult { + item: Box::into_raw(Box::new(item)), + etag: CString::new(href).unwrap().into_raw() + })) + }, + Err(e) => { + e.fill_c_err(err); + mem::zeroed() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_upload( + storage: *mut {path}, + item: *mut Item, + err: *mut VdirsyncerError +) -> *mut VdirsyncerStorageUploadResult { + match (*storage).upload((*item).clone()) { + Ok((href, etag)) => { + Box::into_raw(Box::new(VdirsyncerStorageUploadResult { + href: CString::new(href).unwrap().into_raw(), + etag: CString::new(etag).unwrap().into_raw() + })) + }, + Err(e) => { + e.fill_c_err(err); + mem::zeroed() + } + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_update( + storage: *mut {path}, + c_href: *const c_char, + item: *mut Item, + c_etag: *const c_char, + err: *mut VdirsyncerError +) -> *const c_char { + let href = CStr::from_ptr(c_href); + let etag = CStr::from_ptr(c_etag); + match (*storage).update(href.to_str().unwrap(), (*item).clone(), etag.to_str().unwrap()) { + Ok(etag) => CString::new(etag).unwrap().into_raw(), + Err(e) => { + e.fill_c_err(err); + mem::zeroed() + } + + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_delete( + storage: *mut {path}, + c_href: *const c_char, + c_etag: *const c_char, + err: *mut VdirsyncerError +) { + let href = CStr::from_ptr(c_href); + let etag = CStr::from_ptr(c_etag); + match (*storage).delete(href.to_str().unwrap(), etag.to_str().unwrap()) { + Ok(()) => (), + Err(e) => e.fill_c_err(err) + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_buffered(storage: *mut {path}) { + (*storage).buffered(); +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_{name}_flush( + storage: *mut {path}, + err: *mut VdirsyncerError +) { + match (*storage).flush() { + Ok(_) => (), + Err(e) => e.fill_c_err(err) + } +} +"#; + +fn export_storage(f: &mut File, name: &str, path: &str) { + // String formatting in rust is at compile time. That doesn't work well for our case. + write!( + f, + "{}", + TEMPLATE_EACH + .replace("{name}", name) + .replace("{path}", path) + ).unwrap(); +} fn main() { let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap(); - let mut config: cbindgen::Config = Default::default(); - config.language = cbindgen::Language::C; - cbindgen::generate_with_config(&crate_dir, config) - .unwrap() - .write_to_file("target/vdirsyncer_rustext.h"); + + let mut f = File::create(Path::new(&crate_dir).join("src/storage/exports.rs")).unwrap(); + write!(f, "// Auto-generated, do not check in.\n").unwrap(); + write!(f, "use std::os::raw::c_char;\n").unwrap(); + write!(f, "use std::mem;\n").unwrap(); + write!(f, "use std::ffi::{{CStr, CString}};\n").unwrap(); + write!(f, "use errors::*;\n").unwrap(); + write!(f, "use item::Item;\n").unwrap(); + write!(f, "use super::VdirsyncerStorageListing;\n").unwrap(); + write!(f, "use super::VdirsyncerStorageGetResult;\n").unwrap(); + write!(f, "use super::VdirsyncerStorageUploadResult;\n").unwrap(); + write!(f, "use super::Storage;\n").unwrap(); + + write!(f, "use super::singlefile;\n").unwrap(); + export_storage(&mut f, "singlefile", "singlefile::SinglefileStorage"); + drop(f); + + let _ = remove_file(Path::new(&crate_dir).join("target/vdirsyncer_rustext.h")); + + let res = cbindgen::Builder::new() + .with_crate(crate_dir) + .with_language(cbindgen::Language::C) + .generate(); + + match res { + Ok(x) => x.write_to_file("target/vdirsyncer_rustext.h"), + Err(e) => println!("FAILED TO GENERATE BINDINGS: {:?}", e), + } } diff --git a/rust/src/item.rs b/rust/src/item.rs index d2308e6..99697c1 100644 --- a/rust/src/item.rs +++ b/rust/src/item.rs @@ -1,69 +1,111 @@ -use std::ffi::{CStr, CString}; -use std::os::raw::c_char; -use std::mem; -use std::ptr; - use vobject; use ring; use std::fmt::Write; -use VdirsyncerError; +use errors::*; -const EMPTY_STRING: *const c_char = b"\0" as *const u8 as *const c_char; - -pub struct VdirsyncerComponent(vobject::Component); - -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_get_uid(c: *mut VdirsyncerComponent) -> *const c_char { - match safe_get_uid(&(*c).0) { - Some(x) => CString::new(x).unwrap().into_raw(), - None => EMPTY_STRING - } +#[derive(Clone)] +pub enum Item { + Parsed(vobject::Component), + Unparseable(String), // FIXME: maybe use https://crates.io/crates/terminated } -#[inline] -fn safe_get_uid(c: &vobject::Component) -> Option { - let mut stack = vec![c]; - - while let Some(vobj) = stack.pop() { - if let Some(prop) = vobj.get_only("UID") { - return Some(prop.value_as_string()); - } - stack.extend(vobj.subcomponents.iter()); - }; - None -} - - -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_parse_component(s: *const c_char, err: *mut VdirsyncerError) -> *mut VdirsyncerComponent { - let cstring = CStr::from_ptr(s); - match vobject::parse_component(cstring.to_str().unwrap()) { - Ok(x) => mem::transmute(Box::new(VdirsyncerComponent(x))), - Err(e) => { - (*err).failed = true; - (*err).msg = CString::new(e.description()).unwrap().into_raw(); - mem::zeroed() +impl Item { + pub fn from_raw(raw: String) -> Self { + match vobject::parse_component(&raw) { + Ok(x) => Item::Parsed(x), + // Don't chain vobject error here because it cannot be stored/cloned FIXME + _ => Item::Unparseable(raw), } } -} -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_free_component(c: *mut VdirsyncerComponent) { - let _: Box = mem::transmute(c); -} + pub fn from_component(component: vobject::Component) -> Self { + Item::Parsed(component) + } -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_clear_err(e: *mut VdirsyncerError) { - CString::from_raw((*e).msg); - (*e).msg = ptr::null_mut(); -} + /// Global identifier of the item, across storages, doesn't change after a modification of the + /// item. + pub fn get_uid(&self) -> Option { + // FIXME: Cache + if let Item::Parsed(ref c) = *self { + let mut stack: Vec<&vobject::Component> = vec![c]; -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_change_uid(c: *mut VdirsyncerComponent, uid: *const c_char) { - let uid_cstring = CStr::from_ptr(uid); - change_uid(&mut (*c).0, uid_cstring.to_str().unwrap()); + while let Some(vobj) = stack.pop() { + if let Some(prop) = vobj.get_only("UID") { + return Some(prop.value_as_string()); + } + stack.extend(vobj.subcomponents.iter()); + } + } + None + } + + pub fn with_uid(&self, uid: &str) -> Result { + if let Item::Parsed(ref component) = *self { + let mut new_component = component.clone(); + change_uid(&mut new_component, uid); + Ok(Item::from_raw(vobject::write_component(&new_component))) + } else { + Err(ErrorKind::ItemUnparseable.into()) + } + } + + /// Raw unvalidated content of the item + pub fn get_raw(&self) -> String { + match *self { + Item::Parsed(ref component) => vobject::write_component(component), + Item::Unparseable(ref x) => x.to_owned(), + } + } + + /// Component of item if parseable + pub fn get_component(&self) -> Result<&vobject::Component> { + match *self { + Item::Parsed(ref component) => Ok(component), + _ => Err(ErrorKind::ItemUnparseable.into()), + } + } + + /// Component of item if parseable + pub fn into_component(self) -> Result { + match self { + Item::Parsed(component) => Ok(component), + _ => Err(ErrorKind::ItemUnparseable.into()), + } + } + + /// Used for etags + pub fn get_hash(&self) -> Result { + // FIXME: cache + if let Item::Parsed(ref component) = *self { + Ok(hash_component(component)) + } else { + Err(ErrorKind::ItemUnparseable.into()) + } + } + + /// Used for generating hrefs and matching up items during synchronization. This is either the + /// UID or the hash of the item's content. + pub fn get_ident(&self) -> Result { + if let Some(x) = self.get_uid() { + return Ok(x); + } + // We hash the item instead of directly using its raw content, because + // 1. The raw content might be really large, e.g. when it's a contact + // with a picture, which bloats the status file. + // + // 2. The status file would contain really sensitive information. + self.get_hash() + } + + pub fn is_parseable(&self) -> bool { + if let Item::Parsed(_) = *self { + true + } else { + false + } + } } fn change_uid(c: &mut vobject::Component, uid: &str) { @@ -76,30 +118,15 @@ fn change_uid(c: &mut vobject::Component, uid: &str) { } else { component.remove("UID"); } - }, - _ => () + } + _ => (), } stack.extend(component.subcomponents.iter_mut()); } } -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_clone_component(c: *mut VdirsyncerComponent) -> *mut VdirsyncerComponent { - mem::transmute(Box::new((*c).0.clone())) -} - -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_write_component(c: *mut VdirsyncerComponent) -> *const c_char { - CString::new(vobject::write_component(&(*c).0)).unwrap().into_raw() -} - -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_hash_component(c: *mut VdirsyncerComponent) -> *const c_char { - CString::new(safe_hash_component(&(*c).0)).unwrap().into_raw() -} - -fn safe_hash_component(c: &vobject::Component) -> String { +fn hash_component(c: &vobject::Component) -> String { let mut new_c = c.clone(); { let mut stack = vec![&mut new_c]; @@ -110,9 +137,6 @@ fn safe_hash_component(c: &vobject::Component) -> String { component.remove("METHOD"); // X-RADICALE-NAME is used by radicale, because hrefs don't really exist in their filesystem backend component.remove("X-RADICALE-NAME"); - // Apparently this is set by Horde? - // https://github.com/pimutils/vdirsyncer/issues/318 - component.remove("X-WR-CALNAME"); // Those are from the VCARD specification and is supposed to change when the // item does -- however, we can determine that ourselves component.remove("REV"); @@ -128,7 +152,22 @@ fn safe_hash_component(c: &vobject::Component) -> String { component.remove("UID"); if component.name == "VCALENDAR" { - component.subcomponents.retain(|ref c| c.name != "VTIMEZONE"); + // CALSCALE's default value is gregorian + let calscale = component.get_only("CALSCALE").map(|x| x.value_as_string()); + + if let Some(x) = calscale { + if x == "GREGORIAN" { + component.remove("CALSCALE"); + } + } + + // Apparently this is set by Horde? + // https://github.com/pimutils/vdirsyncer/issues/318 + // Also Google sets those properties + component.remove("X-WR-CALNAME"); + component.remove("X-WR-TIMEZONE"); + + component.subcomponents.retain(|c| c.name != "VTIMEZONE"); } stack.extend(component.subcomponents.iter_mut()); @@ -136,10 +175,84 @@ fn safe_hash_component(c: &vobject::Component) -> String { } // FIXME: Possible optimization: Stream component to hasher instead of allocating new string - let digest = ring::digest::digest(&ring::digest::SHA256, vobject::write_component(&new_c).as_bytes()); + let raw = vobject::write_component(&new_c); + let mut lines: Vec<_> = raw.lines().collect(); + lines.sort(); + let digest = ring::digest::digest(&ring::digest::SHA256, lines.join("\r\n").as_bytes()); let mut rv = String::new(); for &byte in digest.as_ref() { write!(&mut rv, "{:x}", byte).unwrap(); } rv } + +pub mod exports { + use super::Item; + use std::mem; + use std::ffi::{CStr, CString}; + use std::os::raw::c_char; + use errors::*; + + const EMPTY_STRING: *const c_char = b"\0" as *const u8 as *const c_char; + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_get_uid(c: *mut Item) -> *const c_char { + match (*c).get_uid() { + Some(x) => CString::new(x).unwrap().into_raw(), + None => EMPTY_STRING, + } + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_get_raw(c: *mut Item) -> *const c_char { + CString::new((*c).get_raw()).unwrap().into_raw() + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_item_from_raw(s: *const c_char) -> *mut Item { + let cstring = CStr::from_ptr(s); + Box::into_raw(Box::new(Item::from_raw( + cstring.to_str().unwrap().to_owned(), + ))) + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_free_item(c: *mut Item) { + let _: Box = Box::from_raw(c); + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_with_uid( + c: *mut Item, + uid: *const c_char, + err: *mut VdirsyncerError, + ) -> *mut Item { + let uid_cstring = CStr::from_ptr(uid); + match (*c).with_uid(uid_cstring.to_str().unwrap()) { + Ok(x) => Box::into_raw(Box::new(x)), + Err(e) => { + e.fill_c_err(err); + mem::zeroed() + } + } + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_get_hash( + c: *mut Item, + err: *mut VdirsyncerError, + ) -> *const c_char { + match (*c).get_hash() { + Ok(x) => CString::new(x).unwrap().into_raw(), + Err(e) => { + e.fill_c_err(err); + mem::zeroed() + } + } + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_item_is_parseable(c: *mut Item) -> bool { + (*c).is_parseable() + } +} diff --git a/rust/src/lib.rs b/rust/src/lib.rs index 417b5ad..dcd30ea 100644 --- a/rust/src/lib.rs +++ b/rust/src/lib.rs @@ -1,19 +1,104 @@ -extern crate vobject; +extern crate atomicwrites; +#[macro_use] +extern crate error_chain; extern crate ring; - -use std::ffi::CStr; -use std::os::raw::c_char; +extern crate vobject; pub mod item; +pub mod storage; -#[repr(C)] -pub struct VdirsyncerError { - pub failed: bool, - pub msg: *mut c_char, +mod errors { + use std::ffi::CString; + use std::os::raw::c_char; + use vobject; + use atomicwrites; + + error_chain!{ + links { + Vobject(vobject::error::VObjectError, vobject::error::VObjectErrorKind); + } + + foreign_links { + Io(::std::io::Error); + } + + errors { + ItemUnparseable { + description("ItemUnparseable: The item cannot be parsed."), + display("The item cannot be parsed."), + } + + VobjectVersionMismatch(first: String, second: String) { + description("Incompatible vobject versions."), + display("Conflict between {} and {}", first, second), + } + + UnexpectedVobject(found: String, expected: String) { + description("Unexpected component type"), + display("Found type {}, expected {}", found, expected), + } + + ItemNotFound(href: String) { + description("ItemNotFound: The item could not be found"), + display("The item '{}' could not be found", href), + } + + AlreadyExisting(href: String) { + description("AlreadyExisting: An item at this href already exists"), + display("The href '{}' is already taken", href), + } + + WrongEtag(href: String) { + description("WrongEtag: A wrong etag was provided."), + display("A wrong etag for '{}' was provided. This indicates that two clients are writing data at the same time.", href), + } + + MtimeMismatch(filepath: String) { + description("MtimeMismatch: Two programs access the same file."), + display("The mtime of {} has unexpectedly changed. Please close other programs accessing this file.", filepath), + } + } + } + + impl From> for Error { + fn from(e: atomicwrites::Error) -> Error { + match e { + atomicwrites::Error::Internal(x) => x.into(), + atomicwrites::Error::User(x) => x, + } + } + } + + pub trait ErrorExt: ::std::error::Error { + unsafe fn fill_c_err(&self, err: *mut VdirsyncerError) { + (*err).failed = true; + (*err).msg = CString::new(self.description()).unwrap().into_raw(); + } + } + + impl ErrorExt for Error {} + + #[repr(C)] + pub struct VdirsyncerError { + pub failed: bool, + pub msg: *mut c_char, + } } +pub mod exports { + use std::ffi::{CStr, CString}; + use std::ptr; + use std::os::raw::c_char; + use errors::*; -#[no_mangle] -pub unsafe extern "C" fn vdirsyncer_free_str(s: *const c_char) { - CStr::from_ptr(s); + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_free_str(s: *const c_char) { + CStr::from_ptr(s); + } + + #[no_mangle] + pub unsafe extern "C" fn vdirsyncer_clear_err(e: *mut VdirsyncerError) { + CString::from_raw((*e).msg); + (*e).msg = ptr::null_mut(); + } } diff --git a/rust/src/storage/mod.rs b/rust/src/storage/mod.rs new file mode 100644 index 0000000..bbe3b3d --- /dev/null +++ b/rust/src/storage/mod.rs @@ -0,0 +1,174 @@ +pub mod singlefile; +pub mod exports; +use std::ffi::CString; +use std::os::raw::c_char; +use errors::*; +use item::Item; + +type ItemAndEtag = (Item, String); + +pub trait Storage: Sized { + /// returns an iterator of `(href, etag)` + fn list<'a>(&'a mut self) -> Result + 'a>>; + + ///Fetch a single item. + /// + ///:param href: href to fetch + ///:returns: (item, etag) + ///:raises: :exc:`vdirsyncer.exceptions.PreconditionFailed` if item can't be found. + fn get(&mut self, href: &str) -> Result; + + /// Fetch multiple items. Duplicate hrefs must be ignored. + /// + /// Functionally similar to `get`, but might bring performance benefits on some storages when + /// used cleverly. + /// + /// # Parameters + /// - `hrefs`: list of hrefs to fetch + /// - returns an iterator of `(href, item, etag)` + fn get_multi<'a, I: Iterator + 'a>( + &'a mut self, + hrefs: I, + ) -> Box)> + 'a> { + Box::new(DefaultGetMultiIterator { + storage: self, + href_iter: hrefs, + }) + } + + /// Upload a new item. + /// + /// In cases where the new etag cannot be atomically determined (i.e. in the same + /// "transaction" as the upload itself), this method may return `None` as etag. This + /// special case only exists because of DAV. Avoid this situation whenever possible. + /// + /// Returns `(href, etag)` + fn upload(&mut self, item: Item) -> Result<(String, String)>; + + /// Update an item. + /// + /// The etag may be none in some cases, see `upload`. + /// + /// Returns `etag` + fn update(&mut self, href: &str, item: Item, etag: &str) -> Result; + + /// Delete an item by href. + fn delete(&mut self, href: &str, etag: &str) -> Result<()>; + + /// Enter buffered mode for storages that support it. + /// + /// Uploads, updates and deletions may not be effective until `flush` is explicitly called. + /// + /// Use this if you will potentially write a lot of data to the storage, it improves + /// performance for storages that implement it. + fn buffered(&mut self) {} + + /// Write back all changes to the collection. + fn flush(&mut self) -> Result<()> { + Ok(()) + } +} + +struct DefaultGetMultiIterator<'a, S: Storage + 'a, I: Iterator> { + storage: &'a mut S, + href_iter: I, +} + +impl<'a, S, I> Iterator for DefaultGetMultiIterator<'a, S, I> +where + S: Storage, + I: Iterator, +{ + type Item = (String, Result); + + fn next(&mut self) -> Option { + match self.href_iter.next() { + Some(x) => Some((x.to_owned(), self.storage.get(&x))), + None => None, + } + } +} + +pub struct VdirsyncerStorageListing { + iterator: Box>, + href: Option, + etag: Option, +} + +impl VdirsyncerStorageListing { + pub fn advance(&mut self) -> bool { + match self.iterator.next() { + Some((href, etag)) => { + self.href = Some(href); + self.etag = Some(etag); + true + } + None => { + self.href = None; + self.etag = None; + false + } + } + } + + pub fn get_href(&mut self) -> Option { + self.href.take() + } + pub fn get_etag(&mut self) -> Option { + self.etag.take() + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_free_storage_listing(listing: *mut VdirsyncerStorageListing) { + let _: Box = Box::from_raw(listing); +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_advance_storage_listing( + listing: *mut VdirsyncerStorageListing, +) -> bool { + (*listing).advance() +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_storage_listing_get_href( + listing: *mut VdirsyncerStorageListing, +) -> *const c_char { + CString::new((*listing).get_href().unwrap()) + .unwrap() + .into_raw() +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_storage_listing_get_etag( + listing: *mut VdirsyncerStorageListing, +) -> *const c_char { + CString::new((*listing).get_etag().unwrap()) + .unwrap() + .into_raw() +} + +#[repr(C)] +pub struct VdirsyncerStorageGetResult { + pub item: *mut Item, + pub etag: *const c_char, +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_free_storage_get_result(res: *mut VdirsyncerStorageGetResult) { + let _: Box = Box::from_raw(res); +} + +#[repr(C)] +pub struct VdirsyncerStorageUploadResult { + pub href: *const c_char, + pub etag: *const c_char, +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_free_storage_upload_result( + res: *mut VdirsyncerStorageUploadResult, +) { + let _: Box = Box::from_raw(res); +} diff --git a/rust/src/storage/singlefile.rs b/rust/src/storage/singlefile.rs new file mode 100644 index 0000000..a421c6a --- /dev/null +++ b/rust/src/storage/singlefile.rs @@ -0,0 +1,349 @@ +use std::ffi::CStr; +use std::os::raw::c_char; +use std::path::{Path, PathBuf}; +use std::collections::{BTreeMap, BTreeSet}; +use std::collections::btree_map::Entry::*; +use std::fs::{metadata, File}; +use std::io::{Read, Write}; +use std::time::SystemTime; +use super::Storage; +use errors::*; +use vobject; + +use atomicwrites::{AllowOverwrite, AtomicFile}; + +use item::Item; + +type ItemCache = BTreeMap; + +pub struct SinglefileStorage { + path: PathBuf, + // href -> (item, etag) + items_cache: Option<(ItemCache, SystemTime)>, + buffered_mode: bool, + dirty_cache: bool, +} + +impl SinglefileStorage { + pub fn new>(path: P) -> Self { + SinglefileStorage { + path: path.as_ref().to_owned(), + items_cache: None, + buffered_mode: false, + dirty_cache: false, + } + } + + fn get_items(&mut self) -> Result<&mut ItemCache> { + if self.items_cache.is_none() { + self.list()?; + } + Ok(&mut self.items_cache.as_mut().unwrap().0) + } + + fn write_back(&mut self) -> Result<()> { + self.dirty_cache = true; + if self.buffered_mode { + return Ok(()); + } + + self.flush()?; + Ok(()) + } +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_init_singlefile(path: *const c_char) -> *mut SinglefileStorage { + let cstring = CStr::from_ptr(path); + Box::into_raw(Box::new(SinglefileStorage::new(cstring.to_str().unwrap()))) +} + +#[no_mangle] +pub unsafe extern "C" fn vdirsyncer_free_singlefile(s: *mut SinglefileStorage) { + let _: Box = Box::from_raw(s); +} + +impl Storage for SinglefileStorage { + fn list<'a>(&'a mut self) -> Result + 'a>> { + let mut new_cache = BTreeMap::new(); + let mtime = metadata(&self.path)?.modified()?; + let mut f = File::open(&self.path)?; + let mut s = String::new(); + f.read_to_string(&mut s)?; + for component in split_collection(&s)? { + let item = Item::from_component(component); + let hash = item.get_hash()?; + let ident = item.get_ident()?; + new_cache.insert(ident, (item, hash)); + } + + self.items_cache = Some((new_cache, mtime)); + self.dirty_cache = false; + Ok(Box::new(self.items_cache.as_ref().unwrap().0.iter().map( + |(href, &(_, ref etag))| (href.clone(), etag.clone()), + ))) + } + + fn get(&mut self, href: &str) -> Result<(Item, String)> { + match self.get_items()?.get(href) { + Some(&(ref href, ref etag)) => Ok((href.clone(), etag.clone())), + None => Err(ErrorKind::ItemNotFound(href.to_owned()))?, + } + } + + fn upload(&mut self, item: Item) -> Result<(String, String)> { + let hash = item.get_hash()?; + let href = item.get_ident()?; + match self.get_items()?.entry(href.clone()) { + Occupied(_) => Err(ErrorKind::AlreadyExisting(href.clone()))?, + Vacant(vc) => vc.insert((item, hash.clone())), + }; + self.write_back()?; + Ok((href, hash)) + } + + fn update(&mut self, href: &str, item: Item, etag: &str) -> Result { + let hash = match self.get_items()?.entry(href.to_owned()) { + Occupied(mut oc) => { + if oc.get().1 == etag { + let hash = item.get_hash()?; + oc.insert((item, hash.clone())); + Ok(hash) + } else { + Err(ErrorKind::WrongEtag(href.to_owned())) + } + } + Vacant(_) => Err(ErrorKind::ItemNotFound(href.to_owned())), + }?; + self.write_back()?; + Ok(hash) + } + + fn delete(&mut self, href: &str, etag: &str) -> Result<()> { + match self.get_items()?.entry(href.to_owned()) { + Occupied(oc) => { + if oc.get().1 == etag { + oc.remove(); + } else { + Err(ErrorKind::WrongEtag(href.to_owned()))? + } + } + Vacant(_) => Err(ErrorKind::ItemNotFound(href.to_owned()))?, + } + self.write_back()?; + Ok(()) + } + + fn buffered(&mut self) { + self.buffered_mode = true; + } + + fn flush(&mut self) -> Result<()> { + if !self.dirty_cache { + return Ok(()); + } + let (items, mtime) = self.items_cache.take().unwrap(); + + let af = AtomicFile::new(&self.path, AllowOverwrite); + let content = join_collection(items.into_iter().map(|(_, (item, _))| item))?; + af.write::<(), Error, _>(|f| { + f.write_all(content.as_bytes())?; + + let real_mtime = metadata(&self.path)?.modified()?; + if mtime != real_mtime { + Err(ErrorKind::MtimeMismatch( + self.path.to_string_lossy().into_owned(), + ))?; + } + Ok(()) + })?; + + self.dirty_cache = false; + + Ok(()) + } +} + +fn split_collection(mut input: &str) -> Result> { + let mut rv = vec![]; + while !input.is_empty() { + let (component, remainder) = vobject::read_component(input)?; + input = remainder; + + match component.name.as_ref() { + "VCALENDAR" => rv.extend(split_vcalendar(component)?), + "VCARD" => rv.push(component), + "VADDRESSBOOK" => for vcard in component.subcomponents { + if vcard.name != "VCARD" { + Err(ErrorKind::UnexpectedVobject( + vcard.name.clone(), + "VCARD".to_owned(), + ))?; + } + rv.push(vcard); + }, + _ => Err(ErrorKind::UnexpectedVobject( + component.name.clone(), + "VCALENDAR | VCARD | VADDRESSBOOK".to_owned(), + ))?, + } + } + + Ok(rv) +} + +/// Split one VCALENDAR component into multiple VCALENDAR components +#[inline] +fn split_vcalendar(mut vcalendar: vobject::Component) -> Result> { + vcalendar.props.remove("METHOD"); + + let mut timezones = BTreeMap::new(); // tzid => component + let mut subcomponents = vec![]; + + for component in vcalendar.subcomponents.drain(..) { + match component.name.as_ref() { + "VTIMEZONE" => { + let tzid = match component.get_only("TZID") { + Some(x) => x.value_as_string().clone(), + None => continue, + }; + timezones.insert(tzid, component); + } + "VTODO" | "VEVENT" | "VJOURNAL" => subcomponents.push(component), + _ => Err(ErrorKind::UnexpectedVobject( + component.name.clone(), + "VTIMEZONE | VTODO | VEVENT | VJOURNAL".to_owned(), + ))?, + }; + } + + let mut by_uid = BTreeMap::new(); + let mut no_uid = vec![]; + + for component in subcomponents { + let uid = component.get_only("UID").cloned(); + + let mut wrapper = match uid.as_ref() + .and_then(|u| by_uid.remove(&u.value_as_string())) + { + Some(x) => x, + None => vcalendar.clone(), + }; + + let mut required_tzids = BTreeSet::new(); + for props in component.props.values() { + for prop in props { + if let Some(x) = prop.params.get("TZID") { + required_tzids.insert(x.to_owned()); + } + } + } + + for tzid in required_tzids { + if let Some(tz) = timezones.get(&tzid) { + wrapper.subcomponents.push(tz.clone()); + } + } + + wrapper.subcomponents.push(component); + + match uid { + Some(p) => { + by_uid.insert(p.value_as_string(), wrapper); + } + None => no_uid.push(wrapper), + } + } + + Ok(by_uid + .into_iter() + .map(|(_, v)| v) + .chain(no_uid.into_iter()) + .collect()) +} + +fn join_collection>(item_iter: I) -> Result { + let mut items = item_iter.peekable(); + + let item_name = match items.peek() { + Some(x) => x.get_component()?.name.clone(), + None => return Ok("".to_owned()), + }; + + let wrapper_name = match item_name.as_ref() { + "VCARD" => "VADDRESSBOOK", + "VCALENDAR" => "VCALENDAR", + _ => Err(ErrorKind::UnexpectedVobject( + item_name.clone(), + "VCARD | VCALENDAR".to_owned(), + ))?, + }; + + let mut wrapper = vobject::Component::new(wrapper_name); + let mut version: Option = None; + + for item in items { + let mut c = item.into_component()?; + if c.name != item_name { + return Err(ErrorKind::UnexpectedVobject(c.name, item_name.clone()).into()); + } + + if item_name == wrapper_name { + wrapper.subcomponents.extend(c.subcomponents.drain(..)); + match (version.as_ref(), c.get_only("VERSION")) { + (Some(x), Some(y)) if x.raw_value != y.raw_value => { + return Err(ErrorKind::VobjectVersionMismatch( + x.raw_value.clone(), + y.raw_value.clone(), + ).into()); + } + (None, Some(_)) => (), + _ => continue, + } + version = c.get_only("VERSION").cloned(); + } else { + wrapper.subcomponents.push(c); + } + } + + if let Some(v) = version { + wrapper.set(v); + } + + Ok(vobject::write_component(&wrapper)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn check_roundtrip(raw: &str) { + let components = split_collection(raw).unwrap(); + let raw2 = join_collection(components.into_iter().map(Item::from_component)).unwrap(); + assert_eq!( + Item::from_raw(raw.to_owned()).get_hash().unwrap(), + Item::from_raw(raw2.to_owned()).get_hash().unwrap() + ); + } + + #[test] + fn test_wrapper_properties_roundtrip() { + let raw = r#"BEGIN:VCALENDAR +PRODID:-//Google Inc//Google Calendar 70.9054//EN +X-WR-CALNAME:markus.unterwaditzer@runtastic.com +X-WR-TIMEZONE:Europe/Vienna +VERSION:2.0 +CALSCALE:GREGORIAN +BEGIN:VEVENT +DTSTART;TZID=Europe/Vienna:20171012T153000 +DTEND;TZID=Europe/Vienna:20171012T170000 +DTSTAMP:20171009T085029Z +UID:test@test.com +STATUS:CONFIRMED +SUMMARY:Test +TRANSP:OPAQUE +END:VEVENT +END:VCALENDAR"#; + check_roundtrip(raw); + } +} diff --git a/scripts/circleci-install.sh b/scripts/circleci-install.sh index 22db2b6..759039b 100644 --- a/scripts/circleci-install.sh +++ b/scripts/circleci-install.sh @@ -1,6 +1,8 @@ make install-rust echo "export PATH=$HOME/.cargo/bin/:$PATH" >> $BASH_ENV +sudo apt-get install -y cmake + pip install --user virtualenv ~/.local/bin/virtualenv ~/env echo ". ~/env/bin/activate" >> $BASH_ENV diff --git a/setup.py b/setup.py index 46fc83b..93aa988 100644 --- a/setup.py +++ b/setup.py @@ -52,7 +52,7 @@ def build_native(spec): 'vdirsyncer_rustext.h', in_path='rust/target'), # Rust bug: If thread-local storage is used, this flag is necessary # (mitsuhiko) - rtld_flags=['NODELETE'] + rtld_flags=['NOW', 'NODELETE'] ) diff --git a/tests/storage/__init__.py b/tests/storage/__init__.py index c12afcc..41a2899 100644 --- a/tests/storage/__init__.py +++ b/tests/storage/__init__.py @@ -344,3 +344,58 @@ class StorageTests(object): item2, etag2 = s.get(href) assert item2.raw.count('BEGIN:VEVENT') == 2 assert 'RRULE' in item2.raw + + def test_buffered(self, get_storage_args, get_item, requires_collections): + args = get_storage_args() + s1 = self.storage_class(**args) + s2 = self.storage_class(**args) + s1.upload(get_item()) + assert sorted(list(s1.list())) == sorted(list(s2.list())) + + s1.buffered() + s1.upload(get_item()) + s1.flush() + assert sorted(list(s1.list())) == sorted(list(s2.list())) + + def test_retain_timezones(self, item_type, s): + if item_type != 'VEVENT': + pytest.skip('This storage instance doesn\'t support iCalendar.') + + item = Item(textwrap.dedent(''' + BEGIN:VCALENDAR + PRODID:-//ownCloud calendar v1.4.0 + VERSION:2.0 + CALSCALE:GREGORIAN + BEGIN:VEVENT + CREATED:20161004T110533 + DTSTAMP:20161004T110533 + LAST-MODIFIED:20161004T110533 + UID:y2lmgz48mg + SUMMARY:Test + CLASS:PUBLIC + STATUS:CONFIRMED + DTSTART;TZID=Europe/Berlin:20161014T101500 + DTEND;TZID=Europe/Berlin:20161014T114500 + END:VEVENT + BEGIN:VTIMEZONE + TZID:Europe/Berlin + BEGIN:DAYLIGHT + DTSTART:20160327T030000 + TZNAME:CEST + TZOFFSETFROM:+0100 + TZOFFSETTO:+0200 + END:DAYLIGHT + BEGIN:STANDARD + DTSTART:20161030T020000 + TZNAME:CET + TZOFFSETFROM:+0200 + TZOFFSETTO:+0100 + END:STANDARD + END:VTIMEZONE + END:VCALENDAR + ''').strip()) + + href, etag = s.upload(item) + item2, _ = s.get(href) + assert 'VTIMEZONE' in item2.raw + assert item2.hash == item.hash diff --git a/tests/system/cli/test_sync.py b/tests/system/cli/test_sync.py index 8bdde4b..94bab84 100644 --- a/tests/system/cli/test_sync.py +++ b/tests/system/cli/test_sync.py @@ -40,7 +40,8 @@ def test_simple_run(tmpdir, runner): tmpdir.join('path_a/haha.txt').write(item.raw) result = runner.invoke(['sync']) assert 'Copying (uploading) item haha to my_b' in result.output - assert tmpdir.join('path_b/haha.txt').read() == item.raw + assert tmpdir.join('path_b/haha.txt').read().splitlines() == \ + item.raw.splitlines() def test_sync_inexistant_pair(tmpdir, runner): @@ -393,21 +394,16 @@ def test_no_configured_pairs(tmpdir, runner, cmd): assert result.exception.code == 5 -item_a = format_item('lol') -item_b = format_item('lol') +def test_conflict_resolution(tmpdir, runner): + item_a = format_item('lol') + item_b = format_item('lol') - -@pytest.mark.parametrize('resolution,expect_foo,expect_bar', [ - (['command', 'cp'], item_a.raw, item_a.raw) -]) -def test_conflict_resolution(tmpdir, runner, resolution, expect_foo, - expect_bar): runner.write_with_general(dedent(''' [pair foobar] a = "foo" b = "bar" collections = null - conflict_resolution = {val} + conflict_resolution = ["command", "cp"] [storage foo] type = "filesystem" @@ -418,7 +414,7 @@ def test_conflict_resolution(tmpdir, runner, resolution, expect_foo, type = "filesystem" fileext = ".txt" path = "{base}/bar" - '''.format(base=str(tmpdir), val=json.dumps(resolution)))) + '''.format(base=str(tmpdir)))) foo = tmpdir.join('foo') bar = tmpdir.join('bar') @@ -433,8 +429,8 @@ def test_conflict_resolution(tmpdir, runner, resolution, expect_foo, r = runner.invoke(['sync']) assert not r.exception - assert fooitem.read() == expect_foo - assert baritem.read() == expect_bar + assert fooitem.read().splitlines() == item_a.raw.splitlines() + assert baritem.read().splitlines() == item_a.raw.splitlines() @pytest.mark.parametrize('partial_sync', ['error', 'ignore', 'revert', None]) diff --git a/tests/unit/utils/test_vobject.py b/tests/unit/utils/test_vobject.py index 4660f2a..a0f4937 100644 --- a/tests/unit/utils/test_vobject.py +++ b/tests/unit/utils/test_vobject.py @@ -15,6 +15,17 @@ from tests import BARE_EVENT_TEMPLATE, EVENT_TEMPLATE, \ import vdirsyncer.vobject as vobject +@pytest.fixture +def check_roundtrip(benchmark): + def inner(split): + joined = benchmark(lambda: vobject.join_collection(split)) + split2 = benchmark(lambda: list(vobject.split_collection(joined))) + + assert [vobject.Item(item).hash for item in split] == \ + [vobject.Item(item).hash for item in split2] + return inner + + _simple_split = [ VCARD_TEMPLATE.format(r=123, uid=123), VCARD_TEMPLATE.format(r=345, uid=345), @@ -28,7 +39,9 @@ _simple_joined = u'\r\n'.join( ) -def test_split_collection_simple(benchmark): +def test_split_collection_simple(benchmark, check_roundtrip): + check_roundtrip(_simple_split) + given = benchmark(lambda: list(vobject.split_collection(_simple_joined))) assert [vobject.Item(item).hash for item in given] == \ @@ -46,6 +59,7 @@ def test_split_collection_multiple_wrappers(benchmark): for x in _simple_split ) given = benchmark(lambda: list(vobject.split_collection(joined))) + check_roundtrip(given) assert [vobject.Item(item).hash for item in given] == \ [vobject.Item(item).hash for item in _simple_split] @@ -395,3 +409,44 @@ def test_hash_item_timezones(): ) assert item1.hash == item2.hash + + +def test_hash_item_line_wrapping(): + item1 = vobject.Item( + 'BEGIN:VCALENDAR\r\n' + 'PROP:a\r\n' + ' b\r\n' + ' c\r\n' + 'END:VCALENDAR\r\n' + ) + + item2 = vobject.Item( + 'BEGIN:VCALENDAR\r\n' + 'PROP:abc\r\n' + 'END:VCALENDAR\r\n' + ) + + assert item1.hash == item2.hash + + +def test_wrapper_properties(check_roundtrip): + raws = [dedent(''' + BEGIN:VCALENDAR + PRODID:-//Google Inc//Google Calendar 70.9054//EN + VERSION:2.0 + CALSCALE:GREGORIAN + X-WR-CALNAME:hans.gans@gmail.com + X-WR-TIMEZONE:Europe/Vienna + BEGIN:VEVENT + DTSTART;TZID=Europe/Vienna:20171012T153000 + DTEND;TZID=Europe/Vienna:20171012T170000 + DTSTAMP:20171009T085029Z + UID:test@test.com + STATUS:CONFIRMED + SUMMARY:Test + TRANSP:OPAQUE + END:VEVENT + END:VCALENDAR + ''').strip()] + + check_roundtrip(raws) diff --git a/vdirsyncer/cli/utils.py b/vdirsyncer/cli/utils.py index 17b9e2c..617a9af 100644 --- a/vdirsyncer/cli/utils.py +++ b/vdirsyncer/cli/utils.py @@ -244,6 +244,9 @@ def save_status(base_path, pair, collection=None, data_type=None, data=None): def storage_class_from_config(config): config = dict(config) + if 'type' not in config: + raise exceptions.UserError('Missing parameter "type"') + storage_name = config.pop('type') try: cls = storage_names[storage_name] diff --git a/vdirsyncer/native.py b/vdirsyncer/native.py index cf4f5c1..22f88a8 100644 --- a/vdirsyncer/native.py +++ b/vdirsyncer/native.py @@ -1,45 +1,32 @@ from ._native import ffi, lib -from .exceptions import VobjectParseError +from . import exceptions -def parse_component(raw): - e = ffi.new('VdirsyncerError *') - try: - c = lib.vdirsyncer_parse_component(raw, e) - if e.failed: - raise VobjectParseError(ffi.string(e.msg).decode('utf-8')) - return _component_rv(c) - finally: - if e.failed: - lib.vdirsyncer_clear_err(e) - - -def write_component(component): - return _string_rv(lib.vdirsyncer_write_component(component)) - - -def get_uid(component): - return _string_rv(lib.vdirsyncer_get_uid(component)) - - -def _string_rv(c_str): +def string_rv(c_str): try: return ffi.string(c_str).decode('utf-8') finally: lib.vdirsyncer_free_str(c_str) -def change_uid(component, uid): - lib.vdirsyncer_change_uid(component, uid.encode('utf-8')) +def item_rv(c): + return ffi.gc(c, lib.vdirsyncer_free_item) -def _component_rv(c): - return ffi.gc(c, lib.vdirsyncer_free_component) - - -def clone_component(c): - return _component_rv(lib.vdirsyncer_clone_component(c)) - - -def hash_component(c): - return _string_rv(lib.vdirsyncer_hash_component(c)) +def check_error(e): + try: + if e.failed: + msg = ffi.string(e.msg).decode('utf-8') + if msg.startswith('ItemNotFound'): + raise exceptions.NotFoundError(msg) + elif msg.startswith('AlreadyExisting'): + raise exceptions.AlreadyExistingError(msg) + elif msg.startswith('WrongEtag'): + raise exceptions.WrongEtagError(msg) + elif msg.startswith('ItemUnparseable'): + raise ValueError(msg) + else: + raise Exception(msg) + finally: + if e.failed: + lib.vdirsyncer_clear_err(e) diff --git a/vdirsyncer/storage/_rust.py b/vdirsyncer/storage/_rust.py new file mode 100644 index 0000000..87732aa --- /dev/null +++ b/vdirsyncer/storage/_rust.py @@ -0,0 +1,73 @@ +from .. import native +from ..vobject import Item +from functools import partial + + +class RustStorageMixin: + _native_storage = None + + def _native(self, name): + return partial( + getattr(native.lib, + 'vdirsyncer_{}_{}'.format(self.storage_name, name)), + self._native_storage + ) + + def list(self): + e = native.ffi.new('VdirsyncerError *') + listing = self._native('list')(e) + native.check_error(e) + listing = native.ffi.gc(listing, + native.lib.vdirsyncer_free_storage_listing) + while native.lib.vdirsyncer_advance_storage_listing(listing): + href = native.string_rv( + native.lib.vdirsyncer_storage_listing_get_href(listing)) + etag = native.string_rv( + native.lib.vdirsyncer_storage_listing_get_etag(listing)) + yield href, etag + + def get(self, href): + href = href.encode('utf-8') + e = native.ffi.new('VdirsyncerError *') + result = self._native('get')(href, e) + native.check_error(e) + result = native.ffi.gc(result, + native.lib.vdirsyncer_free_storage_get_result) + item = native.item_rv(result.item) + etag = native.string_rv(result.etag) + return Item(None, _native=item), etag + + # FIXME: implement get_multi + + def upload(self, item): + e = native.ffi.new('VdirsyncerError *') + result = self._native('upload')(item._native, e) + native.check_error(e) + result = native.ffi.gc( + result, native.lib.vdirsyncer_free_storage_upload_result) + href = native.string_rv(result.href) + etag = native.string_rv(result.etag) + return href, etag + + def update(self, href, item, etag): + href = href.encode('utf-8') + etag = etag.encode('utf-8') + e = native.ffi.new('VdirsyncerError *') + etag = self._native('update')(href, item._native, etag, e) + native.check_error(e) + return native.string_rv(etag) + + def delete(self, href, etag): + href = href.encode('utf-8') + etag = etag.encode('utf-8') + e = native.ffi.new('VdirsyncerError *') + self._native('delete')(href, etag, e) + native.check_error(e) + + def buffered(self): + self._native('buffered')() + + def flush(self): + e = native.ffi.new('VdirsyncerError *') + self._native('flush')(e) + native.check_error(e) diff --git a/vdirsyncer/storage/base.py b/vdirsyncer/storage/base.py index 89fe919..d6f1738 100644 --- a/vdirsyncer/storage/base.py +++ b/vdirsyncer/storage/base.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -import contextlib import functools from .. import exceptions @@ -198,26 +197,6 @@ class Storage(metaclass=StorageMeta): ''' raise NotImplementedError() - @contextlib.contextmanager - def at_once(self): - '''A contextmanager that buffers all writes. - - Essentially, this:: - - s.upload(...) - s.update(...) - - becomes this:: - - with s.at_once(): - s.upload(...) - s.update(...) - - Note that this removes guarantees about which exceptions are returned - when. - ''' - yield - def get_meta(self, key): '''Get metadata value for collection/storage. @@ -240,6 +219,14 @@ class Storage(metaclass=StorageMeta): raise NotImplementedError('This storage does not support metadata.') + def buffered(self): + '''See documentation in rust/storage/mod.rs''' + pass + + def flush(self): + '''See documentation in rust/storage/mod.rs''' + pass + def normalize_meta_value(value): # `None` is returned by iCloud for empty properties. diff --git a/vdirsyncer/storage/etesync.py b/vdirsyncer/storage/etesync.py index ebdfea4..04b1b62 100644 --- a/vdirsyncer/storage/etesync.py +++ b/vdirsyncer/storage/etesync.py @@ -1,4 +1,3 @@ -import contextlib import functools import logging import os @@ -30,10 +29,10 @@ logger = logging.getLogger(__name__) def _writing_op(f): @functools.wraps(f) def inner(self, *args, **kwargs): - if not self._at_once: + if not self._buffered: self._sync_journal() rv = f(self, *args, **kwargs) - if not self._at_once: + if not self._buffered: self._sync_journal() return rv return inner @@ -110,7 +109,7 @@ class EtesyncStorage(Storage): _collection_type = None _item_type = None - _at_once = False + _buffered = False def __init__(self, email, secrets_dir, server_url=None, db_path=None, **kwargs): @@ -213,15 +212,11 @@ class EtesyncStorage(Storage): except etesync.exceptions.DoesNotExist as e: raise exceptions.NotFoundError(e) - @contextlib.contextmanager - def at_once(self): + def buffered(self): + self._buffered = True + + def flush(self): self._sync_journal() - self._at_once = True - try: - yield self - self._sync_journal() - finally: - self._at_once = False class EtesyncContacts(EtesyncStorage): diff --git a/vdirsyncer/storage/singlefile.py b/vdirsyncer/storage/singlefile.py index 17f224b..40665f6 100644 --- a/vdirsyncer/storage/singlefile.py +++ b/vdirsyncer/storage/singlefile.py @@ -1,35 +1,18 @@ # -*- coding: utf-8 -*- -import collections -import contextlib -import functools import glob import logging import os -from atomicwrites import atomic_write - from .base import Storage -from .. import exceptions -from ..utils import checkfile, expand_path, get_etag_from_file -from ..vobject import Item, join_collection, split_collection +from ._rust import RustStorageMixin +from .. import native +from ..utils import checkfile, expand_path logger = logging.getLogger(__name__) -def _writing_op(f): - @functools.wraps(f) - def inner(self, *args, **kwargs): - if self._items is None or not self._at_once: - self.list() - rv = f(self, *args, **kwargs) - if not self._at_once: - self._write() - return rv - return inner - - -class SingleFileStorage(Storage): +class SingleFileStorage(RustStorageMixin, Storage): '''Save data in single local ``.vcf`` or ``.ics`` file. The storage basically guesses how items should be joined in the file. @@ -83,21 +66,20 @@ class SingleFileStorage(Storage): storage_name = 'singlefile' _repr_attributes = ('path',) - _write_mode = 'wb' - _append_mode = 'ab' - _read_mode = 'rb' - _items = None _last_etag = None - def __init__(self, path, encoding='utf-8', **kwargs): + def __init__(self, path, **kwargs): super(SingleFileStorage, self).__init__(**kwargs) path = os.path.abspath(expand_path(path)) checkfile(path, create=False) self.path = path - self.encoding = encoding - self._at_once = False + + self._native_storage = native.ffi.gc( + native.lib.vdirsyncer_init_singlefile(path.encode('utf-8')), + native.lib.vdirsyncer_free_singlefile + ) @classmethod def discover(cls, path, **kwargs): @@ -144,94 +126,3 @@ class SingleFileStorage(Storage): kwargs['path'] = path kwargs['collection'] = collection return kwargs - - def list(self): - self._items = collections.OrderedDict() - - try: - self._last_etag = get_etag_from_file(self.path) - with open(self.path, self._read_mode) as f: - text = f.read().decode(self.encoding) - except OSError as e: - import errno - if e.errno != errno.ENOENT: # file not found - raise IOError(e) - text = None - - if not text: - return () - - for item in split_collection(text): - item = Item(item) - etag = item.hash - self._items[item.ident] = item, etag - - return ((href, etag) for href, (item, etag) in self._items.items()) - - def get(self, href): - if self._items is None or not self._at_once: - self.list() - - try: - return self._items[href] - except KeyError: - raise exceptions.NotFoundError(href) - - @_writing_op - def upload(self, item): - href = item.ident - if href in self._items: - raise exceptions.AlreadyExistingError(existing_href=href) - - self._items[href] = item, item.hash - return href, item.hash - - @_writing_op - def update(self, href, item, etag): - if href not in self._items: - raise exceptions.NotFoundError(href) - - _, actual_etag = self._items[href] - if etag != actual_etag: - raise exceptions.WrongEtagError(etag, actual_etag) - - self._items[href] = item, item.hash - return item.hash - - @_writing_op - def delete(self, href, etag): - if href not in self._items: - raise exceptions.NotFoundError(href) - - _, actual_etag = self._items[href] - if etag != actual_etag: - raise exceptions.WrongEtagError(etag, actual_etag) - - del self._items[href] - - def _write(self): - if self._last_etag is not None and \ - self._last_etag != get_etag_from_file(self.path): - raise exceptions.PreconditionFailed( - 'Some other program modified the file {r!}. Re-run the ' - 'synchronization and make sure absolutely no other program is ' - 'writing into the same file.'.format(self.path)) - text = join_collection( - item.raw for item, etag in self._items.values() - ) - try: - with atomic_write(self.path, mode='wb', overwrite=True) as f: - f.write(text.encode(self.encoding)) - finally: - self._items = None - self._last_etag = None - - @contextlib.contextmanager - def at_once(self): - self.list() - self._at_once = True - try: - yield self - self._write() - finally: - self._at_once = False diff --git a/vdirsyncer/sync/__init__.py b/vdirsyncer/sync/__init__.py index 077d4d0..dd2bd6f 100644 --- a/vdirsyncer/sync/__init__.py +++ b/vdirsyncer/sync/__init__.py @@ -150,20 +150,25 @@ def sync(storage_a, storage_b, status, conflict_resolution=None, actions = list(_get_actions(a_info, b_info)) - with storage_a.at_once(), storage_b.at_once(): - for action in actions: - try: - action.run( - a_info, - b_info, - conflict_resolution, - partial_sync - ) - except Exception as e: - if error_callback: - error_callback(e) - else: - raise + storage_a.buffered() + storage_b.buffered() + + for action in actions: + try: + action.run( + a_info, + b_info, + conflict_resolution, + partial_sync + ) + except Exception as e: + if error_callback: + error_callback(e) + else: + raise + + storage_a.flush() + storage_b.flush() class Action: diff --git a/vdirsyncer/vobject.py b/vdirsyncer/vobject.py index fa161ee..4ff474f 100644 --- a/vdirsyncer/vobject.py +++ b/vdirsyncer/vobject.py @@ -3,7 +3,7 @@ from itertools import chain, tee from .utils import cached_property, uniq -from . import exceptions, native +from . import native class Item(object): @@ -11,73 +11,53 @@ class Item(object): '''Immutable wrapper class for VCALENDAR (VEVENT, VTODO) and VCARD''' - def __init__(self, raw, component=None): - assert isinstance(raw, str), type(raw) - self._raw = raw - if component is not None: - self.__dict__['_component'] = component + def __init__(self, raw, _native=None): + if raw is None: + assert _native + self._native = _native + return - @cached_property - def _component(self): - try: - return native.parse_component(self.raw.encode('utf-8')) - except exceptions.VobjectParseError: - return None + assert isinstance(raw, str), type(raw) + assert _native is None + self._native = native.item_rv( + native.lib.vdirsyncer_item_from_raw(raw.encode('utf-8')) + ) def with_uid(self, new_uid): - if not self._component: - raise ValueError('Item malformed.') + new_uid = new_uid or '' + assert isinstance(new_uid, str), type(new_uid) - new_c = native.clone_component(self._component) - native.change_uid(new_c, new_uid or '') - return Item(native.write_component(new_c), component=new_c) + e = native.ffi.new('VdirsyncerError *') + rv = native.lib.vdirsyncer_with_uid(self._native, + new_uid.encode('utf-8'), + e) + native.check_error(e) + return Item(None, _native=native.item_rv(rv)) - @property + @cached_property def is_parseable(self): - return bool(self._component) + return native.lib.vdirsyncer_item_is_parseable(self._native) - @property + @cached_property def raw(self): - '''Raw content of the item, as unicode string. - - Vdirsyncer doesn't validate the content in any way. - ''' - return self._raw + return native.string_rv(native.lib.vdirsyncer_get_raw(self._native)) @cached_property def uid(self): - '''Global identifier of the item, across storages, doesn't change after - a modification of the item.''' - if not self._component: - return None - return native.get_uid(self._component) or None + rv = native.string_rv(native.lib.vdirsyncer_get_uid(self._native)) + return rv or None @cached_property def hash(self): - '''Used for etags.''' - if not self.is_valid: - raise ValueError('Item malformed.') - - return native.hash_component(self._component) + e = native.ffi.new('VdirsyncerError *') + rv = native.lib.vdirsyncer_get_hash(self._native, e) + native.check_error(e) + return native.string_rv(rv) @cached_property def ident(self): - '''Used for generating hrefs and matching up items during - synchronization. This is either the UID or the hash of the item's - content.''' - - # We hash the item instead of directly using its raw content, because - # - # 1. The raw content might be really large, e.g. when it's a contact - # with a picture, which bloats the status file. - # - # 2. The status file would contain really sensitive information. return self.uid or self.hash - @property - def is_valid(self): - return bool(self._component) - def split_collection(text): assert isinstance(text, str)