Implement http storage in rust (#730)

* Port http storage to rust (#729)

* Port http storage to rust

* implement rest of parameters as far as possible

* stylefixes

* rustup

* fix invalid timestamp

* fix header file

* Fix compilation errors

* basic impl of dav

* dockerize xandikos

* add xandikos build

* Fix circleci build

* Fix circleci config

* fix nextcloud port

* stylefix

* implement upload, upload, delete in rust

* fix exc handling

* python stylefixes

* move caldav.list to rust

* fix exc again (fastmail)

* stylefixes

* add basic logging, fix fastmail

* stylefixes

* fix tests for etag=None (icloud)

* overwrite busted cargo-install-update

* install clippy from git

* fix rustfmt

* rustfmt

* clear cache

.circleci/config.yml

@@ -3,15 +3,14 @@ version: 2
 references:
   basic_env: &basic_env
     CI: true
-    DAV_SERVER: xandikos
   restore_caches: &restore_caches
     restore_cache:
       keys:
-        - cache-{{ arch }}-{{ .Branch }}
+        - cache2-{{ arch }}-{{ .Branch }}
 
   save_caches: &save_caches
     save_cache:
-      key: cache-{{ arch }}-{{ .Branch }}
+      key: cache2-{{ arch }}-{{ .Branch }}
       paths:
         - "rust/target/"
         - "~/.cargo/"
@@ -89,6 +88,23 @@ jobs:
 
       - run: make -e storage-test
 
+  xandikos:
+    docker:
+      - image: circleci/python:3.6
+        environment:
+          <<: *basic_env
+          DAV_SERVER: xandikos
+      - image: vdirsyncer/xandikos:0.0.1
+    steps:
+      - checkout
+      - *restore_caches
+      - *basic_setup
+      - run: make -e install-dev install-test
+      - *save_caches
+
+      - run: wget -O - --retry-connrefused http://localhost:5001/
+      - run: make -e storage-test
+
   style:
     docker:
       - image: circleci/python:3.6
@@ -178,22 +194,6 @@ jobs:
 
       - run: make -e test
 
-  py36-release-radicale:
-    docker:
-      - image: circleci/python:3.6
-        environment:
-          <<: *basic_env
-          REQUIREMENTS: release
-          DAV_SERVER: radicale
-    steps:
-      - checkout
-      - *restore_caches
-      - *basic_setup
-      - run: make -e install-dev install-test
-      - *save_caches
-
-      - run: make -e test
-
   py36-devel:
     docker:
       - image: circleci/python:3.6
@@ -232,6 +232,7 @@ workflows:
       - fastmail
       - icloud
       - davical
+      - xandikos
       - style
       - py34-minimal
       - py34-release

Makefile

@@ -20,9 +20,15 @@ export ETESYNC_TESTS := false
 # systemwide.
 export CI := false
 
+# Enable debug symbols and backtrace printing for rust lib
+export RUST_BACKTRACE := $(CI)
+
 # Whether to generate coverage data while running tests.
 export COVERAGE := $(CI)
 
+# Log everything
+export RUST_LOG := vdirsyncer_rustext=debug
+
 # Additional arguments that should be passed to py.test.
 PYTEST_ARGS =
 
@@ -93,10 +99,8 @@ install-test: install-servers
 
 install-style: install-docs
 	pip install -U flake8 flake8-import-order 'flake8-bugbear>=17.3.0'
-	which cargo-install-update || cargo install cargo-update
+	rustup component add rustfmt-preview
-	cargo +nightly install-update -i clippy
+	cargo install --force --git https://github.com/rust-lang-nursery/rust-clippy clippy
-	cargo +nightly install-update -i rustfmt-nightly
-	cargo +nightly install-update -i cargo-update
 
 style:
 	flake8
@@ -104,7 +108,7 @@ style:
 	! git grep -i 'text/icalendar' */*
 	sphinx-build -W -b html ./docs/ ./docs/_build/html/
 	cd rust/ && cargo +nightly clippy
-	cd rust/ && cargo +nightly fmt --all -- --write-mode=diff
+	cd rust/ && cargo +nightly fmt --all -- --check
 
 install-docs:
 	pip install -Ur docs-requirements.txt
@@ -149,6 +153,11 @@ install-rust:
 	rustup update nightly
 
 rust/vdirsyncer_rustext.h:
-	cbindgen -c rust/cbindgen.toml rust/ > $@
+	cd rust/ && cargo build # hack to work around cbindgen bugs
+	CARGO_EXPAND_TARGET_DIR=rust/target/ cbindgen -c rust/cbindgen.toml rust/ > $@
 
-.PHONY: docs rust/vdirsyncer_rustext.h
+docker/xandikos:
+	docker build -t vdirsyncer/xandikos:0.0.1 $@
+	docker push vdirsyncer/xandikos:0.0.1
+
+.PHONY: docs rust/vdirsyncer_rustext.h docker/xandikos

docker-compose.yml

@@ -4,8 +4,15 @@ services:
     nextcloud:
         image: nextcloud
         ports:
-            - '8080:80'
+            - '5000:80'
         environment:
             - SQLITE_DATABASE=nextcloud
             - NEXTCLOUD_ADMIN_USER=asdf
             - NEXTCLOUD_ADMIN_PASSWORD=asdf
+
+    xandikos:
+        build:
+            context: .
+            dockerfile: docker/xandikos/Dockerfile
+        ports:
+            - '5001:5001'

docker/xandikos/Dockerfile

@@ -0,0 +1,13 @@
+# Original file copyright 2017 Jelmer Vernooij
+
+FROM ubuntu:latest
+RUN apt-get update && apt-get -y install xandikos locales
+EXPOSE 8080
+
+RUN locale-gen en_US.UTF-8
+ENV PYTHONIOENCODING=utf-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+
+CMD xandikos -d /tmp/dav -l 0.0.0.0 -p 5001 --autocreate

docs/config.rst

@@ -511,19 +511,10 @@ leads to an error.
     of the normalized item content.
 
     :param url: URL to the ``.ics`` file.
-    :param username: Username for authentication.
+    :param username: Username for HTTP basic authentication.
-    :param password: Password for authentication.
+    :param password: Password for HTTP basic authentication.
-    :param verify: Verify SSL certificate, default True. This can also be a
+    :param useragent: Default ``vdirsyncer``.
-        local path to a self-signed SSL certificate. See :ref:`ssl-tutorial`
+    :param verify_cert: Add one new root certificate file in PEM format. Useful
-        for more information.
+                        for servers with self-signed certificates.
-    :param verify_fingerprint: Optional. SHA1 or MD5 fingerprint of the
-        expected server certificate. See :ref:`ssl-tutorial` for more
-        information.
-    :param auth: Optional. Either ``basic``, ``digest`` or ``guess``. The
-        default is preemptive Basic auth, sending credentials even if server
-        didn't request them. This saves from an additional roundtrip per
-        request. Consider setting ``guess`` if this causes issues with your
-        server.
     :param auth_cert: Optional. Either a path to a certificate with a client
         certificate and the key or a list of paths to the files with them.
-    :param useragent: Default ``vdirsyncer``.

rust/Cargo.toml

@@ -16,3 +16,8 @@ atomicwrites = "0.2.0"
 uuid = { version = "0.6", features = ["v4"] }
 libc = "0.2"
 log = "0.4"
+reqwest = "0.8"
+quick-xml = "0.12.0"
+url = "1.7"
+chrono = "0.4.0"
+env_logger = "0.5"

rust/src/errors.rs

@@ -10,40 +10,39 @@ pub enum Error {
     ItemUnparseable,
 
     #[fail(display = "Unexpected version {}, expected {}", found, expected)]
-    UnexpectedVobjectVersion {
+    UnexpectedVobjectVersion { found: String, expected: String },
-        found: String,
-        expected: String,
-    },
 
     #[fail(display = "Unexpected component {}, expected {}", found, expected)]
-    UnexpectedVobject {
+    UnexpectedVobject { found: String, expected: String },
-        found: String,
-        expected: String,
-    },
 
     #[fail(display = "Item '{}' not found", href)]
-    ItemNotFound {
+    ItemNotFound { href: String },
-        href: String,
-    },
 
     #[fail(display = "The href '{}' is already taken", href)]
-    ItemAlreadyExisting {
+    ItemAlreadyExisting { href: String },
-        href: String,
-    },
 
-    #[fail(display = "A wrong etag for '{}' was provided. Another client's requests might \
+    #[fail(
-                      conflict with vdirsyncer.",
+        display = "A wrong etag for '{}' was provided. Another client's requests might \
-           href)]
+                   conflict with vdirsyncer.",
-    WrongEtag {
+        href
-        href: String,
+    )]
-    },
+    WrongEtag { href: String },
 
-    #[fail(display = "The mtime for '{}' has unexpectedly changed. Please close other programs\
+    #[fail(
-                      accessing this file.",
+        display = "The mtime for '{}' has unexpectedly changed. Please close other programs\
-           filepath)]
+                   accessing this file.",
-    MtimeMismatch {
+        filepath
-        filepath: String,
+    )]
-    },
+    MtimeMismatch { filepath: String },
+
+    #[fail(
+        display = "The item '{}' has been rejected by the server because the vobject type was unexpected",
+        href
+    )]
+    UnsupportedVobject { href: String },
+
+    #[fail(display = "This storage is read-only.")]
+    ReadOnly,
 }
 
 pub unsafe fn export_result<V>(

rust/src/item.rs

@@ -1,7 +1,7 @@
 use vobject;
 
-use std::fmt::Write;
 use sha2::{Digest, Sha256};
+use std::fmt::Write;
 
 use errors::*;
 
@@ -190,10 +190,10 @@ fn hash_component(c: &vobject::Component) -> String {
 
 pub mod exports {
     use super::Item;
-    use std::ptr;
+    use errors::*;
     use std::ffi::{CStr, CString};
     use std::os::raw::c_char;
-    use errors::*;
+    use std::ptr;
 
     const EMPTY_STRING: *const c_char = b"\0" as *const u8 as *const c_char;
 

rust/src/lib.rs

@@ -1,3 +1,5 @@
+#![cfg_attr(feature = "cargo-clippy", allow(single_match))]
+
 extern crate atomicwrites;
 #[macro_use]
 extern crate failure;
@@ -8,11 +10,16 @@ extern crate uuid;
 extern crate vobject;
 #[macro_use]
 extern crate log;
+extern crate chrono;
+extern crate env_logger;
+extern crate quick_xml;
+extern crate reqwest;
 extern crate sha2;
+extern crate url;
 
+pub mod errors;
 mod item;
 mod storage;
-pub mod errors;
 
 pub mod exports {
     use std::ffi::CStr;
@@ -25,4 +32,9 @@ pub mod exports {
     pub unsafe extern "C" fn vdirsyncer_free_str(s: *const c_char) {
         CStr::from_ptr(s);
     }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn vdirsyncer_init_logger() {
+        ::env_logger::init();
+    }
 }

rust/src/storage/dav/mod.rs

@@ -0,0 +1,465 @@
+mod parser;
+
+use chrono;
+
+use std::collections::BTreeSet;
+use std::io::{BufReader, Read};
+use std::str::FromStr;
+
+use quick_xml;
+use reqwest;
+use reqwest::header::{ContentType, ETag, EntityTag, IfMatch, IfNoneMatch};
+use url::Url;
+
+use super::http::{handle_http_error, send_request, HttpConfig};
+use super::utils::generate_href;
+use super::Storage;
+use errors::*;
+
+use item::Item;
+
+#[inline]
+fn propfind() -> reqwest::Method {
+    reqwest::Method::Extension("PROPFIND".to_owned())
+}
+
+#[inline]
+fn report() -> reqwest::Method {
+    reqwest::Method::Extension("REPORT".to_owned())
+}
+
+static CALDAV_DT_FORMAT: &'static str = "%Y%m%dT%H%M%SZ";
+
+struct DavStorage {
+    pub url: String,
+    pub http_config: HttpConfig,
+    pub http: Option<reqwest::Client>,
+}
+
+impl DavStorage {
+    pub fn new(url: &str, http_config: HttpConfig) -> Self {
+        DavStorage {
+            url: format!("{}/", url.trim_right_matches('/')),
+            http_config,
+            http: None,
+        }
+    }
+}
+
+impl DavStorage {
+    #[inline]
+    pub fn get_http(&mut self) -> Fallible<reqwest::Client> {
+        if let Some(ref http) = self.http {
+            return Ok(http.clone());
+        }
+        let client = self.http_config.clone().into_connection()?.build()?;
+        self.http = Some(client.clone());
+        Ok(client)
+    }
+
+    #[inline]
+    pub fn send_request(&mut self, request: reqwest::Request) -> Fallible<reqwest::Response> {
+        let url = request.url().to_string();
+        handle_http_error(&url, send_request(&self.get_http()?, request)?)
+    }
+
+    pub fn get(&mut self, href: &str) -> Fallible<(Item, String)> {
+        let base = Url::parse(&self.url)?;
+        let url = base.join(href)?;
+        if href != url.path() {
+            Err(Error::ItemNotFound {
+                href: href.to_owned(),
+            })?;
+        }
+
+        let request = self.get_http()?.get(url).build()?;
+        let mut response = self.send_request(request)?;
+        let mut s = String::new();
+        response.read_to_string(&mut s)?;
+        let etag = match response.headers().get::<ETag>() {
+            Some(x) => format!("\"{}\"", x.tag()),
+            None => Err(DavError::EtagNotFound)?,
+        };
+        Ok((Item::from_raw(s), etag))
+    }
+
+    pub fn list<'a>(
+        &'a mut self,
+        mimetype_contains: &'a str,
+    ) -> Fallible<Box<Iterator<Item = (String, String)> + 'a>> {
+        let mut headers = reqwest::header::Headers::new();
+        headers.set(ContentType::xml());
+        headers.set_raw("Depth", "1");
+
+        let request = self
+            .get_http()?
+            .request(propfind(), &self.url)
+            .headers(headers)
+            .body(
+                r#"<?xml version="1.0" encoding="utf-8" ?>
+                <D:propfind xmlns:D="DAV:">
+                    <D:prop>
+                        <D:resourcetype/>
+                        <D:getcontenttype/>
+                        <D:getetag/>
+                    </D:prop>
+                </D:propfind>"#,
+            )
+            .build()?;
+        let response = self.send_request(request)?;
+        self.parse_prop_response(response, mimetype_contains)
+    }
+
+    fn parse_prop_response<'a>(
+        &'a mut self,
+        response: reqwest::Response,
+        mimetype_contains: &'a str,
+    ) -> Fallible<Box<Iterator<Item = (String, String)> + 'a>> {
+        let buf_reader = BufReader::new(response);
+        let xml_reader = quick_xml::Reader::from_reader(buf_reader);
+
+        let mut parser = parser::ListingParser::new(xml_reader);
+        let base = Url::parse(&self.url)?;
+        let mut seen_hrefs = BTreeSet::new();
+
+        Ok(Box::new(
+            parser
+                .get_all_responses()?
+                .into_iter()
+                .filter_map(move |response| {
+                    if response.has_collection_tag {
+                        return None;
+                    }
+                    if !response.mimetype?.contains(mimetype_contains) {
+                        return None;
+                    }
+
+                    let href = base.join(&response.href?).ok()?.path().to_owned();
+
+                    if seen_hrefs.contains(&href) {
+                        return None;
+                    }
+                    seen_hrefs.insert(href.clone());
+                    Some((href, response.etag?))
+                }),
+        ))
+    }
+
+    fn put(
+        &mut self,
+        href: &str,
+        item: &Item,
+        mimetype: &str,
+        etag: Option<&str>,
+    ) -> Fallible<(String, String)> {
+        let base = Url::parse(&self.url)?;
+        let url = base.join(href)?;
+        let mut request = self.get_http()?.request(reqwest::Method::Put, url);
+        request.header(ContentType(reqwest::mime::Mime::from_str(mimetype)?));
+        if let Some(etag) = etag {
+            request.header(IfMatch::Items(vec![EntityTag::new(
+                false,
+                etag.trim_matches('"').to_owned(),
+            )]));
+        } else {
+            request.header(IfNoneMatch::Any);
+        }
+
+        let raw = item.get_raw();
+        let response = send_request(&self.get_http()?, request.body(raw).build()?)?;
+
+        match (etag, response.status()) {
+            (Some(_), reqwest::StatusCode::PreconditionFailed) => Err(Error::WrongEtag {
+                href: href.to_owned(),
+            })?,
+            (None, reqwest::StatusCode::PreconditionFailed) => Err(Error::ItemAlreadyExisting {
+                href: href.to_owned(),
+            })?,
+            _ => (),
+        }
+
+        let response = assert_multistatus_success(handle_http_error(href, response)?)?;
+
+        // The server may not return an etag under certain conditions:
+        //
+        //   An origin server MUST NOT send a validator header field (Section
+        //   7.2), such as an ETag or Last-Modified field, in a successful
+        //   response to PUT unless the request's representation data was saved
+        //   without any transformation applied to the body (i.e., the
+        //   resource's new representation data is identical to the
+        //   representation data received in the PUT request) and the validator
+        //   field value reflects the new representation.
+        //
+        // -- https://tools.ietf.org/html/rfc7231#section-4.3.4
+        //
+        // In such cases we return a constant etag. The next synchronization
+        // will then detect an etag change and will download the new item.
+        let etag = match response.headers().get::<ETag>() {
+            Some(x) => format!("\"{}\"", x.tag()),
+            None => "".to_owned(),
+        };
+        Ok((response.url().path().to_owned(), etag))
+    }
+
+    fn delete(&mut self, href: &str, etag: &str) -> Fallible<()> {
+        let base = Url::parse(&self.url)?;
+        let url = base.join(href)?;
+        let request = self
+            .get_http()?
+            .request(reqwest::Method::Delete, url)
+            .header(IfMatch::Items(vec![EntityTag::new(
+                false,
+                etag.trim_matches('"').to_owned(),
+            )]))
+            .build()?;
+        let response = send_request(&self.get_http()?, request)?;
+
+        if response.status() == reqwest::StatusCode::PreconditionFailed {
+            Err(Error::WrongEtag {
+                href: href.to_owned(),
+            })?;
+        }
+
+        assert_multistatus_success(handle_http_error(href, response)?)?;
+        Ok(())
+    }
+}
+
+fn assert_multistatus_success(r: reqwest::Response) -> Fallible<reqwest::Response> {
+    // TODO
+    Ok(r)
+}
+
+struct CarddavStorage {
+    inner: DavStorage,
+}
+
+impl CarddavStorage {
+    pub fn new(url: &str, http_config: HttpConfig) -> Self {
+        CarddavStorage {
+            inner: DavStorage::new(url, http_config),
+        }
+    }
+}
+
+impl Storage for CarddavStorage {
+    fn list<'a>(&'a mut self) -> Fallible<Box<Iterator<Item = (String, String)> + 'a>> {
+        self.inner.list("vcard")
+    }
+
+    fn get(&mut self, href: &str) -> Fallible<(Item, String)> {
+        self.inner.get(href)
+    }
+
+    fn upload(&mut self, item: Item) -> Fallible<(String, String)> {
+        let href = format!("{}.vcf", generate_href(&item.get_ident()?));
+        self.inner.put(&href, &item, "text/vcard", None)
+    }
+
+    fn update(&mut self, href: &str, item: Item, etag: &str) -> Fallible<String> {
+        self.inner
+            .put(&href, &item, "text/vcard", Some(etag))
+            .map(|x| x.1)
+    }
+
+    fn delete(&mut self, href: &str, etag: &str) -> Fallible<()> {
+        self.inner.delete(href, etag)
+    }
+}
+
+struct CaldavStorage {
+    inner: DavStorage,
+    start_date: Option<chrono::DateTime<chrono::Utc>>, // FIXME: store as Option<(start, end)>
+    end_date: Option<chrono::DateTime<chrono::Utc>>,
+    item_types: Vec<&'static str>,
+}
+
+impl CaldavStorage {
+    pub fn new(
+        url: &str,
+        http_config: HttpConfig,
+        start_date: Option<chrono::DateTime<chrono::Utc>>,
+        end_date: Option<chrono::DateTime<chrono::Utc>>,
+        item_types: Vec<&'static str>,
+    ) -> Self {
+        CaldavStorage {
+            inner: DavStorage::new(url, http_config),
+            start_date,
+            end_date,
+            item_types,
+        }
+    }
+
+    #[inline]
+    fn get_caldav_filters(&self) -> Vec<String> {
+        let mut item_types = self.item_types.clone();
+        let mut timefilter = "".to_owned();
+
+        if let (Some(start), Some(end)) = (self.start_date, self.end_date) {
+            timefilter = format!(
+                "<C:time-range start=\"{}\" end=\"{}\" />",
+                start.format(CALDAV_DT_FORMAT),
+                end.format(CALDAV_DT_FORMAT)
+            );
+
+            if item_types.is_empty() {
+                item_types.push("VTODO");
+                item_types.push("VEVENT");
+            }
+        }
+
+        item_types
+            .into_iter()
+            .map(|item_type| {
+                format!(
+                    "<C:comp-filter name=\"VCALENDAR\">\
+                     <C:comp-filter name=\"{}\">{}</C:comp-filter>\
+                     </C:comp-filter>",
+                    item_type, timefilter
+                )
+            })
+            .collect()
+    }
+}
+
+impl Storage for CaldavStorage {
+    fn list<'a>(&'a mut self) -> Fallible<Box<Iterator<Item = (String, String)> + 'a>> {
+        let filters = self.get_caldav_filters();
+        if filters.is_empty() {
+            // If we don't have any filters (which is the default), taking the
+            // risk of sending a calendar-query is not necessary. There doesn't
+            // seem to be a widely-usable way to send calendar-queries with the
+            // same semantics as a PROPFIND request... so why not use PROPFIND
+            // instead?
+            //
+            // See https://github.com/dmfs/tasks/issues/118 for backstory.
+            self.inner.list("text/calendar")
+        } else {
+            let mut rv = vec![];
+            let mut headers = reqwest::header::Headers::new();
+            headers.set(ContentType::xml());
+            headers.set_raw("Depth", "1");
+
+            for filter in filters {
+                let data =
+                    format!(
+                    "<?xml version=\"1.0\" encoding=\"utf-8\" ?>\
+                     <C:calendar-query xmlns:D=\"DAV:\" xmlns:C=\"urn:ietf:params:xml:ns:caldav\">\
+                     <D:prop><D:getcontenttype/><D:getetag/></D:prop>\
+                     <C:filter>{}</C:filter>\
+                     </C:calendar-query>", filter);
+
+                let request = self
+                    .inner
+                    .get_http()?
+                    .request(report(), &self.inner.url)
+                    .headers(headers.clone())
+                    .body(data)
+                    .build()?;
+                let response = self.inner.send_request(request)?;
+                rv.extend(self.inner.parse_prop_response(response, "text/calendar")?);
+            }
+
+            Ok(Box::new(rv.into_iter()))
+        }
+    }
+
+    fn get(&mut self, href: &str) -> Fallible<(Item, String)> {
+        self.inner.get(href)
+    }
+
+    fn upload(&mut self, item: Item) -> Fallible<(String, String)> {
+        let href = format!("{}.ics", generate_href(&item.get_ident()?));
+        self.inner.put(&href, &item, "text/calendar", None)
+    }
+
+    fn update(&mut self, href: &str, item: Item, etag: &str) -> Fallible<String> {
+        self.inner
+            .put(href, &item, "text/calendar", Some(etag))
+            .map(|x| x.1)
+    }
+
+    fn delete(&mut self, href: &str, etag: &str) -> Fallible<()> {
+        self.inner.delete(href, etag)
+    }
+}
+
+pub mod exports {
+    use super::super::http::init_http_config;
+    use super::*;
+
+    #[derive(Debug, Fail, Shippai)]
+    pub enum DavError {
+        #[fail(display = "Server did not return etag.")]
+        EtagNotFound,
+    }
+
+    use std::ffi::CStr;
+    use std::os::raw::c_char;
+
+    #[no_mangle]
+    pub unsafe extern "C" fn vdirsyncer_init_carddav(
+        url: *const c_char,
+        username: *const c_char,
+        password: *const c_char,
+        useragent: *const c_char,
+        verify_cert: *const c_char,
+        auth_cert: *const c_char,
+    ) -> *mut Box<Storage> {
+        let url = CStr::from_ptr(url);
+
+        Box::into_raw(Box::new(Box::new(CarddavStorage::new(
+            url.to_str().unwrap(),
+            init_http_config(username, password, useragent, verify_cert, auth_cert),
+        ))))
+    }
+
+    #[no_mangle]
+    pub unsafe extern "C" fn vdirsyncer_init_caldav(
+        url: *const c_char,
+        username: *const c_char,
+        password: *const c_char,
+        useragent: *const c_char,
+        verify_cert: *const c_char,
+        auth_cert: *const c_char,
+        start_date: i64,
+        end_date: i64,
+        include_vevent: bool,
+        include_vjournal: bool,
+        include_vtodo: bool,
+    ) -> *mut Box<Storage> {
+        let url = CStr::from_ptr(url);
+
+        let parse_date = |i| {
+            if i > 0 {
+                Some(chrono::DateTime::from_utc(
+                    chrono::NaiveDateTime::from_timestamp(i, 0),
+                    chrono::Utc,
+                ))
+            } else {
+                None
+            }
+        };
+
+        let mut item_types = vec![];
+        if include_vevent {
+            item_types.push("VEVENT");
+        }
+        if include_vjournal {
+            item_types.push("VJOURNAL");
+        }
+        if include_vtodo {
+            item_types.push("VTODO");
+        }
+
+        Box::into_raw(Box::new(Box::new(CaldavStorage::new(
+            url.to_str().unwrap(),
+            init_http_config(username, password, useragent, verify_cert, auth_cert),
+            parse_date(start_date),
+            parse_date(end_date),
+            item_types,
+        ))))
+    }
+}
+
+use exports::DavError;

rust/src/storage/dav/parser.rs

@@ -0,0 +1,110 @@
+use quick_xml;
+use quick_xml::events::Event;
+
+use errors::*;
+
+use std::io::BufRead;
+
+#[derive(Debug)]
+pub struct Response {
+    pub href: Option<String>,
+    pub etag: Option<String>,
+    pub mimetype: Option<String>,
+    pub has_collection_tag: bool,
+}
+
+impl Response {
+    pub fn new() -> Self {
+        Response {
+            href: None,
+            etag: None,
+            has_collection_tag: false,
+            mimetype: None,
+        }
+    }
+}
+
+pub struct ListingParser<T: BufRead> {
+    reader: quick_xml::Reader<T>,
+    ns_buf: Vec<u8>,
+}
+
+impl<T: BufRead> ListingParser<T> {
+    pub fn new(mut reader: quick_xml::Reader<T>) -> Self {
+        reader.expand_empty_elements(true);
+        reader.trim_text(true);
+        reader.check_end_names(true);
+        reader.check_comments(false);
+
+        ListingParser {
+            reader,
+            ns_buf: vec![],
+        }
+    }
+
+    fn next_response(&mut self) -> Fallible<Option<Response>> {
+        let mut buf = vec![];
+
+        #[derive(Debug, Clone, Copy)]
+        enum State {
+            Outer,
+            Response,
+            Href,
+            ContentType,
+            Etag,
+        };
+
+        let mut state = State::Outer;
+        let mut current_response = Response::new();
+
+        loop {
+            match self
+                .reader
+                .read_namespaced_event(&mut buf, &mut self.ns_buf)?
+            {
+                (ns, Event::Start(ref e)) => {
+                    match (state, ns, e.local_name()) {
+                        (State::Outer, Some(b"DAV:"), b"response") => state = State::Response,
+                        (State::Response, Some(b"DAV:"), b"href") => state = State::Href,
+                        (State::Response, Some(b"DAV:"), b"getetag") => state = State::Etag,
+                        (State::Response, Some(b"DAV:"), b"getcontenttype") => {
+                            state = State::ContentType
+                        }
+                        (State::Response, Some(b"DAV:"), b"collection") => {
+                            current_response.has_collection_tag = true;
+                        }
+                        _ => (),
+                    }
+
+                    debug!("State: {:?}", state);
+                }
+                (_, Event::Text(e)) => {
+                    let txt = e.unescape_and_decode(&self.reader)?;
+                    match state {
+                        State::Href => current_response.href = Some(txt),
+                        State::ContentType => current_response.mimetype = Some(txt),
+                        State::Etag => current_response.etag = Some(txt),
+                        _ => continue,
+                    }
+                    state = State::Response;
+                }
+                (ns, Event::End(e)) => match (state, ns, e.local_name()) {
+                    (State::Response, Some(b"DAV:"), b"response") => {
+                        return Ok(Some(current_response))
+                    }
+                    _ => (),
+                },
+                (_, Event::Eof) => return Ok(None),
+                _ => (),
+            }
+        }
+    }
+
+    pub fn get_all_responses(&mut self) -> Fallible<Vec<Response>> {
+        let mut rv = vec![];
+        while let Some(x) = self.next_response()? {
+            rv.push(x);
+        }
+        Ok(rv)
+    }
+}

rust/src/storage/exports.rs

@@ -1,11 +1,13 @@
-use std::os::raw::c_char;
+pub use super::dav::exports::*;
-use std::ptr;
+pub use super::filesystem::exports::*;
-use std::ffi::{CStr, CString};
+pub use super::http::exports::*;
+pub use super::singlefile::exports::*;
+use super::Storage;
 use errors::*;
 use item::Item;
-use super::Storage;
+use std::ffi::{CStr, CString};
-pub use super::singlefile::exports::*;
+use std::os::raw::c_char;
-pub use super::filesystem::exports::*;
+use std::ptr;
 
 #[no_mangle]
 pub unsafe extern "C" fn vdirsyncer_storage_free(storage: *mut Box<Storage>) {

rust/src/storage/filesystem.rs

@@ -1,13 +1,13 @@
-use std::path::{Path, PathBuf};
+use super::Storage;
+use errors::*;
+use failure;
+use libc;
 use std::fs;
 use std::io;
 use std::io::{Read, Write};
 use std::os::unix::fs::MetadataExt;
+use std::path::{Path, PathBuf};
 use std::process::Command;
-use super::Storage;
-use errors::*;
-use libc;
-use failure;
 
 use super::utils;
 

rust/src/storage/http.rs

@@ -0,0 +1,230 @@
+use std::collections::BTreeMap;
+use std::fs::File;
+use std::io::Read;
+
+use std::ffi::CStr;
+use std::os::raw::c_char;
+
+use reqwest;
+
+use super::singlefile::split_collection;
+use super::Storage;
+use errors::*;
+
+use item::Item;
+
+type ItemCache = BTreeMap<String, (Item, String)>;
+pub type Username = String;
+pub type Password = String;
+pub type Auth = (Username, Password);
+
+/// Wrapper around Client.execute to enable logging
+#[inline]
+pub fn send_request(
+    client: &reqwest::Client,
+    request: reqwest::Request,
+) -> Fallible<reqwest::Response> {
+    debug!("> {} {}", request.method(), request.url());
+    for header in request.headers().iter() {
+        debug!("> {}: {}", header.name(), header.value_string());
+    }
+    debug!("> {:?}", request.body());
+    debug!("> ---");
+    let response = client.execute(request)?;
+    debug!("< {:?}", response.status());
+    for header in response.headers().iter() {
+        debug!("< {}: {}", header.name(), header.value_string());
+    }
+    Ok(response)
+}
+
+#[derive(Clone)]
+pub struct HttpConfig {
+    pub auth: Option<Auth>,
+    pub useragent: Option<String>,
+    pub verify_cert: Option<String>,
+    pub auth_cert: Option<String>,
+}
+
+impl HttpConfig {
+    pub fn into_connection(self) -> Fallible<reqwest::ClientBuilder> {
+        let mut headers = reqwest::header::Headers::new();
+
+        if let Some((username, password)) = self.auth {
+            headers.set(reqwest::header::Authorization(reqwest::header::Basic {
+                username,
+                password: Some(password),
+            }));
+        }
+
+        if let Some(useragent) = self.useragent {
+            headers.set(reqwest::header::UserAgent::new(useragent));
+        }
+
+        let mut client = reqwest::Client::builder();
+        client.default_headers(headers);
+
+        if let Some(verify_cert) = self.verify_cert {
+            let mut buf = Vec::new();
+            File::open(verify_cert)?.read_to_end(&mut buf)?;
+            let cert = reqwest::Certificate::from_pem(&buf)?;
+            client.add_root_certificate(cert);
+        }
+
+        // TODO: auth_cert https://github.com/sfackler/rust-native-tls/issues/27
+        Ok(client)
+    }
+}
+
+pub struct HttpStorage {
+    url: String,
+    // href -> (item, etag)
+    items_cache: Option<ItemCache>,
+    http_config: HttpConfig,
+}
+
+impl HttpStorage {
+    pub fn new(url: String, http_config: HttpConfig) -> Self {
+        HttpStorage {
+            url,
+            items_cache: None,
+            http_config,
+        }
+    }
+
+    fn get_items(&mut self) -> Fallible<&mut ItemCache> {
+        if self.items_cache.is_none() {
+            self.list()?;
+        }
+        Ok(self.items_cache.as_mut().unwrap())
+    }
+}
+
+impl Storage for HttpStorage {
+    fn list<'a>(&'a mut self) -> Fallible<Box<Iterator<Item = (String, String)> + 'a>> {
+        let client = self.http_config.clone().into_connection()?.build()?;
+
+        let mut response = handle_http_error(&self.url, client.get(&self.url).send()?)?;
+        let s = response.text()?;
+
+        let mut new_cache = BTreeMap::new();
+        for component in split_collection(&s)? {
+            let mut item = Item::from_component(component);
+            item = item.with_uid(&item.get_hash()?)?;
+            let ident = item.get_ident()?;
+            let hash = item.get_hash()?;
+            new_cache.insert(ident, (item, hash));
+        }
+
+        self.items_cache = Some(new_cache);
+        Ok(Box::new(self.items_cache.as_ref().unwrap().iter().map(
+            |(href, &(_, ref etag))| (href.clone(), etag.clone()),
+        )))
+    }
+
+    fn get(&mut self, href: &str) -> Fallible<(Item, String)> {
+        match self.get_items()?.get(href) {
+            Some(&(ref href, ref etag)) => Ok((href.clone(), etag.clone())),
+            None => Err(Error::ItemNotFound {
+                href: href.to_owned(),
+            })?,
+        }
+    }
+
+    fn upload(&mut self, _item: Item) -> Fallible<(String, String)> {
+        Err(Error::ReadOnly)?
+    }
+
+    fn update(&mut self, _href: &str, _item: Item, _etag: &str) -> Fallible<String> {
+        Err(Error::ReadOnly)?
+    }
+
+    fn delete(&mut self, _href: &str, _etag: &str) -> Fallible<()> {
+        Err(Error::ReadOnly)?
+    }
+}
+
+pub mod exports {
+    use super::*;
+    use std::ffi::CStr;
+    use std::os::raw::c_char;
+
+    #[no_mangle]
+    pub unsafe extern "C" fn vdirsyncer_init_http(
+        url: *const c_char,
+        username: *const c_char,
+        password: *const c_char,
+        useragent: *const c_char,
+        verify_cert: *const c_char,
+        auth_cert: *const c_char,
+    ) -> *mut Box<Storage> {
+        let url = CStr::from_ptr(url);
+
+        Box::into_raw(Box::new(Box::new(HttpStorage::new(
+            url.to_str().unwrap().to_owned(),
+            init_http_config(username, password, useragent, verify_cert, auth_cert),
+        ))))
+    }
+}
+
+pub fn handle_http_error(href: &str, mut r: reqwest::Response) -> Fallible<reqwest::Response> {
+    if !r.status().is_success() {
+        debug!("< Error response, dumping body:");
+        debug!("< {:?}", r.text());
+    }
+
+    match r.status() {
+        reqwest::StatusCode::NotFound => Err(Error::ItemNotFound {
+            href: href.to_owned(),
+        })?,
+        reqwest::StatusCode::UnsupportedMediaType => Err(Error::UnsupportedVobject {
+            href: href.to_owned(),
+        })?,
+        _ => Ok(r.error_for_status()?),
+    }
+}
+
+pub unsafe fn init_http_config(
+    username: *const c_char,
+    password: *const c_char,
+    useragent: *const c_char,
+    verify_cert: *const c_char,
+    auth_cert: *const c_char,
+) -> HttpConfig {
+    let username = CStr::from_ptr(username);
+    let password = CStr::from_ptr(password);
+    let username_dec = username.to_str().unwrap();
+    let password_dec = password.to_str().unwrap();
+
+    let useragent = CStr::from_ptr(useragent);
+    let useragent_dec = useragent.to_str().unwrap();
+    let verify_cert = CStr::from_ptr(verify_cert);
+    let verify_cert_dec = verify_cert.to_str().unwrap();
+    let auth_cert = CStr::from_ptr(auth_cert);
+    let auth_cert_dec = auth_cert.to_str().unwrap();
+
+    let auth = if !username_dec.is_empty() && !password_dec.is_empty() {
+        Some((username_dec.to_owned(), password_dec.to_owned()))
+    } else {
+        None
+    };
+
+    HttpConfig {
+        auth,
+        useragent: if useragent_dec.is_empty() {
+            None
+        } else {
+            Some(useragent_dec.to_owned())
+        },
+        verify_cert: if verify_cert_dec.is_empty() {
+            None
+        } else {
+            Some(verify_cert_dec.to_owned())
+        },
+        auth_cert: if auth_cert_dec.is_empty() {
+            None
+        } else {
+            Some(auth_cert_dec.to_owned())
+        },
+    }
+}

rust/src/storage/mod.rs

@@ -1,6 +1,8 @@
-pub mod singlefile;
+mod dav;
 pub mod exports;
-pub mod filesystem;
+mod filesystem;
+mod http;
+mod singlefile;
 mod utils;
 use errors::Fallible;
 use item::Item;

rust/src/storage/singlefile.rs

@@ -1,11 +1,11 @@
-use std::path::{Path, PathBuf};
-use std::collections::{BTreeMap, BTreeSet};
-use std::collections::btree_map::Entry::*;
-use std::fs::{metadata, File};
-use std::io::{Read, Write};
-use std::time::SystemTime;
 use super::Storage;
 use errors::*;
+use std::collections::btree_map::Entry::*;
+use std::collections::{BTreeMap, BTreeSet};
+use std::fs::{metadata, File};
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use std::time::SystemTime;
 use vobject;
 
 use atomicwrites::{AllowOverwrite, AtomicFile};
@@ -180,7 +180,7 @@ impl Storage for SinglefileStorage {
     }
 }
 
-fn split_collection(mut input: &str) -> Fallible<Vec<vobject::Component>> {
+pub fn split_collection(mut input: &str) -> Fallible<Vec<vobject::Component>> {
     let mut rv = vec![];
     while !input.is_empty() {
         let (component, remainder) =
@@ -240,7 +240,8 @@ fn split_vcalendar(mut vcalendar: vobject::Component) -> Fallible<Vec<vobject::C
     for component in subcomponents {
         let uid = component.get_only("UID").cloned();
 
-        let mut wrapper = match uid.as_ref()
+        let mut wrapper = match uid
+            .as_ref()
             .and_then(|u| by_uid.remove(&u.value_as_string()))
         {
             Some(x) => x,

rust/vdirsyncer_rustext.h

@@ -20,6 +20,8 @@ typedef struct {
   const char *etag;
 } VdirsyncerStorageUploadResult;
 
+extern const uint8_t SHIPPAI_VARIANT_DavError_EtagNotFound;
+
 extern const uint8_t SHIPPAI_VARIANT_Error_ItemAlreadyExisting;
 
 extern const uint8_t SHIPPAI_VARIANT_Error_ItemNotFound;
@@ -28,10 +30,14 @@ extern const uint8_t SHIPPAI_VARIANT_Error_ItemUnparseable;
 
 extern const uint8_t SHIPPAI_VARIANT_Error_MtimeMismatch;
 
+extern const uint8_t SHIPPAI_VARIANT_Error_ReadOnly;
+
 extern const uint8_t SHIPPAI_VARIANT_Error_UnexpectedVobject;
 
 extern const uint8_t SHIPPAI_VARIANT_Error_UnexpectedVobjectVersion;
 
+extern const uint8_t SHIPPAI_VARIANT_Error_UnsupportedVobject;
+
 extern const uint8_t SHIPPAI_VARIANT_Error_WrongEtag;
 
 void shippai_free_failure(ShippaiError *t);
@@ -42,8 +48,12 @@ const char *shippai_get_debug(ShippaiError *t);
 
 const char *shippai_get_display(ShippaiError *t);
 
+uint8_t shippai_get_variant_DavError(ShippaiError *t);
+
 uint8_t shippai_get_variant_Error(ShippaiError *t);
 
+bool shippai_is_error_DavError(ShippaiError *t);
+
 bool shippai_is_error_Error(ShippaiError *t);
 
 bool vdirsyncer_advance_storage_listing(VdirsyncerStorageListing *listing);
@@ -64,10 +74,38 @@ const char *vdirsyncer_get_raw(Item *c);
 
 const char *vdirsyncer_get_uid(Item *c);
 
+Box_Storage *vdirsyncer_init_caldav(const char *url,
+                                    const char *username,
+                                    const char *password,
+                                    const char *useragent,
+                                    const char *verify_cert,
+                                    const char *auth_cert,
+                                    int64_t start_date,
+                                    int64_t end_date,
+                                    bool include_vevent,
+                                    bool include_vjournal,
+                                    bool include_vtodo);
+
+Box_Storage *vdirsyncer_init_carddav(const char *url,
+                                     const char *username,
+                                     const char *password,
+                                     const char *useragent,
+                                     const char *verify_cert,
+                                     const char *auth_cert);
+
 Box_Storage *vdirsyncer_init_filesystem(const char *path,
                                         const char *fileext,
                                         const char *post_hook);
 
+Box_Storage *vdirsyncer_init_http(const char *url,
+                                  const char *username,
+                                  const char *password,
+                                  const char *useragent,
+                                  const char *verify_cert,
+                                  const char *auth_cert);
+
+void vdirsyncer_init_logger(void);
+
 Box_Storage *vdirsyncer_init_singlefile(const char *path);
 
 Item *vdirsyncer_item_from_raw(const char *s);

setup.py

@@ -7,6 +7,7 @@ how to package vdirsyncer.
 '''
 
 
+import os
 from setuptools import Command, find_packages, setup
 
 milksnake = 'milksnake'
@@ -42,15 +43,19 @@ requirements = [
 
 
 def build_native(spec):
-    build = spec.add_external_build(
+    cmd = ['cargo', 'build']
-        cmd=['cargo', 'build', '--release'],
+    if os.environ.get('RUST_BACKTRACE', 'false') in ('true', '1', 'full'):
-        path='./rust/'
+        dylib_folder = 'target/debug'
-    )
+    else:
+        dylib_folder = 'target/release'
+        cmd.append('--release')
+
+    build = spec.add_external_build(cmd=cmd, path='./rust/')
 
     spec.add_cffi_module(
         module_path='vdirsyncer._native',
         dylib=lambda: build.find_dylib('vdirsyncer_rustext',
-                                       in_path='target/release'),
+                                       in_path=dylib_folder),
         header_filename='rust/vdirsyncer_rustext.h',
         # Rust bug: If thread-local storage is used, this flag is necessary
         # (mitsuhiko)

tests/__init__.py

@@ -57,6 +57,7 @@ END:VCALENDAR'''
 BARE_EVENT_TEMPLATE = u'''BEGIN:VEVENT
 DTSTART:19970714T170000Z
 DTEND:19970715T035959Z
+DTSTAMP:19970610T172345Z
 SUMMARY:Bastille Day Party
 X-SOMETHING:{r}
 UID:{uid}

tests/storage/__init__.py

@@ -3,7 +3,6 @@
 import uuid
 
 import textwrap
-from urllib.parse import quote as urlquote, unquote as urlunquote
 
 import pytest
 
@@ -133,6 +132,8 @@ class StorageTests(object):
 
     def test_delete(self, s, get_item):
         href, etag = s.upload(get_item())
+        if etag is None:
+            _, etag = s.get(href)
         s.delete(href, etag)
         assert not list(s.list())
 
@@ -150,6 +151,8 @@ class StorageTests(object):
     def test_has(self, s, get_item):
         assert not s.has('asd')
         href, etag = s.upload(get_item())
+        if etag is None:
+            _, etag = s.get(href)
         assert s.has(href)
         assert not s.has('asd')
         s.delete(href, etag)
@@ -236,38 +239,6 @@ class StorageTests(object):
         assert len(items) == 2
         assert len(set(items)) == 2
 
-    def test_specialchars(self, monkeypatch, requires_collections,
-                          get_storage_args, get_item):
-        if getattr(self, 'dav_server', '') == 'radicale':
-            pytest.skip('Radicale is fundamentally broken.')
-        if getattr(self, 'dav_server', '') in ('icloud', 'fastmail'):
-            pytest.skip('iCloud and FastMail reject this name.')
-
-        monkeypatch.setattr('vdirsyncer.utils.generate_href', lambda x: x)
-
-        uid = u'test @ foo ät bar град сатану'
-        collection = 'test @ foo ät bar'
-
-        s = self.storage_class(**get_storage_args(collection=collection))
-        item = get_item(uid=uid)
-
-        href, etag = s.upload(item)
-        item2, etag2 = s.get(href)
-        if etag is not None:
-            assert etag2 == etag
-            assert_item_equals(item2, item)
-
-        (_, etag3), = s.list()
-        assert etag2 == etag3
-
-        # etesync uses UUIDs for collection names
-        if self.storage_class.storage_name.startswith('etesync'):
-            return
-
-        assert collection in urlunquote(s.collection)
-        if self.storage_class.storage_name.endswith('dav'):
-            assert urlquote(uid, '/@:') in href
-
     def test_metadata(self, requires_metadata, s):
         if not getattr(self, 'dav_server', ''):
             assert not s.get_meta('color')

tests/storage/dav/__init__.py

@@ -1,13 +1,7 @@
 # -*- coding: utf-8 -*-
 
-import uuid
-
 import os
 
-import pytest
-
-from tests import assert_item_equals
-
 from .. import StorageTests, get_server_mixin
 
 
@@ -29,14 +23,3 @@ class DAVStorageTests(ServerMixin, StorageTests):
         finally:
             # Make sure monkeypatch doesn't interfere with DAV server teardown
             monkeypatch.undo()
-
-    def test_dav_unicode_href(self, s, get_item, monkeypatch):
-        if self.dav_server == 'radicale':
-            pytest.skip('Radicale is unable to deal with unicode hrefs')
-
-        monkeypatch.setattr(s, '_get_href',
-                            lambda item: item.ident + s.fileext)
-        item = get_item(uid=u'град сатану' + str(uuid.uuid4()))
-        href, etag = s.upload(item)
-        item2, etag2 = s.get(href)
-        assert_item_equals(item, item2)

tests/storage/dav/test_caldav.py

@@ -5,12 +5,8 @@ from textwrap import dedent
 
 import pytest
 
-import requests
-import requests.exceptions
-
 from tests import EVENT_TEMPLATE, TASK_TEMPLATE, VCARD_TEMPLATE
 
-from vdirsyncer import exceptions
 from vdirsyncer.storage.dav import CalDAVStorage
 
 from . import DAVStorageTests, dav_server
@@ -29,33 +25,10 @@ class TestCalDAVStorage(DAVStorageTests):
 
         try:
             s.upload(format_item(item_template=VCARD_TEMPLATE))
-        except (exceptions.Error, requests.exceptions.HTTPError):
+        except Exception:
             pass
         assert not list(s.list())
 
-    # The `arg` param is not named `item_types` because that would hit
-    # https://bitbucket.org/pytest-dev/pytest/issue/745/
-    @pytest.mark.parametrize('arg,calls_num', [
-        (('VTODO',), 1),
-        (('VEVENT',), 1),
-        (('VTODO', 'VEVENT'), 2),
-        (('VTODO', 'VEVENT', 'VJOURNAL'), 3),
-        ((), 1)
-    ])
-    def test_item_types_performance(self, get_storage_args, arg, calls_num,
-                                    monkeypatch):
-        s = self.storage_class(item_types=arg, **get_storage_args())
-        old_parse = s._parse_prop_responses
-        calls = []
-
-        def new_parse(*a, **kw):
-            calls.append(None)
-            return old_parse(*a, **kw)
-
-        monkeypatch.setattr(s, '_parse_prop_responses', new_parse)
-        list(s.list())
-        assert len(calls) == calls_num
-
     @pytest.mark.xfail(dav_server == 'radicale',
                        reason='Radicale doesn\'t support timeranges.')
     def test_timerange_correctness(self, get_storage_args):
@@ -113,40 +86,19 @@ class TestCalDAVStorage(DAVStorageTests):
         (actual_href, _), = s.list()
         assert actual_href == expected_href
 
-    def test_invalid_resource(self, monkeypatch, get_storage_args):
-        calls = []
-        args = get_storage_args(collection=None)
-
-        def request(session, method, url, **kwargs):
-            assert url == args['url']
-            calls.append(None)
-
-            r = requests.Response()
-            r.status_code = 200
-            r._content = b'Hello World.'
-            return r
-
-        monkeypatch.setattr('requests.sessions.Session.request', request)
-
-        with pytest.raises(ValueError):
-            s = self.storage_class(**args)
-            list(s.list())
-        assert len(calls) == 1
-
     @pytest.mark.skipif(dav_server == 'icloud',
                         reason='iCloud only accepts VEVENT')
-    def test_item_types_general(self, s):
+    def test_item_types_general(self, get_storage_args):
+        args = get_storage_args()
+        s = self.storage_class(**args)
         event = s.upload(format_item(item_template=EVENT_TEMPLATE))[0]
         task = s.upload(format_item(item_template=TASK_TEMPLATE))[0]
-        s.item_types = ('VTODO', 'VEVENT')
 
-        def l():
+        for item_types, expected_items in [
-            return set(href for href, etag in s.list())
+            (('VTODO', 'VEVENT'), {event, task}),
-
+            (('VTODO',), {task}),
-        assert l() == {event, task}
+            (('VEVENT',), {event}),
-        s.item_types = ('VTODO',)
+        ]:
-        assert l() == {task}
+            args['item_types'] = item_types
-        s.item_types = ('VEVENT',)
+            s = self.storage_class(**args)
-        assert l() == {event}
+            assert set(href for href, etag in s.list()) == expected_items
-        s.item_types = ()
-        assert l() == {event, task}

tests/storage/servers/nextcloud/__init__.py

@@ -3,7 +3,7 @@ import requests
 import pytest
 
 
-port = os.environ.get('NEXTCLOUD_HOST', None) or 'localhost:8080'
+port = os.environ.get('NEXTCLOUD_HOST', None) or 'localhost:5000'
 user = os.environ.get('NEXTCLOUD_USER', None) or 'asdf'
 pwd = os.environ.get('NEXTCLOUD_PASS', None) or 'asdf'
 

tests/storage/servers/xandikos/__init__.py

@@ -1,35 +1,15 @@
 import pytest
 
-from xandikos.web import XandikosApp, XandikosBackend, WellknownRedirector
-
-import wsgi_intercept
-import wsgi_intercept.requests_intercept
-
 
 class ServerMixin(object):
     @pytest.fixture
     def get_storage_args(self, request, tmpdir, slow_create_collection):
-        tmpdir.mkdir('xandikos')
-        backend = XandikosBackend(path=str(tmpdir))
-        cup = '/user/'
-        backend.create_principal(cup, create_defaults=True)
-        app = XandikosApp(backend, cup)
-
-        app = WellknownRedirector(app, '/')
-
-        wsgi_intercept.requests_intercept.install()
-        wsgi_intercept.add_wsgi_intercept('127.0.0.1', 8080, lambda: app)
-
-        def teardown():
-            wsgi_intercept.remove_wsgi_intercept('127.0.0.1', 8080)
-            wsgi_intercept.requests_intercept.uninstall()
-        request.addfinalizer(teardown)
-
         def inner(collection='test'):
-            url = 'http://127.0.0.1:8080/'
+            url = 'http://127.0.0.1:5001/'
-            args = {'url': url, 'collection': collection}
+            args = {'url': url}
 
             if collection is not None:
-                args = self.storage_class.create_collection(**args)
+                args = slow_create_collection(self.storage_class, args,
+                                              collection)
             return args
         return inner

tests/storage/servers/xandikos/install.sh

@@ -1,13 +0,0 @@
-#!/bin/sh
-set -e
-
-pip install wsgi_intercept
-
-if [ "$REQUIREMENTS" = "release" ] || [ "$REQUIREMENTS" = "minimal" ]; then
-    pip install -U xandikos
-elif [ "$REQUIREMENTS" = "devel" ]; then
-    pip install -U git+https://github.com/jelmer/xandikos
-else
-    echo "Invalid REQUIREMENTS value"
-    false
-fi

tests/storage/test_http.py

@@ -1,122 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import pytest
-
-from requests import Response
-
-from vdirsyncer.exceptions import UserError
-from vdirsyncer.storage.http import HttpStorage, prepare_auth
-from vdirsyncer.vobject import Item
-
-
-def test_list(monkeypatch):
-    collection_url = 'http://127.0.0.1/calendar/collection.ics'
-
-    items = [
-        (u'BEGIN:VEVENT\n'
-         u'SUMMARY:Eine Kurzinfo\n'
-         u'DESCRIPTION:Beschreibung des Termines\n'
-         u'END:VEVENT'),
-        (u'BEGIN:VEVENT\n'
-         u'SUMMARY:Eine zweite Küèrzinfo\n'
-         u'DESCRIPTION:Beschreibung des anderen Termines\n'
-         u'BEGIN:VALARM\n'
-         u'ACTION:AUDIO\n'
-         u'TRIGGER:19980403T120000\n'
-         u'ATTACH;FMTTYPE=audio/basic:http://host.com/pub/ssbanner.aud\n'
-         u'REPEAT:4\n'
-         u'DURATION:PT1H\n'
-         u'END:VALARM\n'
-         u'END:VEVENT')
-    ]
-
-    responses = [
-        u'\n'.join([u'BEGIN:VCALENDAR'] + items + [u'END:VCALENDAR'])
-    ] * 2
-
-    def get(self, method, url, *a, **kw):
-        assert method == 'GET'
-        assert url == collection_url
-        r = Response()
-        r.status_code = 200
-        assert responses
-        r._content = responses.pop().encode('utf-8')
-        r.headers['Content-Type'] = 'text/calendar'
-        r.encoding = 'ISO-8859-1'
-        return r
-
-    monkeypatch.setattr('requests.sessions.Session.request', get)
-
-    s = HttpStorage(url=collection_url)
-
-    found_items = {}
-
-    for href, etag in s.list():
-        item, etag2 = s.get(href)
-        assert item.uid is not None
-        assert etag2 == etag
-        found_items[item.hash] = href
-
-    expected = set(Item(u'BEGIN:VCALENDAR\n' + x + '\nEND:VCALENDAR').hash
-                   for x in items)
-
-    assert set(found_items) == expected
-
-    for href, etag in s.list():
-        item, etag2 = s.get(href)
-        assert item.uid is not None
-        assert etag2 == etag
-        assert found_items[item.hash] == href
-
-
-def test_readonly_param():
-    url = 'http://example.com/'
-    with pytest.raises(ValueError):
-        HttpStorage(url=url, read_only=False)
-
-    a = HttpStorage(url=url, read_only=True).read_only
-    b = HttpStorage(url=url, read_only=None).read_only
-    assert a is b is True
-
-
-def test_prepare_auth():
-    assert prepare_auth(None, '', '') is None
-
-    assert prepare_auth(None, 'user', 'pwd') == ('user', 'pwd')
-    assert prepare_auth('basic', 'user', 'pwd') == ('user', 'pwd')
-
-    with pytest.raises(ValueError) as excinfo:
-        assert prepare_auth('basic', '', 'pwd')
-    assert 'you need to specify username and password' in \
-        str(excinfo.value).lower()
-
-    from requests.auth import HTTPDigestAuth
-    assert isinstance(prepare_auth('digest', 'user', 'pwd'),
-                      HTTPDigestAuth)
-
-    with pytest.raises(ValueError) as excinfo:
-        prepare_auth('ladida', 'user', 'pwd')
-
-    assert 'unknown authentication method' in str(excinfo.value).lower()
-
-
-def test_prepare_auth_guess(monkeypatch):
-    import requests_toolbelt.auth.guess
-
-    assert isinstance(prepare_auth('guess', 'user', 'pwd'),
-                      requests_toolbelt.auth.guess.GuessAuth)
-
-    monkeypatch.delattr(requests_toolbelt.auth.guess, 'GuessAuth')
-
-    with pytest.raises(UserError) as excinfo:
-        prepare_auth('guess', 'user', 'pwd')
-
-    assert 'requests_toolbelt is too old' in str(excinfo.value).lower()
-
-
-def test_verify_false_disallowed():
-    with pytest.raises(ValueError) as excinfo:
-        HttpStorage(url='http://example.com', verify=False)
-
-    assert 'forbidden' in str(excinfo.value).lower()
-    assert 'consider setting verify_fingerprint' in str(excinfo.value).lower()

tests/storage/test_http_with_singlefile.py

@@ -1,81 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import pytest
-
-from requests import Response
-
-import vdirsyncer.storage.http
-from vdirsyncer.storage.base import Storage
-from vdirsyncer.storage.singlefile import SingleFileStorage
-
-from . import StorageTests
-
-
-class CombinedStorage(Storage):
-    '''A subclass of HttpStorage to make testing easier. It supports writes via
-    SingleFileStorage.'''
-    _repr_attributes = ('url', 'path')
-    storage_name = 'http_and_singlefile'
-
-    def __init__(self, url, path, **kwargs):
-        if kwargs.get('collection', None) is not None:
-            raise ValueError()
-
-        super(CombinedStorage, self).__init__(**kwargs)
-        self.url = url
-        self.path = path
-        self._reader = vdirsyncer.storage.http.HttpStorage(url=url)
-        self._reader._ignore_uids = False
-        self._writer = SingleFileStorage(path=path)
-
-    def list(self, *a, **kw):
-        return self._reader.list(*a, **kw)
-
-    def get(self, *a, **kw):
-        self.list()
-        return self._reader.get(*a, **kw)
-
-    def upload(self, *a, **kw):
-        return self._writer.upload(*a, **kw)
-
-    def update(self, *a, **kw):
-        return self._writer.update(*a, **kw)
-
-    def delete(self, *a, **kw):
-        return self._writer.delete(*a, **kw)
-
-
-class TestHttpStorage(StorageTests):
-    storage_class = CombinedStorage
-    supports_collections = False
-    supports_metadata = False
-
-    @pytest.fixture(autouse=True)
-    def setup_tmpdir(self, tmpdir, monkeypatch):
-        self.tmpfile = str(tmpdir.ensure('collection.txt'))
-
-        def _request(method, url, *args, **kwargs):
-            assert method == 'GET'
-            assert url == 'http://localhost:123/collection.txt'
-            assert 'vdirsyncer' in kwargs['headers']['User-Agent']
-            r = Response()
-            r.status_code = 200
-            try:
-                with open(self.tmpfile, 'rb') as f:
-                    r._content = f.read()
-            except IOError:
-                r._content = b''
-
-            r.headers['Content-Type'] = 'text/calendar'
-            r.encoding = 'utf-8'
-            return r
-
-        monkeypatch.setattr(vdirsyncer.storage.http, 'request', _request)
-
-    @pytest.fixture
-    def get_storage_args(self):
-        def inner(collection=None):
-            assert collection is None
-            return {'url': 'http://localhost:123/collection.txt',
-                    'path': self.tmpfile}
-        return inner

vdirsyncer/exceptions.py

@@ -83,3 +83,7 @@ class CollectionRequired(Error):
 
 class VobjectParseError(Error, ValueError):
     '''The parsed vobject is invalid.'''
+
+
+class UnsupportedVobjectError(Error, ValueError):
+    '''The server rejected the vobject because of its type'''

vdirsyncer/native.py

@@ -3,6 +3,8 @@ import shippai
 from . import exceptions
 from ._native import ffi, lib
 
+lib.vdirsyncer_init_logger()
+
 
 errors = shippai.Shippai(ffi, lib)
 
@@ -31,3 +33,7 @@ def check_error(e):
         raise exceptions.AlreadyExistingError(e)
     except errors.Error.WrongEtag as e:
         raise exceptions.WrongEtagError(e)
+    except errors.Error.ReadOnly as e:
+        raise exceptions.ReadOnlyError(e)
+    except errors.Error.UnsupportedVobject as e:
+        raise exceptions.UnsupportedVobjectError(e)

vdirsyncer/storage/_rust.py

@@ -46,7 +46,7 @@ class RustStorageMixin:
             result, native.lib.vdirsyncer_free_storage_upload_result)
         href = native.string_rv(result.href)
         etag = native.string_rv(result.etag)
-        return href, etag
+        return href, etag or None
 
     def update(self, href, item, etag):
         href = href.encode('utf-8')
@@ -54,7 +54,7 @@ class RustStorageMixin:
         e = native.get_error_pointer()
         etag = self._native('update')(href, item._native, etag, e)
         native.check_error(e)
-        return native.string_rv(etag)
+        return native.string_rv(etag) or None
 
     def delete(self, href, etag):
         href = href.encode('utf-8')

vdirsyncer/storage/dav.py

@@ -11,10 +11,10 @@ import requests
 from requests.exceptions import HTTPError
 
 from .base import Storage, normalize_meta_value
-from .. import exceptions, http, utils
+from ._rust import RustStorageMixin
+from .. import exceptions, http, native, utils
 from ..http import USERAGENT, prepare_auth, \
     prepare_client_cert, prepare_verify
-from ..vobject import Item
 
 
 dav_logger = logging.getLogger(__name__)
@@ -41,22 +41,6 @@ def _contains_quoted_reserved_chars(x):
     return False
 
 
-def _assert_multistatus_success(r):
-    # Xandikos returns a multistatus on PUT.
-    try:
-        root = _parse_xml(r.content)
-    except InvalidXMLResponse:
-        return
-    for status in root.findall('.//{DAV:}status'):
-        parts = status.text.strip().split()
-        try:
-            st = int(parts[1])
-        except (ValueError, IndexError):
-            continue
-        if st < 200 or st >= 400:
-            raise HTTPError('Server error: {}'.format(st))
-
-
 def _normalize_href(base, href):
     '''Normalize the href to be a path only relative to hostname and
     schema.'''
@@ -396,7 +380,7 @@ class DAVSession(object):
         }
 
 
-class DAVStorage(Storage):
+class DAVStorage(RustStorageMixin, Storage):
     # the file extension of items. Useful for testing against radicale.
     fileext = None
     # mimetype of items
@@ -443,200 +427,9 @@ class DAVStorage(Storage):
     def _normalize_href(self, *args, **kwargs):
         return _normalize_href(self.session.url, *args, **kwargs)
 
-    def _get_href(self, item):
-        href = utils.generate_href(item.ident)
-        return self._normalize_href(href + self.fileext)
-
     def _is_item_mimetype(self, mimetype):
         return _fuzzy_matches_mimetype(self.item_mimetype, mimetype)
 
-    def get(self, href):
-        ((actual_href, item, etag),) = self.get_multi([href])
-        assert href == actual_href
-        return item, etag
-
-    def get_multi(self, hrefs):
-        hrefs = set(hrefs)
-        href_xml = []
-        for href in hrefs:
-            if href != self._normalize_href(href):
-                raise exceptions.NotFoundError(href)
-            href_xml.append('<D:href>{}</D:href>'.format(href))
-        if not href_xml:
-            return ()
-
-        data = self.get_multi_template \
-            .format(hrefs='\n'.join(href_xml)).encode('utf-8')
-        response = self.session.request(
-            'REPORT',
-            '',
-            data=data,
-            headers=self.session.get_default_headers()
-        )
-        root = _parse_xml(response.content)  # etree only can handle bytes
-        rv = []
-        hrefs_left = set(hrefs)
-        for href, etag, prop in self._parse_prop_responses(root):
-            raw = prop.find(self.get_multi_data_query)
-            if raw is None:
-                dav_logger.warning('Skipping {}, the item content is missing.'
-                                   .format(href))
-                continue
-
-            raw = raw.text or u''
-
-            if isinstance(raw, bytes):
-                raw = raw.decode(response.encoding)
-            if isinstance(etag, bytes):
-                etag = etag.decode(response.encoding)
-
-            try:
-                hrefs_left.remove(href)
-            except KeyError:
-                if href in hrefs:
-                    dav_logger.warning('Server sent item twice: {}'
-                                       .format(href))
-                else:
-                    dav_logger.warning('Server sent unsolicited item: {}'
-                                       .format(href))
-            else:
-                rv.append((href, Item(raw), etag))
-        for href in hrefs_left:
-            raise exceptions.NotFoundError(href)
-        return rv
-
-    def _put(self, href, item, etag):
-        headers = self.session.get_default_headers()
-        headers['Content-Type'] = self.item_mimetype
-        if etag is None:
-            headers['If-None-Match'] = '*'
-        else:
-            headers['If-Match'] = etag
-
-        response = self.session.request(
-            'PUT',
-            href,
-            data=item.raw.encode('utf-8'),
-            headers=headers
-        )
-
-        _assert_multistatus_success(response)
-
-        # The server may not return an etag under certain conditions:
-        #
-        #   An origin server MUST NOT send a validator header field (Section
-        #   7.2), such as an ETag or Last-Modified field, in a successful
-        #   response to PUT unless the request's representation data was saved
-        #   without any transformation applied to the body (i.e., the
-        #   resource's new representation data is identical to the
-        #   representation data received in the PUT request) and the validator
-        #   field value reflects the new representation.
-        #
-        # -- https://tools.ietf.org/html/rfc7231#section-4.3.4
-        #
-        # In such cases we return a constant etag. The next synchronization
-        # will then detect an etag change and will download the new item.
-        etag = response.headers.get('etag', None)
-        href = self._normalize_href(response.url)
-        return href, etag
-
-    def update(self, href, item, etag):
-        if etag is None:
-            raise ValueError('etag must be given and must not be None.')
-        href, etag = self._put(self._normalize_href(href), item, etag)
-        return etag
-
-    def upload(self, item):
-        href = self._get_href(item)
-        return self._put(href, item, None)
-
-    def delete(self, href, etag):
-        href = self._normalize_href(href)
-        headers = self.session.get_default_headers()
-        headers.update({
-            'If-Match': etag
-        })
-
-        self.session.request(
-            'DELETE',
-            href,
-            headers=headers
-        )
-
-    def _parse_prop_responses(self, root, handled_hrefs=None):
-        if handled_hrefs is None:
-            handled_hrefs = set()
-        for response in root.iter('{DAV:}response'):
-            href = response.find('{DAV:}href')
-            if href is None:
-                dav_logger.error('Skipping response, href is missing.')
-                continue
-
-            href = self._normalize_href(href.text)
-
-            if href in handled_hrefs:
-                # Servers that send duplicate hrefs:
-                # - Zimbra
-                #   https://github.com/pimutils/vdirsyncer/issues/88
-                # - Davmail
-                #   https://github.com/pimutils/vdirsyncer/issues/144
-                dav_logger.warning('Skipping identical href: {!r}'
-                                   .format(href))
-                continue
-
-            props = response.findall('{DAV:}propstat/{DAV:}prop')
-            if props is None or not len(props):
-                dav_logger.debug('Skipping {!r}, properties are missing.'
-                                 .format(href))
-                continue
-            else:
-                props = _merge_xml(props)
-
-            if props.find('{DAV:}resourcetype/{DAV:}collection') is not None:
-                dav_logger.debug('Skipping {!r}, is collection.'.format(href))
-                continue
-
-            etag = getattr(props.find('{DAV:}getetag'), 'text', '')
-            if not etag:
-                dav_logger.debug('Skipping {!r}, etag property is missing.'
-                                 .format(href))
-                continue
-
-            contenttype = getattr(props.find('{DAV:}getcontenttype'),
-                                  'text', None)
-            if not self._is_item_mimetype(contenttype):
-                dav_logger.debug('Skipping {!r}, {!r} != {!r}.'
-                                 .format(href, contenttype,
-                                         self.item_mimetype))
-                continue
-
-            handled_hrefs.add(href)
-            yield href, etag, props
-
-    def list(self):
-        headers = self.session.get_default_headers()
-        headers['Depth'] = '1'
-
-        data = '''<?xml version="1.0" encoding="utf-8" ?>
-            <D:propfind xmlns:D="DAV:">
-                <D:prop>
-                    <D:resourcetype/>
-                    <D:getcontenttype/>
-                    <D:getetag/>
-                </D:prop>
-            </D:propfind>
-            '''.encode('utf-8')
-
-        # We use a PROPFIND request instead of addressbook-query due to issues
-        # with Zimbra. See https://github.com/pimutils/vdirsyncer/issues/83
-        response = self.session.request('PROPFIND', '', data=data,
-                                        headers=headers)
-        root = _parse_xml(response.content)
-
-        rv = self._parse_prop_responses(root)
-        for href, etag, _prop in rv:
-            yield href, etag
-
     def get_meta(self, key):
         try:
             tagname, namespace = self._property_table[key]
@@ -734,7 +527,7 @@ class CalDAVStorage(DAVStorage):
         if not isinstance(item_types, (list, tuple)):
             raise exceptions.UserError('item_types must be a list.')
 
-        self.item_types = tuple(item_types)
+        self.item_types = tuple(x.upper() for x in item_types)
         if (start_date is None) != (end_date is None):
             raise exceptions.UserError('If start_date is given, '
                                        'end_date has to be given too.')
@@ -749,77 +542,22 @@ class CalDAVStorage(DAVStorage):
                  if isinstance(end_date, (bytes, str))
                  else end_date)
 
-    @staticmethod
+        self._native_storage = native.ffi.gc(
-    def _get_list_filters(components, start, end):
+            native.lib.vdirsyncer_init_caldav(
-        caldavfilter = '''
+                kwargs['url'].encode('utf-8'),
-            <C:comp-filter name="VCALENDAR">
+                kwargs.get('username', '').encode('utf-8'),
-                <C:comp-filter name="{component}">
+                kwargs.get('password', '').encode('utf-8'),
-                    {timefilter}
+                kwargs.get('useragent', '').encode('utf-8'),
-                </C:comp-filter>
+                kwargs.get('verify_cert', '').encode('utf-8'),
-            </C:comp-filter>
+                kwargs.get('auth_cert', '').encode('utf-8'),
-            '''
+                int(self.start_date.timestamp()) if self.start_date else -1,
-
+                int(self.end_date.timestamp()) if self.end_date else -1,
-        timefilter = ''
+                'VEVENT' in item_types,
-
+                'VJOURNAL' in item_types,
-        if start is not None and end is not None:
+                'VTODO' in item_types
-            start = start.strftime(CALDAV_DT_FORMAT)
+            ),
-            end = end.strftime(CALDAV_DT_FORMAT)
+            native.lib.vdirsyncer_storage_free
-
+        )
-            timefilter = ('<C:time-range start="{start}" end="{end}"/>'
-                          .format(start=start, end=end))
-            if not components:
-                components = ('VTODO', 'VEVENT')
-
-        for component in components:
-            yield caldavfilter.format(component=component,
-                                      timefilter=timefilter)
-
-    def list(self):
-        caldavfilters = list(self._get_list_filters(
-            self.item_types,
-            self.start_date,
-            self.end_date
-        ))
-        if not caldavfilters:
-            # If we don't have any filters (which is the default), taking the
-            # risk of sending a calendar-query is not necessary. There doesn't
-            # seem to be a widely-usable way to send calendar-queries with the
-            # same semantics as a PROPFIND request... so why not use PROPFIND
-            # instead?
-            #
-            # See https://github.com/dmfs/tasks/issues/118 for backstory.
-            yield from DAVStorage.list(self)
-            return
-
-        data = '''<?xml version="1.0" encoding="utf-8" ?>
-            <C:calendar-query xmlns:D="DAV:"
-                xmlns:C="urn:ietf:params:xml:ns:caldav">
-                <D:prop>
-                    <D:getcontenttype/>
-                    <D:getetag/>
-                </D:prop>
-                <C:filter>
-                {caldavfilter}
-                </C:filter>
-            </C:calendar-query>'''
-
-        headers = self.session.get_default_headers()
-        # https://github.com/pimutils/vdirsyncer/issues/166
-        # The default in CalDAV's calendar-queries is 0, but the examples use
-        # an explicit value of 1 for querying items. it is extremely unclear in
-        # the spec which values from WebDAV are actually allowed.
-        headers['Depth'] = '1'
-
-        handled_hrefs = set()
-
-        for caldavfilter in caldavfilters:
-            xml = data.format(caldavfilter=caldavfilter).encode('utf-8')
-            response = self.session.request('REPORT', '', data=xml,
-                                            headers=headers)
-            root = _parse_xml(response.content)
-            rv = self._parse_prop_responses(root, handled_hrefs)
-            for href, etag, _prop in rv:
-                yield href, etag
 
 
 class CardDAVStorage(DAVStorage):
@@ -839,3 +577,18 @@ class CardDAVStorage(DAVStorage):
             </C:addressbook-multiget>'''
 
     get_multi_data_query = '{urn:ietf:params:xml:ns:carddav}address-data'
+
+    def __init__(self, **kwargs):
+        self._native_storage = native.ffi.gc(
+            native.lib.vdirsyncer_init_carddav(
+                kwargs['url'].encode('utf-8'),
+                kwargs.get('username', '').encode('utf-8'),
+                kwargs.get('password', '').encode('utf-8'),
+                kwargs.get('useragent', '').encode('utf-8'),
+                kwargs.get('verify_cert', '').encode('utf-8'),
+                kwargs.get('auth_cert', '').encode('utf-8')
+            ),
+            native.lib.vdirsyncer_storage_free
+        )
+
+        super(CardDAVStorage, self).__init__(**kwargs)

vdirsyncer/storage/http.py

@@ -1,15 +1,12 @@
 # -*- coding: utf-8 -*-
 
-import urllib.parse as urlparse
-
 from .base import Storage
-from .. import exceptions
+from ._rust import RustStorageMixin
-from ..http import USERAGENT, prepare_auth, \
+from .. import exceptions, native
-    prepare_client_cert, prepare_verify, request
+from ..http import USERAGENT
-from ..vobject import Item, split_collection
 
 
-class HttpStorage(Storage):
+class HttpStorage(RustStorageMixin, Storage):
     storage_name = 'http'
     read_only = True
     _repr_attributes = ('username', 'url')
@@ -18,49 +15,27 @@ class HttpStorage(Storage):
     # Required for tests.
     _ignore_uids = True
 
-    def __init__(self, url, username='', password='', verify=True, auth=None,
+    def __init__(self, url, username='', password='', useragent=USERAGENT,
-                 useragent=USERAGENT, verify_fingerprint=None, auth_cert=None,
+                 verify_cert=None, auth_cert=None, **kwargs):
-                 **kwargs):
+        if kwargs.get('collection') is not None:
+            raise exceptions.UserError('HttpStorage does not support '
+                                       'collections.')
+
+        assert auth_cert is None, "not yet supported"
+
         super(HttpStorage, self).__init__(**kwargs)
 
-        self._settings = {
+        self._native_storage = native.ffi.gc(
-            'auth': prepare_auth(auth, username, password),
+            native.lib.vdirsyncer_init_http(
-            'cert': prepare_client_cert(auth_cert),
+                url.encode('utf-8'),
-            'latin1_fallback': False,
+                (username or "").encode('utf-8'),
-        }
+                (password or "").encode('utf-8'),
-        self._settings.update(prepare_verify(verify, verify_fingerprint))
+                (useragent or "").encode('utf-8'),
+                (verify_cert or "").encode('utf-8'),
+                (auth_cert or "").encode('utf-8')
+            ),
+            native.lib.vdirsyncer_storage_free
+        )
 
-        self.username, self.password = username, password
+        self.username = username
-        self.useragent = useragent
-
-        collection = kwargs.get('collection')
-        if collection is not None:
-            url = urlparse.urljoin(url, collection)
         self.url = url
-        self.parsed_url = urlparse.urlparse(self.url)
-
-    def _default_headers(self):
-        return {'User-Agent': self.useragent}
-
-    def list(self):
-        r = request('GET', self.url, headers=self._default_headers(),
-                    **self._settings)
-        self._items = {}
-
-        for item in split_collection(r.text):
-            item = Item(item)
-            if self._ignore_uids:
-                item = item.with_uid(item.hash)
-
-            self._items[item.ident] = item, item.hash
-
-        return ((href, etag) for href, (item, etag) in self._items.items())
-
-    def get(self, href):
-        if self._items is None:
-            self.list()
-
-        try:
-            return self._items[href]
-        except KeyError:
-            raise exceptions.NotFoundError(href)