Add atomicwrites dependency

This commit is contained in:
Markus Unterwaditzer 2015-01-29 12:12:40 +01:00
parent a2e71cc4ca
commit 27f5d54240
6 changed files with 17 additions and 78 deletions

View file

@ -39,7 +39,8 @@ setup(
'lxml>=3.0',
'icalendar>=3.6',
# https://github.com/sigmavirus24/requests-toolbelt/pull/28
'requests_toolbelt>=0.3.0'
'requests_toolbelt>=0.3.0',
'atomicwrites'
],
extras_require={'keyring': ['keyring']}
)

View file

@ -220,17 +220,3 @@ def test_request_ssl(httpsserver):
utils.request('GET', httpsserver.url,
verify_fingerprint=''.join(reversed(sha1)))
assert 'Fingerprints did not match' in str(excinfo.value)
def test_atomic_write(tmpdir):
x = utils.atomic_write
fname = tmpdir.join('ha')
for i in range(2):
with x(str(fname), binary=False, overwrite=True) as f:
f.write('hoho')
with pytest.raises(OSError):
with x(str(fname), binary=False, overwrite=False) as f:
f.write('haha')
assert fname.read() == 'hoho'

View file

@ -375,7 +375,7 @@ def save_status(base_path, pair, collection=None, data_type=None, data=None):
if e.errno != errno.EEXIST:
raise
with atomic_write(path, binary=False, overwrite=True) as f:
with atomic_write(path, mode='w', overwrite=True) as f:
json.dump(data, f)

View file

@ -5,7 +5,8 @@ import os
from .base import Item, Storage
from .. import exceptions, log
from ..utils import atomic_write, checkdir, expand_path, get_etag_from_file
from ..utils import atomic_write, checkdir, expand_path, \
get_etag_from_file, get_etag_from_fileobject
from ..utils.compat import text_type
logger = log.get(__name__)
@ -99,9 +100,9 @@ class FilesystemStorage(Storage):
raise TypeError('item.raw must be a unicode string.')
try:
with atomic_write(fpath, binary=True, overwrite=False) as f:
with atomic_write(fpath, mode='wb', overwrite=False) as f:
f.write(item.raw.encode(self.encoding))
return href, f.get_etag()
return href, get_etag_from_fileobject(f)
except OSError as e:
import errno
if e.errno == errno.EEXIST:
@ -123,9 +124,9 @@ class FilesystemStorage(Storage):
if not isinstance(item.raw, text_type):
raise TypeError('item.raw must be a unicode string.')
with atomic_write(fpath, binary=True, overwrite=True) as f:
with atomic_write(fpath, mode='wb', overwrite=True) as f:
f.write(item.raw.encode(self.encoding))
return f.get_etag()
return get_etag_from_fileobject(f)
def delete(self, href, etag):
fpath = self._get_filepath(href)

View file

@ -159,7 +159,7 @@ class SingleFileStorage(Storage):
(item.raw for item, etag in itervalues(self._items)),
)
try:
with atomic_write(self.path, binary=True, overwrite=True) as f:
with atomic_write(self.path, mode='wb', overwrite=True) as f:
f.write(text.encode(self.encoding))
finally:
self._items = None

View file

@ -4,6 +4,8 @@ import os
import threading
import uuid
from atomicwrites import atomic_write
import requests
from requests.packages.urllib3.poolmanager import PoolManager
@ -235,66 +237,15 @@ def request(method, url, session=None, latin1_fallback=True,
return r
class atomic_write(object):
'''
A helper class for performing atomic writes.
Usage::
with safe_write(fpath, binary=False, overwrite=False) as f:
f.write('hohoho')
:param fpath: The destination filepath. May or may not exist.
:param binary: Whether binary write mode should be used.
:param overwrite: If set to false, an error is raised if ``fpath`` exists.
This should still be atomic.
:param tmppath: An alternative tmpfile location. Must reside on the same
filesystem to be atomic.
'''
f = None
tmppath = None
fpath = None
mode = None
def __init__(self, fpath, binary, overwrite, tmppath=None):
if not tmppath:
base = os.path.dirname(fpath)
tmppath = os.path.join(base, str(uuid.uuid4()) + '.tmp')
self.fpath = fpath
self.binary = binary
self.overwrite = overwrite
self.tmppath = tmppath
def __enter__(self):
self.f = f = open(self.tmppath, 'wb' if self.binary else 'w')
self.write = f.write
return self
def __exit__(self, cls, value, tb):
self.f.close()
if cls is None:
self._commit()
else:
os.remove(self.tmppath)
def _commit(self):
if self.overwrite:
os.rename(self.tmppath, self.fpath) # atomic
else:
os.link(self.tmppath, self.fpath) # atomic, fails if file exists
os.unlink(self.tmppath) # doesn't matter if atomic
def get_etag(self):
self.f.flush()
return get_etag_from_file(self.tmppath)
def get_etag_from_file(fpath):
return '{:.9f}'.format(os.path.getmtime(fpath))
def get_etag_from_fileobject(f):
f.flush()
return get_etag_from_file(f.name)
def get_class_init_specs(cls, stop_at=object):
if cls is stop_at:
return ()