sjs/imapbackup
1
0
Fork 0
mirror of https://github.com/samsonjs/imapbackup.git synced 2026-04-27 14:57:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #11 from ezzra/pep8

Browse source 
make code PEP8 compliant
This commit is contained in:
Rui Carmo 2020-06-04 17:08:58 +01:00 committed by GitHub
commit 4fadc21c74
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 699 additions and 656 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
imapbackup.py
View file

@ -43,15 +43,27 @@ __contributors__ = "jwagnerhki, Bob Ippolito, Michael Leonhard, Giuseppe Scrivan
# - Support host:port
# - Cleaned up code using PyLint to identify problems
# pylint -f html --indent-string=" " --max-line-length=90 imapbackup.py > report.html
import getpass, os, gc, sys, time, platform, getopt
import mailbox, imaplib, socket
import re, hashlib, gzip, bz2
import getpass
import os
import gc
import sys
import time
import platform
import getopt
import mailbox
import imaplib
import socket
import re
import hashlib
import gzip
import bz2
class SkipFolderException(Exception):
"""Indicates aborting processing of current folder, continue with next folder."""
pass
class Spinner:
"""Prints out message with cute spinner, indicating progress"""
@ -79,12 +91,13 @@ class Spinner:
sys.stdout.write("\r" + self.message)
sys.stdout.flush()
def pretty_byte_count(num):
"""Converts integer into a human friendly count of bytes, eg: 12.243 MB"""
if num == 1:
return "1 byte"
elif num < 1024:
return "%s bytes" % (num)
return "%s bytes" % num
elif num < 1048576:
return "%.2f KB" % (num/1024.0)
elif num < 1073741824:
@ -124,8 +137,6 @@ def string_from_file(value):
return content.read().strip()
def download_messages(server, filename, messages, config):
"""Download messages from folder and append to mailbox"""
@ -174,7 +185,7 @@ def download_messages(server, filename, messages, config):
# fetch message
typ, data = server.fetch(messages[msg_id], "RFC822")
assert('OK' == typ)
text = data[0][1].strip().replace('\r','')
text = data[0][1].strip().replace('\r', '')
if config['thunderbird']:
# This avoids Thunderbird mistaking a line starting "From " as the start
# of a new message. _Might_ also apply to other mail lients - unknown
@ -195,6 +206,7 @@ def download_messages(server, filename, messages, config):
print ": %s total, %s for largest message" % (pretty_byte_count(total),
pretty_byte_count(biggest))
def scan_file(filename, compress, overwrite, nospinner):
"""Gets IDs of messages in the specified mbox file"""
# file will be overwritten
@ -205,18 +217,18 @@ def scan_file(filename, compress, overwrite, nospinner):
# file doesn't exist
if not os.path.exists(filename):
print "File %s: not found" % (filename)
print "File %s: not found" % filename
return []
spinner = Spinner("File %s" % (filename), nospinner)
spinner = Spinner("File %s" % filename, nospinner)
# open the file
if compress == 'gzip':
mbox = gzip.GzipFile(filename,'rb')
mbox = gzip.GzipFile(filename, 'rb')
elif compress == 'bzip2':
mbox = bz2.BZ2File(filename,'rb')
mbox = bz2.BZ2File(filename, 'rb')
else:
mbox = file(filename,'rb')
mbox = file(filename, 'rb')
messages = {}
@ -254,20 +266,22 @@ def scan_file(filename, compress, overwrite, nospinner):
print ": %d messages" % (len(messages.keys()))
return messages
def scan_folder(server, foldername, nospinner):
"""Gets IDs of messages in the specified folder, returns id:num dict"""
messages = {}
spinner = Spinner("Folder %s" % (foldername), nospinner)
spinner = Spinner("Folder %s" % foldername, nospinner)
try:
typ, data = server.select(foldername, readonly=True)
if 'OK' != typ:
raise SkipFolderException("SELECT failed: %s" % (data))
raise SkipFolderException("SELECT failed: %s" % data)
num_msgs = int(data[0])
# each message
for num in range(1, num_msgs+1):
# Retrieve Message-Id, making sure we don't mark all messages as read
typ, data = server.fetch(num, '(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])')
typ, data = server.fetch(
num, '(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])')
if 'OK' != typ:
raise SkipFolderException("FETCH %s failed: %s" % (num, data))
@ -282,12 +296,15 @@ def scan_folder(server, foldername, nospinner):
except (IndexError, AttributeError):
# Some messages may have no Message-Id, so we'll synthesise one
# (this usually happens with Sent, Drafts and .Mac news)
typ, data = server.fetch(num, '(BODY[HEADER.FIELDS (FROM TO CC DATE SUBJECT)])')
typ, data = server.fetch(
num, '(BODY[HEADER.FIELDS (FROM TO CC DATE SUBJECT)])')
if 'OK' != typ:
raise SkipFolderException("FETCH %s failed: %s" % (num, data))
raise SkipFolderException(
"FETCH %s failed: %s" % (num, data))
header = data[0][1].strip()
header = header.replace('\r\n','\t')
messages['<' + UUID + '.' + hashlib.sha1(header).hexdigest() + '>'] = num
header = header.replace('\r\n', '\t')
messages['<' + UUID + '.' +
hashlib.sha1(header).hexdigest() + '>'] = num
spinner.spin()
finally:
spinner.stop()
@ -297,6 +314,7 @@ def scan_folder(server, foldername, nospinner):
print "%d messages" % (len(messages.keys()))
return messages
def parse_paren_list(row):
"""Parses the nested list of attributes at the start of a LIST response"""
# eat starting paren
@ -317,7 +335,7 @@ def parse_paren_list(row):
# consume name attribute
else:
match = name_attrib_re.search(row)
assert(match != None)
assert(match is not None)
name_attrib = row[match.start():match.end()]
row = row[match.end():]
#print "MATCHED '%s' '%s'" % (name_attrib, row)
@ -331,11 +349,13 @@ def parse_paren_list(row):
# done!
return result, row
def parse_string_list(row):
"""Parses the quoted and unquoted strings at the end of a LIST response"""
slist = re.compile('\s*(?:"([^"]+)")\s*|\s*(\S+)\s*').split(row)
return [s for s in slist if s]
def parse_list(row):
"""Prases response of LIST command into a list"""
row = row.strip()
@ -344,6 +364,7 @@ def parse_list(row):
assert(len(string_list) == 2)
return [paren_list] + string_list
def get_hierarchy_delimiter(server):
"""Queries the imapd for the hierarchy delimiter, eg. '.' in INBOX.Sent"""
# see RFC 3501 page 39 paragraph 4
@ -357,6 +378,7 @@ def get_hierarchy_delimiter(server):
hierarchy_delim = '.'
return hierarchy_delim
def get_names(server, compress, thunderbird, nospinner):
"""Get list of folders, returns [(FolderName,FileName)]"""
@ -377,11 +399,11 @@ def get_names(server, compress, thunderbird, nospinner):
for row in data:
lst = parse_list(row)
foldername = lst[2]
suffix = {'none':'', 'gzip':'.gz', 'bzip2':'.bz2'}[compress]
suffix = {'none': '', 'gzip': '.gz', 'bzip2': '.bz2'}[compress]
if thunderbird:
filename = '.sbd/'.join(foldername.split(delim)) + suffix
if filename.startswith("INBOX"):
filename = filename.replace("INBOX","Inbox")
filename = filename.replace("INBOX", "Inbox")
else:
filename = '.'.join(foldername.split(delim)) + '.mbox' + suffix
# print "\n*** Folder:", foldername # *DEBUG
@ -393,6 +415,7 @@ def get_names(server, compress, thunderbird, nospinner):
print ": %s folders" % (len(names))
return names
def print_usage():
"""Prints usage, exits"""
# " "
@ -418,6 +441,7 @@ def print_usage():
print "\nNOTE: mbox files are created in the current working directory."
sys.exit(2)
def process_cline():
"""Uses getopt to process command line, returns (config, warnings, errors)"""
# read command line
@ -431,8 +455,8 @@ def process_cline():
print_usage()
warnings = []
config = {'compress':'none', 'overwrite':False, 'usessl':False,
'thunderbird':False, 'nospinner':False}
config = {'compress': 'none', 'overwrite': False, 'usessl': False,
'thunderbird': False, 'nospinner': False}
errors = []
# empty command line
@ -488,18 +512,20 @@ def process_cline():
errors.append("Unknown argument: " + arg)
# done processing command line
return (config, warnings, errors)
return config, warnings, errors
def check_config(config, warnings, errors):
"""Checks the config for consistency, returns (config, warnings, errors)"""
if config['compress'] == 'bzip2' and config['overwrite'] == False:
errors.append("Cannot append new messages to mbox.bz2 files. Please specify -y.")
if config['compress'] == 'gzip' and config['overwrite'] == False:
if config['compress'] == 'bzip2' and config['overwrite'] is False:
errors.append(
"Cannot append new messages to mbox.bz2 files. Please specify -y.")
if config['compress'] == 'gzip' and config['overwrite'] is False:
warnings.append(
"Appending new messages to mbox.gz files is very slow. Please Consider\n"
" using -y and compressing the files yourself with gzip -9 *.mbox")
if 'server' not in config :
if 'server' not in config:
errors.append("No server specified.")
if 'user' not in config:
errors.append("No username specified.")
@ -508,7 +534,8 @@ def check_config(config, warnings, errors):
if 'keyfilename' in config and not config['usessl']:
errors.append("Key specified without SSL. Please use -e or --ssl.")
if 'certfilename' in config and not config['usessl']:
errors.append("Certificate specified without SSL. Please use -e or --ssl.")
errors.append(
"Certificate specified without SSL. Please use -e or --ssl.")
if 'server' in config and ':' in config['server']:
# get host and port strings
bits = config['server'].split(':', 1)
@ -521,16 +548,19 @@ def check_config(config, warnings, errors):
raise ValueError
config['port'] = port
except ValueError:
errors.append("Invalid port. Port must be an integer between 0 and 65535.")
errors.append(
"Invalid port. Port must be an integer between 0 and 65535.")
if 'timeout' in config:
try:
timeout = int(config['timeout'])
if timeout <= 0:
raise ValueError
config['timeout']=timeout
config['timeout'] = timeout
except ValueError:
errors.append("Invalid timeout value. Must be an integer greater than 0.")
return (config, warnings, errors)
errors.append(
"Invalid timeout value. Must be an integer greater than 0.")
return config, warnings, errors
def get_config():
"""Gets config from command line and console, returns config"""
@ -564,17 +594,18 @@ def get_config():
config['pass'] = getpass.getpass()
# defaults
if not 'port' in config:
if 'port' not in config:
if config['usessl']:
config['port'] = 993
else:
config['port'] = 143
if not 'timeout' in config:
if 'timeout' not in config:
config['timeout'] = 60
# done!
return config
def connect_and_login(config):
"""Connects to the server and logs in. Returns IMAP4 object."""
try:
@ -583,16 +614,19 @@ def connect_and_login(config):
socket.setdefaulttimeout(config['timeout'])
if config['usessl'] and 'keyfilename' in config:
print "Connecting to '%s' TCP port %d," % (config['server'], config['port']),
print "Connecting to '%s' TCP port %d," % (
config['server'], config['port']),
print "SSL, key from %s," % (config['keyfilename']),
print "cert from %s " % (config['certfilename'])
server = imaplib.IMAP4_SSL(config['server'], config['port'],
config['keyfilename'], config['certfilename'])
elif config['usessl']:
print "Connecting to '%s' TCP port %d, SSL" % (config['server'], config['port'])
print "Connecting to '%s' TCP port %d, SSL" % (
config['server'], config['port'])
server = imaplib.IMAP4_SSL(config['server'], config['port'])
else:
print "Connecting to '%s' TCP port %d" % (config['server'], config['port'])
print "Connecting to '%s' TCP port %d" % (
config['server'], config['port'])
server = imaplib.IMAP4(config['server'], config['port'])
# speed up interactions on TCP connections using small packets
@ -602,20 +636,25 @@ def connect_and_login(config):
server.login(config['user'], config['pass'])
except socket.gaierror, e:
(err, desc) = e
print "ERROR: problem looking up server '%s' (%s %s)" % (config['server'], err, desc)
print "ERROR: problem looking up server '%s' (%s %s)" % (
config['server'], err, desc)
sys.exit(3)
except socket.error, e:
if str(e) == "SSL_CTX_use_PrivateKey_file error":
print "ERROR: error reading private key file '%s'" % (config['keyfilename'])
print "ERROR: error reading private key file '%s'" % (
config['keyfilename'])
elif str(e) == "SSL_CTX_use_certificate_chain_file error":
print "ERROR: error reading certificate chain file '%s'" % (config['keyfilename'])
print "ERROR: error reading certificate chain file '%s'" % (
config['keyfilename'])
else:
print "ERROR: could not connect to '%s' (%s)" % (config['server'], e)
print "ERROR: could not connect to '%s' (%s)" % (
config['server'], e)
sys.exit(4)
return server
def create_folder_structure(names):
""" Create the folder structure on disk """
for imap_foldername, filename in sorted(names):
@ -628,6 +667,7 @@ def create_folder_structure(names):
if e.errno != 17:
raise
def main():
"""Main entry point"""
try:
@ -636,11 +676,11 @@ def main():
names = get_names(server, config['compress'], config['thunderbird'],
config['nospinner'])
if config.get('folders'):
dirs = map (lambda x: x.strip(), config.get('folders').split(','))
dirs = map(lambda x: x.strip(), config.get('folders').split(','))
if config['thunderbird']:
dirs = [i.replace("Inbox", "INBOX", 1) if i.startswith("Inbox") else i
for i in dirs]
names = filter (lambda x: x[0] in dirs, names)
names = filter(lambda x: x[0] in dirs, names)
# for n, name in enumerate(names): # *DEBUG
# print n, name # *DEBUG
@ -650,7 +690,8 @@ def main():
for name_pair in names:
try:
foldername, filename = name_pair
fol_messages = scan_folder(server, foldername, config['nospinner'])
fol_messages = scan_folder(
server, foldername, config['nospinner'])
fil_messages = scan_file(filename, config['compress'],
config['overwrite'], config['nospinner'])
new_messages = {}
@ -658,7 +699,7 @@ def main():
if msg_id not in fil_messages:
new_messages[msg_id] = fol_messages[msg_id]
#for f in new_messages:
# for f in new_messages:
# print "%s : %s" % (f, new_messages[f])
download_messages(server, filename, new_messages, config)
@ -686,11 +727,11 @@ def cli_exception(typ, value, traceback):
sys.stdout.write("\n")
sys.stdout.flush()
if sys.stdin.isatty():
sys.excepthook = cli_exception
# Hideous fix to counteract http://python.org/sf/1092502
# (which should have been fixed ages ago.)
# Also see http://python.org/sf/1441530
@ -737,10 +778,12 @@ def _fixed_socket_read(self, size=-1):
buf_len += n
return "".join(buffers)
# Platform detection to enable socket patch
if 'Darwin' in platform.platform() and '2.3.5' == platform.python_version():
socket._fileobject.read = _fixed_socket_read
# 20181212: Windows 10 + Python 2.7 doesn't need this fix (fix leads to error: object of type 'cStringIO.StringO' has no len())
# 20181212: Windows 10 + Python 2.7 doesn't need this fix
# (fix leads to error: object of type 'cStringIO.StringO' has no len())
if 'Windows' in platform.platform() and '2.3.5' == platform.python_version():
socket._fileobject.read = _fixed_socket_read