sjs/imapbackup
1
0
Fork 0
mirror of https://github.com/samsonjs/imapbackup.git synced 2026-04-27 14:57:44 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #11 from ezzra/pep8

Browse source 
make code PEP8 compliant
This commit is contained in:
Rui Carmo 2020-06-04 17:08:58 +01:00 committed by GitHub
commit 4fadc21c74
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 699 additions and 656 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

141
imapbackup.py
View file

@ -43,15 +43,27 @@ __contributors__ = "jwagnerhki, Bob Ippolito, Michael Leonhard, Giuseppe Scrivan
# - Support host:port # - Support host:port
# - Cleaned up code using PyLint to identify problems # - Cleaned up code using PyLint to identify problems
# pylint -f html --indent-string=" " --max-line-length=90 imapbackup.py > report.html # pylint -f html --indent-string=" " --max-line-length=90 imapbackup.py > report.html
import getpass, os, gc, sys, time, platform, getopt import getpass
import mailbox, imaplib, socket import os
import re, hashlib, gzip, bz2 import gc
import sys
import time
import platform
import getopt
import mailbox
import imaplib
import socket
import re
import hashlib
import gzip
import bz2
class SkipFolderException(Exception): class SkipFolderException(Exception):
"""Indicates aborting processing of current folder, continue with next folder.""" """Indicates aborting processing of current folder, continue with next folder."""
pass pass
class Spinner: class Spinner:
"""Prints out message with cute spinner, indicating progress""" """Prints out message with cute spinner, indicating progress"""
@ -79,12 +91,13 @@ class Spinner:
sys.stdout.write("\r" + self.message) sys.stdout.write("\r" + self.message)
sys.stdout.flush() sys.stdout.flush()
def pretty_byte_count(num): def pretty_byte_count(num):
"""Converts integer into a human friendly count of bytes, eg: 12.243 MB""" """Converts integer into a human friendly count of bytes, eg: 12.243 MB"""
if num == 1: if num == 1:
return "1 byte" return "1 byte"
elif num < 1024: elif num < 1024:
return "%s bytes" % (num) return "%s bytes" % num
elif num < 1048576: elif num < 1048576:
return "%.2f KB" % (num/1024.0) return "%.2f KB" % (num/1024.0)
elif num < 1073741824: elif num < 1073741824:
@ -124,8 +137,6 @@ def string_from_file(value):
return content.read().strip() return content.read().strip()
def download_messages(server, filename, messages, config): def download_messages(server, filename, messages, config):
"""Download messages from folder and append to mailbox""" """Download messages from folder and append to mailbox"""
@ -174,7 +185,7 @@ def download_messages(server, filename, messages, config):
# fetch message # fetch message
typ, data = server.fetch(messages[msg_id], "RFC822") typ, data = server.fetch(messages[msg_id], "RFC822")
assert('OK' == typ) assert('OK' == typ)
text = data[0][1].strip().replace('\r','') text = data[0][1].strip().replace('\r', '')
if config['thunderbird']: if config['thunderbird']:
# This avoids Thunderbird mistaking a line starting "From " as the start # This avoids Thunderbird mistaking a line starting "From " as the start
# of a new message. _Might_ also apply to other mail lients - unknown # of a new message. _Might_ also apply to other mail lients - unknown
@ -195,6 +206,7 @@ def download_messages(server, filename, messages, config):
print ": %s total, %s for largest message" % (pretty_byte_count(total), print ": %s total, %s for largest message" % (pretty_byte_count(total),
pretty_byte_count(biggest)) pretty_byte_count(biggest))
def scan_file(filename, compress, overwrite, nospinner): def scan_file(filename, compress, overwrite, nospinner):
"""Gets IDs of messages in the specified mbox file""" """Gets IDs of messages in the specified mbox file"""
# file will be overwritten # file will be overwritten
@ -205,18 +217,18 @@ def scan_file(filename, compress, overwrite, nospinner):
# file doesn't exist # file doesn't exist
if not os.path.exists(filename): if not os.path.exists(filename):
print "File %s: not found" % (filename) print "File %s: not found" % filename
return [] return []
spinner = Spinner("File %s" % (filename), nospinner) spinner = Spinner("File %s" % filename, nospinner)
# open the file # open the file
if compress == 'gzip': if compress == 'gzip':
mbox = gzip.GzipFile(filename,'rb') mbox = gzip.GzipFile(filename, 'rb')
elif compress == 'bzip2': elif compress == 'bzip2':
mbox = bz2.BZ2File(filename,'rb') mbox = bz2.BZ2File(filename, 'rb')
else: else:
mbox = file(filename,'rb') mbox = file(filename, 'rb')
messages = {} messages = {}
@ -254,20 +266,22 @@ def scan_file(filename, compress, overwrite, nospinner):
print ": %d messages" % (len(messages.keys())) print ": %d messages" % (len(messages.keys()))
return messages return messages
def scan_folder(server, foldername, nospinner): def scan_folder(server, foldername, nospinner):
"""Gets IDs of messages in the specified folder, returns id:num dict""" """Gets IDs of messages in the specified folder, returns id:num dict"""
messages = {} messages = {}
spinner = Spinner("Folder %s" % (foldername), nospinner) spinner = Spinner("Folder %s" % foldername, nospinner)
try: try:
typ, data = server.select(foldername, readonly=True) typ, data = server.select(foldername, readonly=True)
if 'OK' != typ: if 'OK' != typ:
raise SkipFolderException("SELECT failed: %s" % (data)) raise SkipFolderException("SELECT failed: %s" % data)
num_msgs = int(data[0]) num_msgs = int(data[0])
# each message # each message
for num in range(1, num_msgs+1): for num in range(1, num_msgs+1):
# Retrieve Message-Id, making sure we don't mark all messages as read # Retrieve Message-Id, making sure we don't mark all messages as read
typ, data = server.fetch(num, '(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])') typ, data = server.fetch(
num, '(BODY.PEEK[HEADER.FIELDS (MESSAGE-ID)])')
if 'OK' != typ: if 'OK' != typ:
raise SkipFolderException("FETCH %s failed: %s" % (num, data)) raise SkipFolderException("FETCH %s failed: %s" % (num, data))
@ -282,12 +296,15 @@ def scan_folder(server, foldername, nospinner):
except (IndexError, AttributeError): except (IndexError, AttributeError):
# Some messages may have no Message-Id, so we'll synthesise one # Some messages may have no Message-Id, so we'll synthesise one
# (this usually happens with Sent, Drafts and .Mac news) # (this usually happens with Sent, Drafts and .Mac news)
typ, data = server.fetch(num, '(BODY[HEADER.FIELDS (FROM TO CC DATE SUBJECT)])') typ, data = server.fetch(
num, '(BODY[HEADER.FIELDS (FROM TO CC DATE SUBJECT)])')
if 'OK' != typ: if 'OK' != typ:
raise SkipFolderException("FETCH %s failed: %s" % (num, data)) raise SkipFolderException(
"FETCH %s failed: %s" % (num, data))
header = data[0][1].strip() header = data[0][1].strip()
header = header.replace('\r\n','\t') header = header.replace('\r\n', '\t')
messages['<' + UUID + '.' + hashlib.sha1(header).hexdigest() + '>'] = num messages['<' + UUID + '.' +
hashlib.sha1(header).hexdigest() + '>'] = num
spinner.spin() spinner.spin()
finally: finally:
spinner.stop() spinner.stop()
@ -297,6 +314,7 @@ def scan_folder(server, foldername, nospinner):
print "%d messages" % (len(messages.keys())) print "%d messages" % (len(messages.keys()))
return messages return messages
def parse_paren_list(row): def parse_paren_list(row):
"""Parses the nested list of attributes at the start of a LIST response""" """Parses the nested list of attributes at the start of a LIST response"""
# eat starting paren # eat starting paren
@ -317,7 +335,7 @@ def parse_paren_list(row):
# consume name attribute # consume name attribute
else: else:
match = name_attrib_re.search(row) match = name_attrib_re.search(row)
assert(match != None) assert(match is not None)
name_attrib = row[match.start():match.end()] name_attrib = row[match.start():match.end()]
row = row[match.end():] row = row[match.end():]
#print "MATCHED '%s' '%s'" % (name_attrib, row) #print "MATCHED '%s' '%s'" % (name_attrib, row)
@ -331,11 +349,13 @@ def parse_paren_list(row):
# done! # done!
return result, row return result, row
def parse_string_list(row): def parse_string_list(row):
"""Parses the quoted and unquoted strings at the end of a LIST response""" """Parses the quoted and unquoted strings at the end of a LIST response"""
slist = re.compile('\s*(?:"([^"]+)")\s*|\s*(\S+)\s*').split(row) slist = re.compile('\s*(?:"([^"]+)")\s*|\s*(\S+)\s*').split(row)
return [s for s in slist if s] return [s for s in slist if s]
def parse_list(row): def parse_list(row):
"""Prases response of LIST command into a list""" """Prases response of LIST command into a list"""
row = row.strip() row = row.strip()
@ -344,6 +364,7 @@ def parse_list(row):
assert(len(string_list) == 2) assert(len(string_list) == 2)
return [paren_list] + string_list return [paren_list] + string_list
def get_hierarchy_delimiter(server): def get_hierarchy_delimiter(server):
"""Queries the imapd for the hierarchy delimiter, eg. '.' in INBOX.Sent""" """Queries the imapd for the hierarchy delimiter, eg. '.' in INBOX.Sent"""
# see RFC 3501 page 39 paragraph 4 # see RFC 3501 page 39 paragraph 4
@ -357,6 +378,7 @@ def get_hierarchy_delimiter(server):
hierarchy_delim = '.' hierarchy_delim = '.'
return hierarchy_delim return hierarchy_delim
def get_names(server, compress, thunderbird, nospinner): def get_names(server, compress, thunderbird, nospinner):
"""Get list of folders, returns [(FolderName,FileName)]""" """Get list of folders, returns [(FolderName,FileName)]"""
@ -377,11 +399,11 @@ def get_names(server, compress, thunderbird, nospinner):
for row in data: for row in data:
lst = parse_list(row) lst = parse_list(row)
foldername = lst[2] foldername = lst[2]
suffix = {'none':'', 'gzip':'.gz', 'bzip2':'.bz2'}[compress] suffix = {'none': '', 'gzip': '.gz', 'bzip2': '.bz2'}[compress]
if thunderbird: if thunderbird:
filename = '.sbd/'.join(foldername.split(delim)) + suffix filename = '.sbd/'.join(foldername.split(delim)) + suffix
if filename.startswith("INBOX"): if filename.startswith("INBOX"):
filename = filename.replace("INBOX","Inbox") filename = filename.replace("INBOX", "Inbox")
else: else:
filename = '.'.join(foldername.split(delim)) + '.mbox' + suffix filename = '.'.join(foldername.split(delim)) + '.mbox' + suffix
# print "\n*** Folder:", foldername # *DEBUG # print "\n*** Folder:", foldername # *DEBUG
@ -393,6 +415,7 @@ def get_names(server, compress, thunderbird, nospinner):
print ": %s folders" % (len(names)) print ": %s folders" % (len(names))
return names return names
def print_usage(): def print_usage():
"""Prints usage, exits""" """Prints usage, exits"""
# " " # " "
@ -418,6 +441,7 @@ def print_usage():
print "\nNOTE: mbox files are created in the current working directory." print "\nNOTE: mbox files are created in the current working directory."
sys.exit(2) sys.exit(2)
def process_cline(): def process_cline():
"""Uses getopt to process command line, returns (config, warnings, errors)""" """Uses getopt to process command line, returns (config, warnings, errors)"""
# read command line # read command line
@ -431,8 +455,8 @@ def process_cline():
print_usage() print_usage()
warnings = [] warnings = []
config = {'compress':'none', 'overwrite':False, 'usessl':False, config = {'compress': 'none', 'overwrite': False, 'usessl': False,
'thunderbird':False, 'nospinner':False} 'thunderbird': False, 'nospinner': False}
errors = [] errors = []
# empty command line # empty command line
@ -488,18 +512,20 @@ def process_cline():
errors.append("Unknown argument: " + arg) errors.append("Unknown argument: " + arg)
# done processing command line # done processing command line
return (config, warnings, errors) return config, warnings, errors
def check_config(config, warnings, errors): def check_config(config, warnings, errors):
"""Checks the config for consistency, returns (config, warnings, errors)""" """Checks the config for consistency, returns (config, warnings, errors)"""
if config['compress'] == 'bzip2' and config['overwrite'] == False: if config['compress'] == 'bzip2' and config['overwrite'] is False:
errors.append("Cannot append new messages to mbox.bz2 files. Please specify -y.") errors.append(
if config['compress'] == 'gzip' and config['overwrite'] == False: "Cannot append new messages to mbox.bz2 files. Please specify -y.")
if config['compress'] == 'gzip' and config['overwrite'] is False:
warnings.append( warnings.append(
"Appending new messages to mbox.gz files is very slow. Please Consider\n" "Appending new messages to mbox.gz files is very slow. Please Consider\n"
" using -y and compressing the files yourself with gzip -9 *.mbox") " using -y and compressing the files yourself with gzip -9 *.mbox")
if 'server' not in config : if 'server' not in config:
errors.append("No server specified.") errors.append("No server specified.")
if 'user' not in config: if 'user' not in config:
errors.append("No username specified.") errors.append("No username specified.")
@ -508,7 +534,8 @@ def check_config(config, warnings, errors):
if 'keyfilename' in config and not config['usessl']: if 'keyfilename' in config and not config['usessl']:
errors.append("Key specified without SSL. Please use -e or --ssl.") errors.append("Key specified without SSL. Please use -e or --ssl.")
if 'certfilename' in config and not config['usessl']: if 'certfilename' in config and not config['usessl']:
errors.append("Certificate specified without SSL. Please use -e or --ssl.") errors.append(
"Certificate specified without SSL. Please use -e or --ssl.")
if 'server' in config and ':' in config['server']: if 'server' in config and ':' in config['server']:
# get host and port strings # get host and port strings
bits = config['server'].split(':', 1) bits = config['server'].split(':', 1)
@ -521,16 +548,19 @@ def check_config(config, warnings, errors):
raise ValueError raise ValueError
config['port'] = port config['port'] = port
except ValueError: except ValueError:
errors.append("Invalid port. Port must be an integer between 0 and 65535.") errors.append(
"Invalid port. Port must be an integer between 0 and 65535.")
if 'timeout' in config: if 'timeout' in config:
try: try:
timeout = int(config['timeout']) timeout = int(config['timeout'])
if timeout <= 0: if timeout <= 0:
raise ValueError raise ValueError
config['timeout']=timeout config['timeout'] = timeout
except ValueError: except ValueError:
errors.append("Invalid timeout value. Must be an integer greater than 0.") errors.append(
return (config, warnings, errors) "Invalid timeout value. Must be an integer greater than 0.")
return config, warnings, errors
def get_config(): def get_config():
"""Gets config from command line and console, returns config""" """Gets config from command line and console, returns config"""
@ -564,17 +594,18 @@ def get_config():
config['pass'] = getpass.getpass() config['pass'] = getpass.getpass()
# defaults # defaults
if not 'port' in config: if 'port' not in config:
if config['usessl']: if config['usessl']:
config['port'] = 993 config['port'] = 993
else: else:
config['port'] = 143 config['port'] = 143
if not 'timeout' in config: if 'timeout' not in config:
config['timeout'] = 60 config['timeout'] = 60
# done! # done!
return config return config
def connect_and_login(config): def connect_and_login(config):
"""Connects to the server and logs in. Returns IMAP4 object.""" """Connects to the server and logs in. Returns IMAP4 object."""
try: try:
@ -583,16 +614,19 @@ def connect_and_login(config):
socket.setdefaulttimeout(config['timeout']) socket.setdefaulttimeout(config['timeout'])
if config['usessl'] and 'keyfilename' in config: if config['usessl'] and 'keyfilename' in config:
print "Connecting to '%s' TCP port %d," % (config['server'], config['port']), print "Connecting to '%s' TCP port %d," % (
config['server'], config['port']),
print "SSL, key from %s," % (config['keyfilename']), print "SSL, key from %s," % (config['keyfilename']),
print "cert from %s " % (config['certfilename']) print "cert from %s " % (config['certfilename'])
server = imaplib.IMAP4_SSL(config['server'], config['port'], server = imaplib.IMAP4_SSL(config['server'], config['port'],
config['keyfilename'], config['certfilename']) config['keyfilename'], config['certfilename'])
elif config['usessl']: elif config['usessl']:
print "Connecting to '%s' TCP port %d, SSL" % (config['server'], config['port']) print "Connecting to '%s' TCP port %d, SSL" % (
config['server'], config['port'])
server = imaplib.IMAP4_SSL(config['server'], config['port']) server = imaplib.IMAP4_SSL(config['server'], config['port'])
else: else:
print "Connecting to '%s' TCP port %d" % (config['server'], config['port']) print "Connecting to '%s' TCP port %d" % (
config['server'], config['port'])
server = imaplib.IMAP4(config['server'], config['port']) server = imaplib.IMAP4(config['server'], config['port'])
# speed up interactions on TCP connections using small packets # speed up interactions on TCP connections using small packets
@ -602,20 +636,25 @@ def connect_and_login(config):
server.login(config['user'], config['pass']) server.login(config['user'], config['pass'])
except socket.gaierror, e: except socket.gaierror, e:
(err, desc) = e (err, desc) = e
print "ERROR: problem looking up server '%s' (%s %s)" % (config['server'], err, desc) print "ERROR: problem looking up server '%s' (%s %s)" % (
config['server'], err, desc)
sys.exit(3) sys.exit(3)
except socket.error, e: except socket.error, e:
if str(e) == "SSL_CTX_use_PrivateKey_file error": if str(e) == "SSL_CTX_use_PrivateKey_file error":
print "ERROR: error reading private key file '%s'" % (config['keyfilename']) print "ERROR: error reading private key file '%s'" % (
config['keyfilename'])
elif str(e) == "SSL_CTX_use_certificate_chain_file error": elif str(e) == "SSL_CTX_use_certificate_chain_file error":
print "ERROR: error reading certificate chain file '%s'" % (config['keyfilename']) print "ERROR: error reading certificate chain file '%s'" % (
config['keyfilename'])
else: else:
print "ERROR: could not connect to '%s' (%s)" % (config['server'], e) print "ERROR: could not connect to '%s' (%s)" % (
config['server'], e)
sys.exit(4) sys.exit(4)
return server return server
def create_folder_structure(names): def create_folder_structure(names):
""" Create the folder structure on disk """ """ Create the folder structure on disk """
for imap_foldername, filename in sorted(names): for imap_foldername, filename in sorted(names):
@ -628,6 +667,7 @@ def create_folder_structure(names):
if e.errno != 17: if e.errno != 17:
raise raise
def main(): def main():
"""Main entry point""" """Main entry point"""
try: try:
@ -636,11 +676,11 @@ def main():
names = get_names(server, config['compress'], config['thunderbird'], names = get_names(server, config['compress'], config['thunderbird'],
config['nospinner']) config['nospinner'])
if config.get('folders'): if config.get('folders'):
dirs = map (lambda x: x.strip(), config.get('folders').split(',')) dirs = map(lambda x: x.strip(), config.get('folders').split(','))
if config['thunderbird']: if config['thunderbird']:
dirs = [i.replace("Inbox", "INBOX", 1) if i.startswith("Inbox") else i dirs = [i.replace("Inbox", "INBOX", 1) if i.startswith("Inbox") else i
for i in dirs] for i in dirs]
names = filter (lambda x: x[0] in dirs, names) names = filter(lambda x: x[0] in dirs, names)
# for n, name in enumerate(names): # *DEBUG # for n, name in enumerate(names): # *DEBUG
# print n, name # *DEBUG # print n, name # *DEBUG
@ -650,7 +690,8 @@ def main():
for name_pair in names: for name_pair in names:
try: try:
foldername, filename = name_pair foldername, filename = name_pair
fol_messages = scan_folder(server, foldername, config['nospinner']) fol_messages = scan_folder(
server, foldername, config['nospinner'])
fil_messages = scan_file(filename, config['compress'], fil_messages = scan_file(filename, config['compress'],
config['overwrite'], config['nospinner']) config['overwrite'], config['nospinner'])
new_messages = {} new_messages = {}
@ -658,7 +699,7 @@ def main():
if msg_id not in fil_messages: if msg_id not in fil_messages:
new_messages[msg_id] = fol_messages[msg_id] new_messages[msg_id] = fol_messages[msg_id]
#for f in new_messages: # for f in new_messages:
# print "%s : %s" % (f, new_messages[f]) # print "%s : %s" % (f, new_messages[f])
download_messages(server, filename, new_messages, config) download_messages(server, filename, new_messages, config)
@ -686,11 +727,11 @@ def cli_exception(typ, value, traceback):
sys.stdout.write("\n") sys.stdout.write("\n")
sys.stdout.flush() sys.stdout.flush()
if sys.stdin.isatty(): if sys.stdin.isatty():
sys.excepthook = cli_exception sys.excepthook = cli_exception
# Hideous fix to counteract http://python.org/sf/1092502 # Hideous fix to counteract http://python.org/sf/1092502
# (which should have been fixed ages ago.) # (which should have been fixed ages ago.)
# Also see http://python.org/sf/1441530 # Also see http://python.org/sf/1441530
@ -737,10 +778,12 @@ def _fixed_socket_read(self, size=-1):
buf_len += n buf_len += n
return "".join(buffers) return "".join(buffers)
# Platform detection to enable socket patch # Platform detection to enable socket patch
if 'Darwin' in platform.platform() and '2.3.5' == platform.python_version(): if 'Darwin' in platform.platform() and '2.3.5' == platform.python_version():
socket._fileobject.read = _fixed_socket_read socket._fileobject.read = _fixed_socket_read
# 20181212: Windows 10 + Python 2.7 doesn't need this fix (fix leads to error: object of type 'cStringIO.StringO' has no len()) # 20181212: Windows 10 + Python 2.7 doesn't need this fix
# (fix leads to error: object of type 'cStringIO.StringO' has no len())
if 'Windows' in platform.platform() and '2.3.5' == platform.python_version(): if 'Windows' in platform.platform() and '2.3.5' == platform.python_version():
socket._fileobject.read = _fixed_socket_read socket._fileobject.read = _fixed_socket_read