diff --git a/sublime/Installed Packages/Package Control.sublime-package b/sublime/Installed Packages/Package Control.sublime-package new file mode 100644 index 0000000..cc9aa19 Binary files /dev/null and b/sublime/Installed Packages/Package Control.sublime-package differ diff --git a/sublime/Packages/Default/Preferences.sublime-settings b/sublime/Packages/Default/Preferences.sublime-settings index bf86fc6..ae3eb45 100644 --- a/sublime/Packages/Default/Preferences.sublime-settings +++ b/sublime/Packages/Default/Preferences.sublime-settings @@ -228,6 +228,10 @@ // inserting tabs. "shift_tab_unindent": false, + // If true, the copy and cut commands will operate on the current line + // when the selection is empty, rather than doing nothing. + "copy_with_empty_selection": true, + // If true, the selected text will be copied into the find panel when it's // shown. // On OS X, this value is overridden in the platform specific settings, so @@ -264,6 +268,13 @@ // Sublime Text must be restarted for this to take effect. "use_simple_full_screen": false, + // OS X only. Valid values are true, false, and "auto". Auto will enable + // the setting when running on a screen 2880 pixels or wider (i.e., a + // Retina display). When this setting is enabled, OpenGL is used to + // accelerate drawing. Sublime Text must be restarted for changes to take + // effect. + "gpu_window_buffer": "auto", + // Valid values are "system", "enabled" and "disabled" "overlay_scroll_bars": "system", diff --git a/sublime/Packages/Package Control/Default.sublime-commands b/sublime/Packages/Package Control/Default.sublime-commands index 7d823ee..dd5513c 100644 --- a/sublime/Packages/Package Control/Default.sublime-commands +++ b/sublime/Packages/Package Control/Default.sublime-commands @@ -5,7 +5,7 @@ }, { "caption": "Package Control: Add Channel", - "command": "add_repository_channel" + "command": "add_channel" }, { "caption": "Package Control: Create Binary Package File", @@ -27,6 +27,10 @@ "caption": "Package Control: Enable Package", "command": "enable_package" }, + { + "caption": "Package Control: Grab CA Certs", + "command": "grab_certs" + }, { "caption": "Package Control: Install Package", "command": "install_package" diff --git a/sublime/Packages/Package Control/Package Control.ca-bundle b/sublime/Packages/Package Control/Package Control.ca-bundle new file mode 100644 index 0000000..b718caa --- /dev/null +++ b/sublime/Packages/Package Control/Package Control.ca-bundle @@ -0,0 +1,43 @@ +----BEGIN CERTIFICATE----- +MIIDVDCCAjygAwIBAgIDAjRWMA0GCSqGSIb3DQEBBQUAMEIxCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1HZW9UcnVzdCBJbmMuMRswGQYDVQQDExJHZW9UcnVzdCBHbG9i +YWwgQ0EwHhcNMDIwNTIxMDQwMDAwWhcNMjIwNTIxMDQwMDAwWjBCMQswCQYDVQQG +EwJVUzEWMBQGA1UEChMNR2VvVHJ1c3QgSW5jLjEbMBkGA1UEAxMSR2VvVHJ1c3Qg +R2xvYmFsIENBMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA2swYYzD9 +9BcjGlZ+W988bDjkcbd4kdS8odhM+KhDtgPpTSEHCIjaWC9mOSm9BXiLnTjoBbdq +fnGk5sRgprDvgOSJKA+eJdbtg/OtppHHmMlCGDUUna2YRpIuT8rxh0PBFpVXLVDv +iS2Aelet8u5fa9IAjbkU+BQVNdnARqN7csiRv8lVK83Qlz6cJmTM386DGXHKTubU +1XupGc1V3sjs0l44U+VcT4wt/lAjNvxm5suOpDkZALeVAjmRCw7+OC7RHQWa9k0+ +bw8HHa8sHo9gOeL6NlMTOdReJivbPagUvTLrGAMoUgRx5aszPeE4uwc2hGKceeoW +MPRfwCvocWvk+QIDAQABo1MwUTAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTA +ephojYn7qwVkDBF9qn1luMrMTjAfBgNVHSMEGDAWgBTAephojYn7qwVkDBF9qn1l +uMrMTjANBgkqhkiG9w0BAQUFAAOCAQEANeMpauUvXVSOKVCUn5kaFOSPeCpilKIn +Z57QzxpeR+nBsqTP3UEaBU6bS+5Kb1VSsyShNwrrZHYqLizz/Tt1kL/6cdjHPTfS +tQWVYrmm3ok9Nns4d0iXrKYgjy6myQzCsplFAMfOEVEiIuCl6rYVSAlk6l5PdPcF +PseKUgzbFbS9bZvlxrFUaKnjaZC2mqUPuLk/IH2uSrW4nOQdtqvmlKXBx4Ot2/Un +hw4EbNX/3aBd7YdStysVAq45pmp06drE57xNNB6pXE0zX5IJL4hmXXeXxx12E6nV +5fEWCRE11azbJHFwLJhWC9kXtNHjUStedejV0NxPNO3CBWaAocvmMw== +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDxTCCAq2gAwIBAgIQAqxcJmoLQJuPC3nyrkYldzANBgkqhkiG9w0BAQUFADBs +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMSswKQYDVQQDEyJEaWdpQ2VydCBIaWdoIEFzc3VyYW5j +ZSBFViBSb290IENBMB4XDTA2MTExMDAwMDAwMFoXDTMxMTExMDAwMDAwMFowbDEL +MAkGA1UEBhMCVVMxFTATBgNVBAoTDERpZ2lDZXJ0IEluYzEZMBcGA1UECxMQd3d3 +LmRpZ2ljZXJ0LmNvbTErMCkGA1UEAxMiRGlnaUNlcnQgSGlnaCBBc3N1cmFuY2Ug +RVYgUm9vdCBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMbM5XPm ++9S75S0tMqbf5YE/yc0lSbZxKsPVlDRnogocsF9ppkCxxLeyj9CYpKlBWTrT3JTW +PNt0OKRKzE0lgvdKpVMSOO7zSW1xkX5jtqumX8OkhPhPYlG++MXs2ziS4wblCJEM +xChBVfvLWokVfnHoNb9Ncgk9vjo4UFt3MRuNs8ckRZqnrG0AFFoEt7oT61EKmEFB +Ik5lYYeBQVCmeVyJ3hlKV9Uu5l0cUyx+mM0aBhakaHPQNAQTXKFx01p8VdteZOE3 +hzBWBOURtCmAEvF5OYiiAhF8J2a3iLd48soKqDirCmTCv2ZdlYTBoSUeh10aUAsg +EsxBu24LUTi4S8sCAwEAAaNjMGEwDgYDVR0PAQH/BAQDAgGGMA8GA1UdEwEB/wQF +MAMBAf8wHQYDVR0OBBYEFLE+w2kD+L9HAdSYJhoIAu9jZCvDMB8GA1UdIwQYMBaA +FLE+w2kD+L9HAdSYJhoIAu9jZCvDMA0GCSqGSIb3DQEBBQUAA4IBAQAcGgaX3Nec +nzyIZgYIVyHbIUf4KmeqvxgydkAQV8GK83rZEWWONfqe/EW1ntlMMUu4kehDLI6z +eM7b41N5cdblIZQB2lWHmiRk9opmzN6cN82oNLFpmyPInngiK3BD41VHMWEZ71jF +hS9OMPagMRYjyOfiZRYzy78aG6A9+MpeizGLYAiJLQwGXFK3xPkKmNEVX58Svnw2 +Yzi9RKR/5CYrCsSXaQ3pjOLAEFe4yHYSkVXySGnYvCoCWw9E1CAx2/S6cCZdkGCe +vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep ++OkuE6N36B9K +-----END CERTIFICATE----- diff --git a/sublime/Packages/Package Control/Package Control.ca-list b/sublime/Packages/Package Control/Package Control.ca-list new file mode 100644 index 0000000..93aa232 --- /dev/null +++ b/sublime/Packages/Package Control/Package Control.ca-list @@ -0,0 +1,4 @@ +[ + "221e907bdfff70d71cea42361ae209d5", + "7d0986b90061d60c8c02aa3b1cf23850" +] diff --git a/sublime/Packages/Package Control/Package Control.py b/sublime/Packages/Package Control/Package Control.py index 515159a..9c47377 100644 --- a/sublime/Packages/Package Control/Package Control.py +++ b/sublime/Packages/Package Control/Package Control.py @@ -1,4810 +1,101 @@ -# coding=utf-8 -import sublime -import sublime_plugin -import os -import sys -import subprocess -import zipfile -import urllib -import urllib2 -import json -from fnmatch import fnmatch -import re -import threading -import datetime -import time -import shutil -import tempfile -import httplib -import socket -import hashlib -import base64 -import locale -import urlparse -import gzip -import StringIO -import zlib - - -if os.name == 'nt': - from ctypes import windll, create_unicode_buffer - - -def add_to_path(path): - # Python 2.x on Windows can't properly import from non-ASCII paths, so - # this code added the DOC 8.3 version of the lib folder to the path in - # case the user's username includes non-ASCII characters - if os.name == 'nt': - buf = create_unicode_buffer(512) - if windll.kernel32.GetShortPathNameW(path, buf, len(buf)): - path = buf.value - - if path not in sys.path: - sys.path.append(path) - - -lib_folder = os.path.join(sublime.packages_path(), 'Package Control', 'lib') -add_to_path(os.path.join(lib_folder, 'all')) - - -import semver - - -if os.name == 'nt': - add_to_path(os.path.join(lib_folder, 'windows')) - from ntlm import ntlm - - -def unicode_from_os(e): - # This is needed as some exceptions coming from the OS are - # already encoded and so just calling unicode(e) will result - # in an UnicodeDecodeError as the string isn't in ascii form. - try: - # Sublime Text on OS X does not seem to report the correct encoding - # so we hard-code that to UTF-8 - encoding = 'UTF-8' if os.name == 'darwin' else locale.getpreferredencoding() - return unicode(str(e), encoding) - - # If the "correct" encoding did not work, try some defaults, and then just - # obliterate characters that we can't seen to decode properly - except UnicodeDecodeError: - encodings = ['utf-8', 'cp1252'] - for encoding in encodings: - try: - return unicode(str(e), encoding, errors='strict') - except: - pass - return unicode(str(e), errors='replace') - - -def create_cmd(args, basename_binary=False): - if basename_binary: - args[0] = os.path.basename(args[0]) - - if os.name == 'nt': - return subprocess.list2cmdline(args) - else: - escaped_args = [] - for arg in args: - if re.search('^[a-zA-Z0-9/_^\\-\\.:=]+$', arg) == None: - arg = u"'" + arg.replace(u"'", u"'\\''") + u"'" - escaped_args.append(arg) - return u' '.join(escaped_args) - - -# Monkey patch AbstractBasicAuthHandler to prevent infinite recursion -def non_recursive_http_error_auth_reqed(self, authreq, host, req, headers): - authreq = headers.get(authreq, None) - - if not hasattr(self, 'retried'): - self.retried = 0 - - if self.retried > 5: - raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", - headers, None) - else: - self.retried += 1 - - if authreq: - mo = urllib2.AbstractBasicAuthHandler.rx.search(authreq) - if mo: - scheme, quote, realm = mo.groups() - if scheme.lower() == 'basic': - return self.retry_http_basic_auth(host, req, realm) - -urllib2.AbstractBasicAuthHandler.http_error_auth_reqed = non_recursive_http_error_auth_reqed - - -class DebuggableHTTPResponse(httplib.HTTPResponse): - """ - A custom HTTPResponse that formats debugging info for Sublime Text - """ - - _debug_protocol = 'HTTP' - - def __init__(self, sock, debuglevel=0, strict=0, method=None): - # We have to use a positive debuglevel to get it passed to here, - # however we don't want to use it because by default debugging prints - # to the stdout and we can't capture it, so we use a special -1 value - if debuglevel == 5: - debuglevel = -1 - httplib.HTTPResponse.__init__(self, sock, debuglevel, strict, method) - - def begin(self): - return_value = httplib.HTTPResponse.begin(self) - if self.debuglevel == -1: - print '%s: Urllib2 %s Debug Read' % (__name__, self._debug_protocol) - headers = self.msg.headers - versions = { - 9: 'HTTP/0.9', - 10: 'HTTP/1.0', - 11: 'HTTP/1.1' - } - status_line = versions[self.version] + ' ' + str(self.status) + ' ' + self.reason - headers.insert(0, status_line) - for line in headers: - print u" %s" % line.rstrip() - return return_value - - def read(self, *args): - try: - return httplib.HTTPResponse.read(self, *args) - except (httplib.IncompleteRead) as (e): - return e.partial - - -class DebuggableHTTPSResponse(DebuggableHTTPResponse): - """ - A version of DebuggableHTTPResponse that sets the debug protocol to HTTPS - """ - - _debug_protocol = 'HTTPS' - - -class DebuggableHTTPConnection(httplib.HTTPConnection): - """ - A custom HTTPConnection that formats debugging info for Sublime Text - """ - - response_class = DebuggableHTTPResponse - _debug_protocol = 'HTTP' - - def __init__(self, host, port=None, strict=None, - timeout=socket._GLOBAL_DEFAULT_TIMEOUT, **kwargs): - self.passwd = kwargs.get('passwd') - - # Python 2.6.1 on OS X 10.6 does not include these - self._tunnel_host = None - self._tunnel_port = None - self._tunnel_headers = {} - - httplib.HTTPConnection.__init__(self, host, port, strict, timeout) - - def connect(self): - if self.debuglevel == -1: - print '%s: Urllib2 %s Debug General' % (__name__, self._debug_protocol) - print u" Connecting to %s on port %s" % (self.host, self.port) - httplib.HTTPConnection.connect(self) - - def send(self, string): - # We have to use a positive debuglevel to get it passed to the - # HTTPResponse object, however we don't want to use it because by - # default debugging prints to the stdout and we can't capture it, so - # we temporarily set it to -1 for the standard httplib code - reset_debug = False - if self.debuglevel == 5: - reset_debug = 5 - self.debuglevel = -1 - httplib.HTTPConnection.send(self, string) - if reset_debug or self.debuglevel == -1: - if len(string.strip()) > 0: - print '%s: Urllib2 %s Debug Write' % (__name__, self._debug_protocol) - for line in string.strip().splitlines(): - print ' ' + line - if reset_debug: - self.debuglevel = reset_debug - - def request(self, method, url, body=None, headers={}): - original_headers = headers.copy() - - # Handles the challenge request response cycle before the real request - proxy_auth = headers.get('Proxy-Authorization') - if os.name == 'nt' and proxy_auth and proxy_auth.lstrip()[0:4] == 'NTLM': - # The default urllib2.AbstractHTTPHandler automatically sets the - # Connection header to close because of urllib.addinfourl(), but in - # this case we are going to do some back and forth first for the NTLM - # proxy auth - headers['Connection'] = 'Keep-Alive' - self._send_request(method, url, body, headers) - - response = self.getresponse() - - content_length = int(response.getheader('content-length', 0)) - if content_length: - response._safe_read(content_length) - - proxy_authenticate = response.getheader('proxy-authenticate', None) - if not proxy_authenticate: - raise URLError('Invalid NTLM proxy authentication response') - ntlm_challenge = re.sub('^\s*NTLM\s+', '', proxy_authenticate) - - if self.host.find(':') != -1: - host_port = self.host - else: - host_port = "%s:%s" % (self.host, self.port) - username, password = self.passwd.find_user_password(None, host_port) - domain = '' - user = username - if username.find('\\') != -1: - domain, user = username.split('\\', 1) - - challenge, negotiate_flags = ntlm.parse_NTLM_CHALLENGE_MESSAGE(ntlm_challenge) - new_proxy_authorization = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(challenge, user, - domain, password, negotiate_flags) - original_headers['Proxy-Authorization'] = new_proxy_authorization - response.close() - - httplib.HTTPConnection.request(self, method, url, body, original_headers) - - -class DebuggableHTTPHandler(urllib2.HTTPHandler): - """ - A custom HTTPHandler that formats debugging info for Sublime Text - """ - - def __init__(self, debuglevel=0, debug=False, **kwargs): - # This is a special value that will not trigger the standard debug - # functionality, but custom code where we can format the output - if debug: - self._debuglevel = 5 - else: - self._debuglevel = debuglevel - self.passwd = kwargs.get('passwd') - - def http_open(self, req): - def http_class_wrapper(host, **kwargs): - kwargs['passwd'] = self.passwd - return DebuggableHTTPConnection(host, **kwargs) - - return self.do_open(http_class_wrapper, req) - - -class RateLimitException(httplib.HTTPException, urllib2.URLError): - """ - An exception for when the rate limit of an API has been exceeded. - """ - - def __init__(self, host, limit): - httplib.HTTPException.__init__(self) - self.host = host - self.limit = limit - - def __str__(self): - return ('Rate limit of %s exceeded for %s' % (self.limit, self.host)) - - -if os.name == 'nt': - class ProxyNtlmAuthHandler(urllib2.BaseHandler): - - handler_order = 300 - auth_header = 'Proxy-Authorization' - - def __init__(self, password_manager=None): - if password_manager is None: - password_manager = HTTPPasswordMgr() - self.passwd = password_manager - self.retried = 0 - - def http_error_407(self, req, fp, code, msg, headers): - proxy_authenticate = headers.get('proxy-authenticate') - if os.name != 'nt' or proxy_authenticate[0:4] != 'NTLM': - return None - - type1_flags = ntlm.NTLM_TYPE1_FLAGS - - if req.host.find(':') != -1: - host_port = req.host - else: - host_port = "%s:%s" % (req.host, req.port) - username, password = self.passwd.find_user_password(None, host_port) - if not username: - return None - - if username.find('\\') == -1: - type1_flags &= ~ntlm.NTLM_NegotiateOemDomainSupplied - - negotiate_message = ntlm.create_NTLM_NEGOTIATE_MESSAGE(username, type1_flags) - auth = 'NTLM %s' % negotiate_message - if req.headers.get(self.auth_header, None) == auth: - return None - req.add_unredirected_header(self.auth_header, auth) - return self.parent.open(req, timeout=req.timeout) - - -# The following code is wrapped in a try because the Linux versions of Sublime -# Text do not include the ssl module due to the fact that different distros -# have different versions -try: - import ssl - - class InvalidCertificateException(httplib.HTTPException, urllib2.URLError): - """ - An exception for when an SSL certification is not valid for the URL - it was presented for. - """ - - def __init__(self, host, cert, reason): - httplib.HTTPException.__init__(self) - self.host = host - self.cert = cert - self.reason = reason - - def __str__(self): - return ('Host %s returned an invalid certificate (%s) %s\n' % - (self.host, self.reason, self.cert)) - - - class ValidatingHTTPSConnection(DebuggableHTTPConnection): - """ - A custom HTTPConnection class that validates SSL certificates, and - allows proxy authentication for HTTPS connections. - """ - - default_port = httplib.HTTPS_PORT - - response_class = DebuggableHTTPSResponse - _debug_protocol = 'HTTPS' - - def __init__(self, host, port=None, key_file=None, cert_file=None, - ca_certs=None, strict=None, **kwargs): - passed_args = {} - if 'timeout' in kwargs: - passed_args['timeout'] = kwargs['timeout'] - DebuggableHTTPConnection.__init__(self, host, port, strict, **passed_args) - - self.passwd = kwargs.get('passwd') - self.key_file = key_file - self.cert_file = cert_file - self.ca_certs = ca_certs - if 'user_agent' in kwargs: - self.user_agent = kwargs['user_agent'] - if self.ca_certs: - self.cert_reqs = ssl.CERT_REQUIRED - else: - self.cert_reqs = ssl.CERT_NONE - - def get_valid_hosts_for_cert(self, cert): - """ - Returns a list of valid hostnames for an SSL certificate - - :param cert: A dict from SSLSocket.getpeercert() - - :return: An array of hostnames - """ - - if 'subjectAltName' in cert: - return [x[1] for x in cert['subjectAltName'] - if x[0].lower() == 'dns'] - else: - return [x[0][1] for x in cert['subject'] - if x[0][0].lower() == 'commonname'] - - def validate_cert_host(self, cert, hostname): - """ - Checks if the cert is valid for the hostname - - :param cert: A dict from SSLSocket.getpeercert() - - :param hostname: A string hostname to check - - :return: A boolean if the cert is valid for the hostname - """ - - hosts = self.get_valid_hosts_for_cert(cert) - for host in hosts: - host_re = host.replace('.', '\.').replace('*', '[^.]*') - if re.search('^%s$' % (host_re,), hostname, re.I): - return True - return False - - def _tunnel(self, ntlm_follow_up=False): - """ - This custom _tunnel method allows us to read and print the debug - log for the whole response before throwing an error, and adds - support for proxy authentication - """ - - self._proxy_host = self.host - self._proxy_port = self.port - self._set_hostport(self._tunnel_host, self._tunnel_port) - - self._tunnel_headers['Host'] = u"%s:%s" % (self.host, self.port) - self._tunnel_headers['User-Agent'] = self.user_agent - self._tunnel_headers['Proxy-Connection'] = 'Keep-Alive' - - request = "CONNECT %s:%d HTTP/1.1\r\n" % (self.host, self.port) - for header, value in self._tunnel_headers.iteritems(): - request += "%s: %s\r\n" % (header, value) - self.send(request + "\r\n") - - response = self.response_class(self.sock, strict=self.strict, - method=self._method) - (version, code, message) = response._read_status() - - status_line = u"%s %s %s" % (version, code, message.rstrip()) - headers = [status_line] - - if self.debuglevel in [-1, 5]: - print '%s: Urllib2 %s Debug Read' % (__name__, self._debug_protocol) - print u" %s" % status_line - - content_length = 0 - close_connection = False - while True: - line = response.fp.readline() - if line == '\r\n': break - - headers.append(line.rstrip()) - - parts = line.rstrip().split(': ', 1) - name = parts[0].lower() - value = parts[1].lower().strip() - if name == 'content-length': - content_length = int(value) - - if name in ['connection', 'proxy-connection'] and value == 'close': - close_connection = True - - if self.debuglevel in [-1, 5]: - print u" %s" % line.rstrip() - - # Handle proxy auth for SSL connections since regular urllib2 punts on this - if code == 407 and self.passwd and ('Proxy-Authorization' not in self._tunnel_headers or ntlm_follow_up): - if content_length: - response._safe_read(content_length) - - supported_auth_methods = {} - for line in headers: - parts = line.split(': ', 1) - if parts[0].lower() != 'proxy-authenticate': - continue - details = parts[1].split(' ', 1) - supported_auth_methods[details[0].lower()] = details[1] if len(details) > 1 else '' - - username, password = self.passwd.find_user_password(None, "%s:%s" % ( - self._proxy_host, self._proxy_port)) - - do_ntlm_follow_up = False - - if 'digest' in supported_auth_methods: - response_value = self.build_digest_response( - supported_auth_methods['digest'], username, password) - if response_value: - self._tunnel_headers['Proxy-Authorization'] = u"Digest %s" % response_value - - elif 'basic' in supported_auth_methods: - response_value = u"%s:%s" % (username, password) - response_value = base64.b64encode(response_value).strip() - self._tunnel_headers['Proxy-Authorization'] = u"Basic %s" % response_value - - elif 'ntlm' in supported_auth_methods and os.name == 'nt': - ntlm_challenge = supported_auth_methods['ntlm'] - if not len(ntlm_challenge): - type1_flags = ntlm.NTLM_TYPE1_FLAGS - if username.find('\\') == -1: - type1_flags &= ~ntlm.NTLM_NegotiateOemDomainSupplied - - negotiate_message = ntlm.create_NTLM_NEGOTIATE_MESSAGE(username, type1_flags) - self._tunnel_headers['Proxy-Authorization'] = 'NTLM %s' % negotiate_message - do_ntlm_follow_up = True - else: - domain = '' - user = username - if username.find('\\') != -1: - domain, user = username.split('\\', 1) - - challenge, negotiate_flags = ntlm.parse_NTLM_CHALLENGE_MESSAGE(ntlm_challenge) - self._tunnel_headers['Proxy-Authorization'] = 'NTLM %s' % ntlm.create_NTLM_AUTHENTICATE_MESSAGE(challenge, user, - domain, password, negotiate_flags) - - if 'Proxy-Authorization' in self._tunnel_headers: - self.host = self._proxy_host - self.port = self._proxy_port - - # If the proxy wanted the connection closed, we need to make a new connection - if close_connection: - self.sock.close() - self.sock = socket.create_connection((self.host, self.port), self.timeout) - - return self._tunnel(do_ntlm_follow_up) - - if code != 200: - self.close() - raise socket.error("Tunnel connection failed: %d %s" % (code, - message.strip())) - - def build_digest_response(self, fields, username, password): - """ - Takes a Proxy-Authenticate: Digest header and creates a response - header - - :param fields: - The string portion of the Proxy-Authenticate header after - "Digest " - - :param username: - The username to use for the response - - :param password: - The password to use for the response - - :return: - None if invalid Proxy-Authenticate header, otherwise the - string of fields for the Proxy-Authorization: Digest header - """ - - fields = urllib2.parse_keqv_list(urllib2.parse_http_list(fields)) - - realm = fields.get('realm') - nonce = fields.get('nonce') - qop = fields.get('qop') - algorithm = fields.get('algorithm') - if algorithm: - algorithm = algorithm.lower() - opaque = fields.get('opaque') - - if algorithm in ['md5', None]: - def hash(string): - return hashlib.md5(string).hexdigest() - elif algorithm == 'sha': - def hash(string): - return hashlib.sha1(string).hexdigest() - else: - return None - - host_port = u"%s:%s" % (self.host, self.port) - - a1 = "%s:%s:%s" % (username, realm, password) - a2 = "CONNECT:%s" % host_port - ha1 = hash(a1) - ha2 = hash(a2) - - if qop == None: - response = hash(u"%s:%s:%s" % (ha1, nonce, ha2)) - elif qop == 'auth': - nc = '00000001' - cnonce = hash(urllib2.randombytes(8))[:8] - response = hash(u"%s:%s:%s:%s:%s:%s" % (ha1, nonce, nc, cnonce, qop, ha2)) - else: - return None - - response_fields = { - 'username': username, - 'realm': realm, - 'nonce': nonce, - 'response': response, - 'uri': host_port - } - if algorithm: - response_fields['algorithm'] = algorithm - if qop == 'auth': - response_fields['nc'] = nc - response_fields['cnonce'] = cnonce - response_fields['qop'] = qop - if opaque: - response_fields['opaque'] = opaque - - return ', '.join([u"%s=\"%s\"" % (field, response_fields[field]) for field in response_fields]) - - def connect(self): - """ - Adds debugging and SSL certification validation - """ - - if self.debuglevel == -1: - print '%s: Urllib2 HTTPS Debug General' % __name__ - print u" Connecting to %s on port %s" % (self.host, self.port) - - self.sock = socket.create_connection((self.host, self.port), self.timeout) - if self._tunnel_host: - self._tunnel() - - if self.debuglevel == -1: - print u"%s: Urllib2 HTTPS Debug General" % __name__ - print u" Connecting to %s on port %s" % (self.host, self.port) - print u" CA certs file at %s" % (self.ca_certs) - - self.sock = ssl.wrap_socket(self.sock, keyfile=self.key_file, - certfile=self.cert_file, cert_reqs=self.cert_reqs, - ca_certs=self.ca_certs) - - if self.debuglevel == -1: - print u" Successfully upgraded connection to %s:%s with SSL" % ( - self.host, self.port) - - # This debugs and validates the SSL certificate - if self.cert_reqs & ssl.CERT_REQUIRED: - cert = self.sock.getpeercert() - - if self.debuglevel == -1: - subjectMap = { - 'organizationName': 'O', - 'commonName': 'CN', - 'organizationalUnitName': 'OU', - 'countryName': 'C', - 'serialNumber': 'serialNumber', - 'commonName': 'CN', - 'localityName': 'L', - 'stateOrProvinceName': 'S' - } - subject_list = list(cert['subject']) - subject_list.reverse() - subject_parts = [] - for pair in subject_list: - if pair[0][0] in subjectMap: - field_name = subjectMap[pair[0][0]] - else: - field_name = pair[0][0] - subject_parts.append(field_name + '=' + pair[0][1]) - - print u" Server SSL certificate:" - print u" subject: " + ','.join(subject_parts) - if 'subjectAltName' in cert: - print u" common name: " + cert['subjectAltName'][0][1] - if 'notAfter' in cert: - print u" expire date: " + cert['notAfter'] - - hostname = self.host.split(':', 0)[0] - - if not self.validate_cert_host(cert, hostname): - if self.debuglevel == -1: - print u" Certificate INVALID" - - raise InvalidCertificateException(hostname, cert, - 'hostname mismatch') - - if self.debuglevel == -1: - print u" Certificate validated for %s" % hostname - - if hasattr(urllib2, 'HTTPSHandler'): - class ValidatingHTTPSHandler(urllib2.HTTPSHandler): - """ - A urllib2 handler that validates SSL certificates for HTTPS requests - """ - - def __init__(self, **kwargs): - # This is a special value that will not trigger the standard debug - # functionality, but custom code where we can format the output - self._debuglevel = 0 - if 'debug' in kwargs and kwargs['debug']: - self._debuglevel = 5 - elif 'debuglevel' in kwargs: - self._debuglevel = kwargs['debuglevel'] - self._connection_args = kwargs - - def https_open(self, req): - def http_class_wrapper(host, **kwargs): - full_kwargs = dict(self._connection_args) - full_kwargs.update(kwargs) - return ValidatingHTTPSConnection(host, **full_kwargs) - - try: - return self.do_open(http_class_wrapper, req) - except urllib2.URLError, e: - if type(e.reason) == ssl.SSLError and e.reason.args[0] == 1: - raise InvalidCertificateException(req.host, '', - e.reason.args[1]) - raise - - https_request = urllib2.AbstractHTTPHandler.do_request_ - -except (ImportError): - pass - - -def preferences_filename(): - """:return: The appropriate settings filename based on the version of Sublime Text""" - - if int(sublime.version()) >= 2174: - return 'Preferences.sublime-settings' - return 'Global.sublime-settings' - - -class ThreadProgress(): - """ - Animates an indicator, [= ], in the status area while a thread runs - - :param thread: - The thread to track for activity - - :param message: - The message to display next to the activity indicator - - :param success_message: - The message to display once the thread is complete - """ - - def __init__(self, thread, message, success_message): - self.thread = thread - self.message = message - self.success_message = success_message - self.addend = 1 - self.size = 8 - sublime.set_timeout(lambda: self.run(0), 100) - - def run(self, i): - if not self.thread.is_alive(): - if hasattr(self.thread, 'result') and not self.thread.result: - sublime.status_message('') - return - sublime.status_message(self.success_message) - return - - before = i % self.size - after = (self.size - 1) - before - - sublime.status_message('%s [%s=%s]' % \ - (self.message, ' ' * before, ' ' * after)) - - if not after: - self.addend = -1 - if not before: - self.addend = 1 - i += self.addend - - sublime.set_timeout(lambda: self.run(i), 100) - - -class PlatformComparator(): - def get_best_platform(self, platforms): - ids = [sublime.platform() + '-' + sublime.arch(), sublime.platform(), - '*'] - - for id in ids: - if id in platforms: - return id - - return None - - -class ChannelProvider(PlatformComparator): - """ - Retrieves a channel and provides an API into the information - - The current channel/repository infrastructure caches repository info into - the channel to improve the Package Control client performance. This also - has the side effect of lessening the load on the GitHub and BitBucket APIs - and getting around not-infrequent HTTP 503 errors from those APIs. - - :param channel: - The URL of the channel - - :param package_manager: - An instance of :class:`PackageManager` used to download the file - """ - - def __init__(self, channel, package_manager): - self.channel_info = None - self.channel = channel - self.package_manager = package_manager - self.unavailable_packages = [] - - def match_url(self): - """Indicates if this provider can handle the provided channel""" - - return True - - def fetch_channel(self): - """Retrieves and loads the JSON for other methods to use""" - - if self.channel_info != None: - return - - channel_json = self.package_manager.download_url(self.channel, - 'Error downloading channel.') - if channel_json == False: - self.channel_info = False - return - - try: - channel_info = json.loads(channel_json) - except (ValueError): - print '%s: Error parsing JSON from channel %s.' % (__name__, - self.channel) - channel_info = False - - self.channel_info = channel_info - - def get_name_map(self): - """:return: A dict of the mapping for URL slug -> package name""" - - self.fetch_channel() - if self.channel_info == False: - return False - return self.channel_info.get('package_name_map', {}) - - def get_renamed_packages(self): - """:return: A dict of the packages that have been renamed""" - - self.fetch_channel() - if self.channel_info == False: - return False - return self.channel_info.get('renamed_packages', {}) - - def get_repositories(self): - """:return: A list of the repository URLs""" - - self.fetch_channel() - if self.channel_info == False: - return False - return self.channel_info['repositories'] - - def get_certs(self): - """ - Provides a secure way for distribution of SSL CA certificates - - Unfortunately Python does not include a bundle of CA certs with urllib2 - to perform SSL certificate validation. To circumvent this issue, - Package Control acts as a distributor of the CA certs for all HTTPS - URLs of package downloads. - - The default channel scrapes and caches info about all packages - periodically, and in the process it checks the CA certs for all of - the HTTPS URLs listed in the repositories. The contents of the CA cert - files are then hashed, and the CA cert is stored in a filename with - that hash. This is a fingerprint to ensure that Package Control has - the appropriate CA cert for a domain name. - - Next, the default channel file serves up a JSON object of the domain - names and the hashes of their current CA cert files. If Package Control - does not have the appropriate hash for a domain, it may retrieve it - from the channel server. To ensure that Package Control is talking to - a trusted authority to get the CA certs from, the CA cert for - sublime.wbond.net is bundled with Package Control. Then when downloading - the channel file, Package Control can ensure that the channel file's - SSL certificate is valid, thus ensuring the resulting CA certs are - legitimate. - - As a matter of optimization, the distribution of Package Control also - includes the current CA certs for all known HTTPS domains that are - included in the channel, as of the time when Package Control was - last released. - - :return: A dict of {'Domain Name': ['cert_file_hash', 'cert_file_download_url']} - """ - - self.fetch_channel() - if self.channel_info == False: - return False - return self.channel_info.get('certs', {}) - - def get_packages(self, repo): - """ - Provides access to the repository info that is cached in a channel - - :param repo: - The URL of the repository to get the cached info of - - :return: - A dict in the format: - { - 'Package Name': { - # Package details - see example-packages.json for format - }, - ... - } - or False if there is an error - """ - - self.fetch_channel() - if self.channel_info == False: - return False - if self.channel_info.get('packages', False) == False: - return False - if self.channel_info['packages'].get(repo, False) == False: - return False - - output = {} - for package in self.channel_info['packages'][repo]: - copy = package.copy() - - platforms = copy['platforms'].keys() - best_platform = self.get_best_platform(platforms) - - if not best_platform: - self.unavailable_packages.append(copy['name']) - continue - - copy['downloads'] = copy['platforms'][best_platform] - - del copy['platforms'] - - copy['url'] = copy['homepage'] - del copy['homepage'] - - output[copy['name']] = copy - - return output - - def get_unavailable_packages(self): - """ - Provides a list of packages that are unavailable for the current - platform/architecture that Sublime Text is running on. - - This list will be empty unless get_packages() is called first. - - :return: A list of package names - """ - - return self.unavailable_packages - - -# The providers (in order) to check when trying to download a channel -_channel_providers = [ChannelProvider] - - -class PackageProvider(PlatformComparator): - """ - Generic repository downloader that fetches package info - - With the current channel/repository architecture where the channel file - caches info from all includes repositories, these package providers just - serve the purpose of downloading packages not in the default channel. - - The structure of the JSON a repository should contain is located in - example-packages.json. - - :param repo: - The URL of the package repository - - :param package_manager: - An instance of :class:`PackageManager` used to download the file - """ - def __init__(self, repo, package_manager): - self.repo_info = None - self.repo = repo - self.package_manager = package_manager - self.unavailable_packages = [] - - def match_url(self): - """Indicates if this provider can handle the provided repo""" - - return True - - def fetch_repo(self): - """Retrieves and loads the JSON for other methods to use""" - - if self.repo_info != None: - return - - repository_json = self.package_manager.download_url(self.repo, - 'Error downloading repository.') - if repository_json == False: - self.repo_info = False - return - - try: - self.repo_info = json.loads(repository_json) - except (ValueError): - print '%s: Error parsing JSON from repository %s.' % (__name__, - self.repo) - self.repo_info = False - - def get_packages(self): - """ - Provides access to the repository info that is cached in a channel - - :return: - A dict in the format: - { - 'Package Name': { - # Package details - see example-packages.json for format - }, - ... - } - or False if there is an error - """ - - self.fetch_repo() - if self.repo_info == False: - return False - - output = {} - - for package in self.repo_info['packages']: - - platforms = package['platforms'].keys() - best_platform = self.get_best_platform(platforms) - - if not best_platform: - self.unavailable_packages.append(package['name']) - continue - - # Rewrites the legacy "zipball" URLs to the new "zip" format - downloads = package['platforms'][best_platform] - rewritten_downloads = [] - for download in downloads: - download['url'] = re.sub( - '^(https://nodeload.github.com/[^/]+/[^/]+/)zipball(/.*)$', - '\\1zip\\2', download['url']) - rewritten_downloads.append(download) - - info = { - 'name': package['name'], - 'description': package.get('description'), - 'url': package.get('homepage', self.repo), - 'author': package.get('author'), - 'last_modified': package.get('last_modified'), - 'downloads': rewritten_downloads - } - - output[package['name']] = info - - return output - - def get_renamed_packages(self): - """:return: A dict of the packages that have been renamed""" - - return self.repo_info.get('renamed_packages', {}) - - def get_unavailable_packages(self): - """ - Provides a list of packages that are unavailable for the current - platform/architecture that Sublime Text is running on. - - This list will be empty unless get_packages() is called first. - - :return: A list of package names - """ - - return self.unavailable_packages - - -class NonCachingProvider(): - """ - Base for package providers that do not need to cache the JSON - """ - - def fetch_json(self, url): - """ - Retrieves and parses the JSON from a URL - - :return: A dict or list from the JSON, or False on error - """ - - repository_json = self.package_manager.download_url(url, - 'Error downloading repository.') - if repository_json == False: - return False - try: - return json.loads(repository_json) - except (ValueError): - print '%s: Error parsing JSON from repository %s.' % (__name__, - url) - return False - - def get_unavailable_packages(self): - """ - Method for compatibility with PackageProvider class. These providers - are based on API calls, and thus do not support different platform - downloads, making it impossible for there to be unavailable packages. - - :return: An empty list - """ - - return [] - - -class GitHubPackageProvider(NonCachingProvider): - """ - Allows using a public GitHub repository as the source for a single package - - :param repo: - The public web URL to the GitHub repository. Should be in the format - `https://github.com/user/package` for the master branch, or - `https://github.com/user/package/tree/{branch_name}` for any other - branch. - - :param package_manager: - An instance of :class:`PackageManager` used to access the API - """ - - def __init__(self, repo, package_manager): - # Clean off the trailing .git to be more forgiving - self.repo = re.sub('\.git$', '', repo) - self.package_manager = package_manager - - def match_url(self): - """Indicates if this provider can handle the provided repo""" - - master = re.search('^https?://github.com/[^/]+/[^/]+/?$', self.repo) - branch = re.search('^https?://github.com/[^/]+/[^/]+/tree/[^/]+/?$', - self.repo) - return master != None or branch != None - - def get_packages(self): - """Uses the GitHub API to construct necessary info for a package""" - - branch = 'master' - branch_match = re.search( - '^https?://github.com/[^/]+/[^/]+/tree/([^/]+)/?$', self.repo) - if branch_match != None: - branch = branch_match.group(1) - - api_url = re.sub('^https?://github.com/([^/]+)/([^/]+)($|/.*$)', - 'https://api.github.com/repos/\\1/\\2', self.repo) - - repo_info = self.fetch_json(api_url) - if repo_info == False: - return False - - # In addition to hitting the main API endpoint for this repo, we - # also have to list the commits to get the timestamp of the last - # commit since we use that to generate a version number - commit_api_url = api_url + '/commits?' + \ - urllib.urlencode({'sha': branch, 'per_page': 1}) - - commit_info = self.fetch_json(commit_api_url) - if commit_info == False: - return False - - # We specifically use nodeload.github.com here because the download - # URLs all redirect there, and some of the downloaders don't follow - # HTTP redirect headers - download_url = 'https://nodeload.github.com/' + \ - repo_info['owner']['login'] + '/' + \ - repo_info['name'] + '/zip/' + urllib.quote(branch) - - commit_date = commit_info[0]['commit']['committer']['date'] - timestamp = datetime.datetime.strptime(commit_date[0:19], - '%Y-%m-%dT%H:%M:%S') - utc_timestamp = timestamp.strftime( - '%Y.%m.%d.%H.%M.%S') - - homepage = repo_info['homepage'] - if not homepage: - homepage = repo_info['html_url'] - - package = { - 'name': repo_info['name'], - 'description': repo_info['description'] if \ - repo_info['description'] else 'No description provided', - 'url': homepage, - 'author': repo_info['owner']['login'], - 'last_modified': timestamp.strftime('%Y-%m-%d %H:%M:%S'), - 'downloads': [ - { - 'version': utc_timestamp, - 'url': download_url - } - ] - } - return {package['name']: package} - - def get_renamed_packages(self): - """For API-compatibility with :class:`PackageProvider`""" - - return {} - - -class GitHubUserProvider(NonCachingProvider): - """ - Allows using a GitHub user/organization as the source for multiple packages - - :param repo: - The public web URL to the GitHub user/org. Should be in the format - `https://github.com/user`. - - :param package_manager: - An instance of :class:`PackageManager` used to access the API - """ - - def __init__(self, repo, package_manager): - self.repo = repo - self.package_manager = package_manager - - def match_url(self): - """Indicates if this provider can handle the provided repo""" - - return re.search('^https?://github.com/[^/]+/?$', self.repo) != None - - def get_packages(self): - """Uses the GitHub API to construct necessary info for all packages""" - - user_match = re.search('^https?://github.com/([^/]+)/?$', self.repo) - user = user_match.group(1) - - api_url = 'https://api.github.com/users/%s/repos?per_page=100' % user - - repo_info = self.fetch_json(api_url) - if repo_info == False: - return False - - packages = {} - for package_info in repo_info: - # All packages for the user are made available, and always from - # the master branch. Anything else requires a custom packages.json - commit_api_url = ('https://api.github.com/repos/%s/%s/commits' + \ - '?sha=master&per_page=1') % (user, package_info['name']) - - commit_info = self.fetch_json(commit_api_url) - if commit_info == False: - return False - - commit_date = commit_info[0]['commit']['committer']['date'] - timestamp = datetime.datetime.strptime(commit_date[0:19], - '%Y-%m-%dT%H:%M:%S') - utc_timestamp = timestamp.strftime( - '%Y.%m.%d.%H.%M.%S') - - homepage = package_info['homepage'] - if not homepage: - homepage = package_info['html_url'] - - package = { - 'name': package_info['name'], - 'description': package_info['description'] if \ - package_info['description'] else 'No description provided', - 'url': homepage, - 'author': package_info['owner']['login'], - 'last_modified': timestamp.strftime('%Y-%m-%d %H:%M:%S'), - 'downloads': [ - { - 'version': utc_timestamp, - # We specifically use nodeload.github.com here because - # the download URLs all redirect there, and some of the - # downloaders don't follow HTTP redirect headers - 'url': 'https://nodeload.github.com/' + \ - package_info['owner']['login'] + '/' + \ - package_info['name'] + '/zip/master' - } - ] - } - packages[package['name']] = package - return packages - - def get_renamed_packages(self): - """For API-compatibility with :class:`PackageProvider`""" - - return {} - - -class BitBucketPackageProvider(NonCachingProvider): - """ - Allows using a public BitBucket repository as the source for a single package - - :param repo: - The public web URL to the BitBucket repository. Should be in the format - `https://bitbucket.org/user/package`. - - :param package_manager: - An instance of :class:`PackageManager` used to access the API - """ - - def __init__(self, repo, package_manager): - self.repo = repo - self.package_manager = package_manager - - def match_url(self): - """Indicates if this provider can handle the provided repo""" - - return re.search('^https?://bitbucket.org', self.repo) != None - - def get_packages(self): - """Uses the BitBucket API to construct necessary info for a package""" - - api_url = re.sub('^https?://bitbucket.org/', - 'https://api.bitbucket.org/1.0/repositories/', self.repo) - api_url = api_url.rstrip('/') - - repo_info = self.fetch_json(api_url) - if repo_info == False: - return False - - # Since HG allows for arbitrary main branch names, we have to hit - # this URL just to get that info - main_branch_url = api_url + '/main-branch/' - main_branch_info = self.fetch_json(main_branch_url) - if main_branch_info == False: - return False - - # Grabbing the changesets is necessary because we construct the - # version number from the last commit timestamp - changeset_url = api_url + '/changesets/' + main_branch_info['name'] - last_commit = self.fetch_json(changeset_url) - if last_commit == False: - return False - - commit_date = last_commit['timestamp'] - timestamp = datetime.datetime.strptime(commit_date[0:19], - '%Y-%m-%d %H:%M:%S') - utc_timestamp = timestamp.strftime( - '%Y.%m.%d.%H.%M.%S') - - homepage = repo_info['website'] - if not homepage: - homepage = self.repo - package = { - 'name': repo_info['name'], - 'description': repo_info['description'] if \ - repo_info['description'] else 'No description provided', - 'url': homepage, - 'author': repo_info['owner'], - 'last_modified': timestamp.strftime('%Y-%m-%d %H:%M:%S'), - 'downloads': [ - { - 'version': utc_timestamp, - 'url': self.repo + '/get/' + \ - last_commit['node'] + '.zip' - } - ] - } - return {package['name']: package} - - def get_renamed_packages(self): - """For API-compatibility with :class:`PackageProvider`""" - - return {} - - -# The providers (in order) to check when trying to download repository info -_package_providers = [BitBucketPackageProvider, GitHubPackageProvider, - GitHubUserProvider, PackageProvider] - - -class BinaryNotFoundError(Exception): - """If a necessary executable is not found in the PATH on the system""" - - pass - - -class NonCleanExitError(Exception): - """ - When an subprocess does not exit cleanly - - :param returncode: - The command line integer return code of the subprocess - """ - - def __init__(self, returncode): - self.returncode = returncode - - def __str__(self): - return repr(self.returncode) - - -class NonHttpError(Exception): - """If a downloader had a non-clean exit, but it was not due to an HTTP error""" - - pass - - -class Downloader(): - """ - A base downloader that actually performs downloading URLs - - The SSL module is not included with the bundled Python for Linux - users of Sublime Text, so Linux machines will fall back to using curl - or wget for HTTPS URLs. - """ - - def check_certs(self, domain, timeout): - """ - Ensures that the SSL CA cert for a domain is present on the machine - - :param domain: - The domain to ensure there is a CA cert for - - :param timeout: - The int timeout for downloading the CA cert from the channel - - :return: - The CA cert bundle path on success, or False on error - """ - - cert_match = False - - certs_list = self.settings.get('certs', {}) - certs_dir = os.path.join(sublime.packages_path(), 'Package Control', - 'certs') - ca_bundle_path = os.path.join(certs_dir, 'ca-bundle.crt') - - cert_info = certs_list.get(domain) - if cert_info: - cert_match = self.locate_cert(certs_dir, cert_info[0], cert_info[1]) - - wildcard_info = certs_list.get('*') - if wildcard_info: - cert_match = self.locate_cert(certs_dir, wildcard_info[0], wildcard_info[1]) or cert_match - - if not cert_match: - print '%s: No CA certs available for %s.' % (__name__, domain) - return False - - return ca_bundle_path - - def locate_cert(self, certs_dir, cert_id, location): - """ - Makes sure the SSL cert specified has been added to the CA cert - bundle that is present on the machine - - :param certs_dir: - The path of the folder that contains the cert files - - :param cert_id: - The identifier for CA cert(s). For those provided by the channel - system, this will be an md5 of the contents of the cert(s). For - user-provided certs, this is something they provide. - - :param location: - An http(s) URL, or absolute filesystem path to the CA cert(s) - - :return: - If the cert specified (by cert_id) is present on the machine and - part of the ca-bundle.crt file in the certs_dir - """ - - cert_path = os.path.join(certs_dir, cert_id) - if not os.path.exists(cert_path): - if str(location) != '': - if re.match('^https?://', location): - contents = self.download_cert(cert_id, location) - else: - contents = self.load_cert(cert_id, location) - if contents: - self.save_cert(certs_dir, cert_id, contents) - return True - return False - return True - - def download_cert(self, cert_id, url): - """ - Downloads CA cert(s) from a URL - - :param cert_id: - The identifier for CA cert(s). For those provided by the channel - system, this will be an md5 of the contents of the cert(s). For - user-provided certs, this is something they provide. - - :param url: - An http(s) URL to the CA cert(s) - - :return: - The contents of the CA cert(s) - """ - - cert_downloader = self.__class__(self.settings) - return cert_downloader.download(url, - 'Error downloading CA certs for %s.' % (domain), timeout, 1) - - def load_cert(self, cert_id, path): - """ - Copies CA cert(s) from a file path - - :param cert_id: - The identifier for CA cert(s). For those provided by the channel - system, this will be an md5 of the contents of the cert(s). For - user-provided certs, this is something they provide. - - :param path: - The absolute filesystem path to a file containing the CA cert(s) - - :return: - The contents of the CA cert(s) - """ - - if os.path.exists(path): - with open(path, 'rb') as f: - return f.read() - - def save_cert(self, certs_dir, cert_id, contents): - """ - Saves CA cert(s) to the certs_dir (and ca-bundle.crt file) - - :param certs_dir: - The path of the folder that contains the cert files - - :param cert_id: - The identifier for CA cert(s). For those provided by the channel - system, this will be an md5 of the contents of the cert(s). For - user-provided certs, this is something they provide. - - :param contents: - The contents of the CA cert(s) - """ - - ca_bundle_path = os.path.join(certs_dir, 'ca-bundle.crt') - cert_path = os.path.join(certs_dir, cert_id) - with open(cert_path, 'wb') as f: - f.write(contents) - with open(ca_bundle_path, 'ab') as f: - f.write("\n" + contents) - - def decode_response(self, encoding, response): - if encoding == 'gzip': - return gzip.GzipFile(fileobj=StringIO.StringIO(response)).read() - elif encoding == 'deflate': - decompresser = zlib.decompressobj(-zlib.MAX_WBITS) - return decompresser.decompress(response) + decompresser.flush() - return response - - -class CliDownloader(Downloader): - """ - Base for downloaders that use a command line program - - :param settings: - A dict of the various Package Control settings. The Sublime Text - Settings API is not used because this code is run in a thread. - """ - - def __init__(self, settings): - self.settings = settings - - def clean_tmp_file(self): - if os.path.exists(self.tmp_file): - os.remove(self.tmp_file) - - def find_binary(self, name): - """ - Finds the given executable name in the system PATH - - :param name: - The exact name of the executable to find - - :return: - The absolute path to the executable - - :raises: - BinaryNotFoundError when the executable can not be found - """ - - for dir in os.environ['PATH'].split(os.pathsep): - path = os.path.join(dir, name) - if os.path.exists(path): - return path - - raise BinaryNotFoundError('The binary %s could not be located' % name) - - def execute(self, args): - """ - Runs the executable and args and returns the result - - :param args: - A list of the executable path and all arguments to be passed to it - - :return: - The text output of the executable - - :raises: - NonCleanExitError when the executable exits with an error - """ - - if self.settings.get('debug'): - print u"%s: Trying to execute command %s" % ( - __name__, create_cmd(args)) - - proc = subprocess.Popen(args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - output = proc.stdout.read() - self.stderr = proc.stderr.read() - returncode = proc.wait() - if returncode != 0: - error = NonCleanExitError(returncode) - error.stderr = self.stderr - error.stdout = output - raise error - return output - - -class UrlLib2Downloader(Downloader): - """ - A downloader that uses the Python urllib2 module - - :param settings: - A dict of the various Package Control settings. The Sublime Text - Settings API is not used because this code is run in a thread. - """ - - def __init__(self, settings): - self.settings = settings - - def download(self, url, error_message, timeout, tries): - """ - Downloads a URL and returns the contents - - Uses the proxy settings from the Package Control.sublime-settings file, - however there seem to be a decent number of proxies that this code - does not work with. Patches welcome! - - :param url: - The URL to download - - :param error_message: - A string to include in the console error that is printed - when an error occurs - - :param timeout: - The int number of seconds to set the timeout to - - :param tries: - The int number of times to try and download the URL in the case of - a timeout or HTTP 503 error - - :return: - The string contents of the URL, or False on error - """ - - http_proxy = self.settings.get('http_proxy') - https_proxy = self.settings.get('https_proxy') - if http_proxy or https_proxy: - proxies = {} - if http_proxy: - proxies['http'] = http_proxy - if https_proxy: - proxies['https'] = https_proxy - proxy_handler = urllib2.ProxyHandler(proxies) - else: - proxy_handler = urllib2.ProxyHandler() - - password_manager = urllib2.HTTPPasswordMgrWithDefaultRealm() - proxy_username = self.settings.get('proxy_username') - proxy_password = self.settings.get('proxy_password') - if proxy_username and proxy_password: - if http_proxy: - password_manager.add_password(None, http_proxy, proxy_username, - proxy_password) - if https_proxy: - password_manager.add_password(None, https_proxy, proxy_username, - proxy_password) - - handlers = [proxy_handler] - if os.name == 'nt': - ntlm_auth_handler = ProxyNtlmAuthHandler(password_manager) - handlers.append(ntlm_auth_handler) - - basic_auth_handler = urllib2.ProxyBasicAuthHandler(password_manager) - digest_auth_handler = urllib2.ProxyDigestAuthHandler(password_manager) - handlers.extend([digest_auth_handler, basic_auth_handler]) - - debug = self.settings.get('debug') - - if debug: - print u"%s: Urllib2 Debug Proxy" % __name__ - print u" http_proxy: %s" % http_proxy - print u" https_proxy: %s" % https_proxy - print u" proxy_username: %s" % proxy_username - print u" proxy_password: %s" % proxy_password - - secure_url_match = re.match('^https://([^/]+)', url) - if secure_url_match != None: - secure_domain = secure_url_match.group(1) - bundle_path = self.check_certs(secure_domain, timeout) - if not bundle_path: - return False - bundle_path = bundle_path.encode(sys.getfilesystemencoding()) - handlers.append(ValidatingHTTPSHandler(ca_certs=bundle_path, - debug=debug, passwd=password_manager, - user_agent=self.settings.get('user_agent'))) - else: - handlers.append(DebuggableHTTPHandler(debug=debug, - passwd=password_manager)) - urllib2.install_opener(urllib2.build_opener(*handlers)) - - while tries > 0: - tries -= 1 - try: - request = urllib2.Request(url, headers={ - "User-Agent": self.settings.get('user_agent'), - # Don't be alarmed if the response from the server does not - # select one of these since the server runs a relatively new - # version of OpenSSL which supports compression on the SSL - # layer, and Apache will use that instead of HTTP-level - # encoding. - "Accept-Encoding": "gzip,deflate"}) - http_file = urllib2.urlopen(request, timeout=timeout) - self.handle_rate_limit(http_file, url) - result = http_file.read() - encoding = http_file.headers.get('Content-Encoding') - return self.decode_response(encoding, result) - - except (httplib.HTTPException) as (e): - print '%s: %s HTTP exception %s (%s) downloading %s.' % ( - __name__, error_message, e.__class__.__name__, - unicode_from_os(e), url) - - except (urllib2.HTTPError) as (e): - # Make sure we obey Github's rate limiting headers - self.handle_rate_limit(e, url) - - # Bitbucket and Github return 503 a decent amount - if unicode_from_os(e.code) == '503': - print ('%s: Downloading %s was rate limited, ' + - 'trying again') % (__name__, url) - continue - print '%s: %s HTTP error %s downloading %s.' % (__name__, - error_message, unicode_from_os(e.code), url) - - except (urllib2.URLError) as (e): - - # Bitbucket and Github timeout a decent amount - if unicode_from_os(e.reason) == 'The read operation timed out' \ - or unicode_from_os(e.reason) == 'timed out': - print (u'%s: Downloading %s timed out, trying ' + - u'again') % (__name__, url) - continue - print u'%s: %s URL error %s downloading %s.' % (__name__, - error_message, unicode_from_os(e.reason), url) - break - return False - - def handle_rate_limit(self, response, url): - """ - Checks the headers of a respone object to make sure we are obeying the - rate limit - - :param response: - The response object that has a headers dict - - :param url: - The URL that was requested - - :raises: - RateLimitException when the rate limit has been hit - """ - - limit_remaining = response.headers.get('X-RateLimit-Remaining', 1) - if str(limit_remaining) == '0': - hostname = urlparse.urlparse(url).hostname - limit = response.headers.get('X-RateLimit-Limit', 1) - raise RateLimitException(hostname, limit) - - -class WgetDownloader(CliDownloader): - """ - A downloader that uses the command line program wget - - :param settings: - A dict of the various Package Control settings. The Sublime Text - Settings API is not used because this code is run in a thread. - """ - - def __init__(self, settings): - self.settings = settings - self.debug = settings.get('debug') - self.wget = self.find_binary('wget') - - def download(self, url, error_message, timeout, tries): - """ - Downloads a URL and returns the contents - - :param url: - The URL to download - - :param error_message: - A string to include in the console error that is printed - when an error occurs - - :param timeout: - The int number of seconds to set the timeout to - - :param tries: - The int number of times to try and download the URL in the case of - a timeout or HTTP 503 error - - :return: - The string contents of the URL, or False on error - """ - - if not self.wget: - return False - - self.tmp_file = tempfile.NamedTemporaryFile().name - command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o', - self.tmp_file, '-O', '-', '-U', - self.settings.get('user_agent'), '--header', - # Don't be alarmed if the response from the server does not select - # one of these since the server runs a relatively new version of - # OpenSSL which supports compression on the SSL layer, and Apache - # will use that instead of HTTP-level encoding. - 'Accept-Encoding: gzip,deflate'] - - secure_url_match = re.match('^https://([^/]+)', url) - if secure_url_match != None: - secure_domain = secure_url_match.group(1) - bundle_path = self.check_certs(secure_domain, timeout) - if not bundle_path: - return False - command.append(u'--ca-certificate=' + bundle_path) - - if self.debug: - command.append('-d') - else: - command.append('-S') - - http_proxy = self.settings.get('http_proxy') - https_proxy = self.settings.get('https_proxy') - proxy_username = self.settings.get('proxy_username') - proxy_password = self.settings.get('proxy_password') - - if proxy_username: - command.append(u"--proxy-user=%s" % proxy_username) - if proxy_password: - command.append(u"--proxy-password=%s" % proxy_password) - - if self.debug: - print u"%s: Wget Debug Proxy" % __name__ - print u" http_proxy: %s" % http_proxy - print u" https_proxy: %s" % https_proxy - print u" proxy_username: %s" % proxy_username - print u" proxy_password: %s" % proxy_password - - command.append(url) - - if http_proxy: - os.putenv('http_proxy', http_proxy) - if https_proxy: - os.putenv('https_proxy', https_proxy) - - while tries > 0: - tries -= 1 - try: - result = self.execute(command) - - general, headers = self.parse_output() - encoding = headers.get('content-encoding') - if encoding: - result = self.decode_response(encoding, result) - - return result - - except (NonCleanExitError) as (e): - - try: - general, headers = self.parse_output() - self.handle_rate_limit(headers, url) - - if general['status'] == '503': - # GitHub and BitBucket seem to rate limit via 503 - print ('%s: Downloading %s was rate limited' + - ', trying again') % (__name__, url) - continue - - error_string = 'HTTP error %s %s' % (general['status'], - general['message']) - - except (NonHttpError) as (e): - - error_string = unicode_from_os(e) - - # GitHub and BitBucket seem to time out a lot - if error_string.find('timed out') != -1: - print ('%s: Downloading %s timed out, ' + - 'trying again') % (__name__, url) - continue - - print (u'%s: %s %s downloading %s.' % (__name__, error_message, - error_string, url)).encode('UTF-8') - - break - return False - - def parse_output(self): - with open(self.tmp_file, 'r') as f: - output = f.read().splitlines() - self.clean_tmp_file() - - error = None - header_lines = [] - if self.debug: - section = 'General' - last_section = None - for line in output: - if section == 'General': - if self.skippable_line(line): - continue - - # Skip blank lines - if line.strip() == '': - continue - - # Error lines - if line[0:5] == 'wget:': - error = line[5:].strip() - if line[0:7] == 'failed:': - error = line[7:].strip() - - if line == '---request begin---': - section = 'Write' - continue - elif line == '---request end---': - section = 'General' - continue - elif line == '---response begin---': - section = 'Read' - continue - elif line == '---response end---': - section = 'General' - continue - - if section != last_section: - print "%s: Wget HTTP Debug %s" % (__name__, section) - - if section == 'Read': - header_lines.append(line) - - print ' ' + line - last_section = section - - else: - for line in output: - if self.skippable_line(line): - continue - - # Check the resolving and connecting to lines for errors - if re.match('(Resolving |Connecting to )', line): - failed_match = re.search(' failed: (.*)$', line) - if failed_match: - error = failed_match.group(1).strip() - - # Error lines - if line[0:5] == 'wget:': - error = line[5:].strip() - if line[0:7] == 'failed:': - error = line[7:].strip() - - if line[0:2] == ' ': - header_lines.append(line.lstrip()) - - if error: - raise NonHttpError(error) - - return self.parse_headers(header_lines) - - def skippable_line(self, line): - # Skip date lines - if re.match('--\d{4}-\d{2}-\d{2}', line): - return True - if re.match('\d{4}-\d{2}-\d{2}', line): - return True - # Skip HTTP status code lines since we already have that info - if re.match('\d{3} ', line): - return True - # Skip Saving to and progress lines - if re.match('(Saving to:|\s*\d+K)', line): - return True - # Skip notice about ignoring body on HTTP error - if re.match('Skipping \d+ byte', line): - return True - - def parse_headers(self, output=None): - if not output: - with open(self.tmp_file, 'r') as f: - output = f.read().splitlines() - self.clean_tmp_file() - - general = { - 'version': '0.9', - 'status': '200', - 'message': 'OK' - } - headers = {} - for line in output: - # When using the -S option, headers have two spaces before them, - # additionally, valid headers won't have spaces, so this is always - # a safe operation to perform - line = line.lstrip() - if line.find('HTTP/') == 0: - match = re.match('HTTP/(\d\.\d)\s+(\d+)\s+(.*)$', line) - general['version'] = match.group(1) - general['status'] = match.group(2) - general['message'] = match.group(3) - else: - name, value = line.split(':', 1) - headers[name.lower()] = value.strip() - - return (general, headers) - - def handle_rate_limit(self, headers, url): - limit_remaining = headers.get('x-ratelimit-remaining', '1') - limit = headers.get('x-ratelimit-limit', '1') - - if str(limit_remaining) == '0': - hostname = urlparse.urlparse(url).hostname - raise RateLimitException(hostname, limit) - - -class CurlDownloader(CliDownloader): - """ - A downloader that uses the command line program curl - - :param settings: - A dict of the various Package Control settings. The Sublime Text - Settings API is not used because this code is run in a thread. - """ - - def __init__(self, settings): - self.settings = settings - self.curl = self.find_binary('curl') - - def download(self, url, error_message, timeout, tries): - """ - Downloads a URL and returns the contents - - :param url: - The URL to download - - :param error_message: - A string to include in the console error that is printed - when an error occurs - - :param timeout: - The int number of seconds to set the timeout to - - :param tries: - The int number of times to try and download the URL in the case of - a timeout or HTTP 503 error - - :return: - The string contents of the URL, or False on error - """ - - if not self.curl: - return False - - self.tmp_file = tempfile.NamedTemporaryFile().name - command = [self.curl, '--user-agent', self.settings.get('user_agent'), - '--connect-timeout', str(int(timeout)), '-sSL', - # Don't be alarmed if the response from the server does not select - # one of these since the server runs a relatively new version of - # OpenSSL which supports compression on the SSL layer, and Apache - # will use that instead of HTTP-level encoding. - '--compressed', - # We have to capture the headers to check for rate limit info - '--dump-header', self.tmp_file] - - secure_url_match = re.match('^https://([^/]+)', url) - if secure_url_match != None: - secure_domain = secure_url_match.group(1) - bundle_path = self.check_certs(secure_domain, timeout) - if not bundle_path: - return False - command.extend(['--cacert', bundle_path]) - - debug = self.settings.get('debug') - if debug: - command.append('-v') - - http_proxy = self.settings.get('http_proxy') - https_proxy = self.settings.get('https_proxy') - proxy_username = self.settings.get('proxy_username') - proxy_password = self.settings.get('proxy_password') - - if debug: - print u"%s: Curl Debug Proxy" % __name__ - print u" http_proxy: %s" % http_proxy - print u" https_proxy: %s" % https_proxy - print u" proxy_username: %s" % proxy_username - print u" proxy_password: %s" % proxy_password - - if http_proxy or https_proxy: - command.append('--proxy-anyauth') - - if proxy_username or proxy_password: - command.extend(['-U', u"%s:%s" % (proxy_username, proxy_password)]) - - if http_proxy: - os.putenv('http_proxy', http_proxy) - if https_proxy: - os.putenv('HTTPS_PROXY', https_proxy) - - command.append(url) - - while tries > 0: - tries -= 1 - try: - output = self.execute(command) - - with open(self.tmp_file, 'r') as f: - headers = f.read() - self.clean_tmp_file() - - limit = 1 - limit_remaining = 1 - status = '200 OK' - for header in headers.splitlines(): - if header[0:5] == 'HTTP/': - status = re.sub('^HTTP/\d\.\d\s+', '', header) - if header.lower()[0:22] == 'x-ratelimit-remaining:': - limit_remaining = header.lower()[22:].strip() - if header.lower()[0:18] == 'x-ratelimit-limit:': - limit = header.lower()[18:].strip() - - if debug: - self.print_debug(self.stderr) - - if str(limit_remaining) == '0': - hostname = urlparse.urlparse(url).hostname - raise RateLimitException(hostname, limit) - - if status != '200 OK': - e = NonCleanExitError(22) - e.stderr = status - raise e - - return output - - except (NonCleanExitError) as (e): - # Stderr is used for both the error message and the debug info - # so we need to process it to extra the debug info - if self.settings.get('debug'): - e.stderr = self.print_debug(e.stderr) - - self.clean_tmp_file() - - if e.returncode == 22: - code = re.sub('^.*?(\d+)([\w\s]+)?$', '\\1', e.stderr) - if code == '503': - # GitHub and BitBucket seem to rate limit via 503 - print ('%s: Downloading %s was rate limited' + - ', trying again') % (__name__, url) - continue - error_string = 'HTTP error ' + code - elif e.returncode == 6: - error_string = 'URL error host not found' - elif e.returncode == 28: - # GitHub and BitBucket seem to time out a lot - print ('%s: Downloading %s timed out, trying ' + - 'again') % (__name__, url) - continue - else: - error_string = e.stderr.rstrip() - - print '%s: %s %s downloading %s.' % (__name__, error_message, - error_string, url) - break - - return False - - def print_debug(self, string): - section = 'General' - last_section = None - - output = '' - - for line in string.splitlines(): - # Placeholder for body of request - if line and line[0:2] == '{ ': - continue - - if len(line) > 1: - subtract = 0 - if line[0:2] == '* ': - section = 'General' - subtract = 2 - elif line[0:2] == '> ': - section = 'Write' - subtract = 2 - elif line[0:2] == '< ': - section = 'Read' - subtract = 2 - line = line[subtract:] - - # If the line does not start with "* ", "< ", "> " or " " - # then it is a real stderr message - if subtract == 0 and line[0:2] != ' ': - output += line - continue - - if line.strip() == '': - continue - - if section != last_section: - print "%s: Curl HTTP Debug %s" % (__name__, section) - - print ' ' + line - last_section = section - - return output - - -# A cache of channel and repository info to allow users to install multiple -# packages without having to wait for the metadata to be downloaded more -# than once. The keys are managed locally by the utilizing code. -_channel_repository_cache = {} - - -class RepositoryDownloader(threading.Thread): - """ - Downloads information about a repository in the background - - :param package_manager: - An instance of :class:`PackageManager` used to download files - - :param name_map: - The dict of name mapping for URL slug -> package name - - :param repo: - The URL of the repository to download info about - """ - - def __init__(self, package_manager, name_map, repo): - self.package_manager = package_manager - self.repo = repo - self.packages = {} - self.name_map = name_map - threading.Thread.__init__(self) - - def run(self): - for provider_class in _package_providers: - provider = provider_class(self.repo, self.package_manager) - if provider.match_url(): - break - packages = provider.get_packages() - if packages == False: - self.packages = False - return - - mapped_packages = {} - for package in packages.keys(): - mapped_package = self.name_map.get(package, package) - mapped_packages[mapped_package] = packages[package] - mapped_packages[mapped_package]['name'] = mapped_package - packages = mapped_packages - - self.packages = packages - self.renamed_packages = provider.get_renamed_packages() - self.unavailable_packages = provider.get_unavailable_packages() - - -class VcsUpgrader(): - """ - Base class for updating packages that are a version control repository on local disk - - :param vcs_binary: - The full filesystem path to the executable for the version control - system. May be set to None to allow the code to try and find it. - - :param update_command: - The command to pass to the version control executable to update the - repository. - - :param working_copy: - The local path to the working copy/package directory - - :param cache_length: - The lenth of time to cache if incoming changesets are available - """ - - def __init__(self, vcs_binary, update_command, working_copy, cache_length, debug): - self.binary = vcs_binary - self.update_command = update_command - self.working_copy = working_copy - self.cache_length = cache_length - self.debug = debug - - def execute(self, args, dir): - """ - Creates a subprocess with the executable/args - - :param args: - A list of the executable path and all arguments to it - - :param dir: - The directory in which to run the executable - - :return: A string of the executable output - """ - - startupinfo = None - if os.name == 'nt': - startupinfo = subprocess.STARTUPINFO() - startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW - - if self.debug: - print u"%s: Trying to execute command %s" % ( - __name__, create_cmd(args)) - proc = subprocess.Popen(args, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - startupinfo=startupinfo, cwd=dir) - - return proc.stdout.read().replace('\r\n', '\n').rstrip(' \n\r') - - def find_binary(self, name): - """ - Locates the executable by looking in the PATH and well-known directories - - :param name: - The string filename of the executable - - :return: The filesystem path to the executable, or None if not found - """ - - if self.binary: - if self.debug: - print u"%s: Using \"%s_binary\" from settings \"%s\"" % ( - __name__, self.vcs_type, self.binary) - return self.binary - - # Try the path first - for dir in os.environ['PATH'].split(os.pathsep): - path = os.path.join(dir, name) - if os.path.exists(path): - if self.debug: - print u"%s: Found %s at \"%s\"" % (__name__, self.vcs_type, - path) - return path - - # This is left in for backwards compatibility and for windows - # users who may have the binary, albeit in a common dir that may - # not be part of the PATH - if os.name == 'nt': - dirs = ['C:\\Program Files\\Git\\bin', - 'C:\\Program Files (x86)\\Git\\bin', - 'C:\\Program Files\\TortoiseGit\\bin', - 'C:\\Program Files\\Mercurial', - 'C:\\Program Files (x86)\\Mercurial', - 'C:\\Program Files (x86)\\TortoiseHg', - 'C:\\Program Files\\TortoiseHg', - 'C:\\cygwin\\bin'] - else: - dirs = ['/usr/local/git/bin'] - - for dir in dirs: - path = os.path.join(dir, name) - if os.path.exists(path): - if self.debug: - print u"%s: Found %s at \"%s\"" % (__name__, self.vcs_type, - path) - return path - - if self.debug: - print u"%s: Could not find %s on your machine" % (__name__, - self.vcs_type) - return None - - -class GitUpgrader(VcsUpgrader): - """ - Allows upgrading a local git-repository-based package - """ - - vcs_type = 'git' - - def retrieve_binary(self): - """ - Returns the path to the git executable - - :return: The string path to the executable or False on error - """ - - name = 'git' - if os.name == 'nt': - name += '.exe' - binary = self.find_binary(name) - if binary and os.path.isdir(binary): - full_path = os.path.join(binary, name) - if os.path.exists(full_path): - binary = full_path - if not binary: - sublime.error_message(('%s: Unable to find %s. ' + - 'Please set the git_binary setting by accessing the ' + - 'Preferences > Package Settings > %s > ' + - u'Settings – User menu entry. The Settings – Default entry ' + - 'can be used for reference, but changes to that will be ' + - 'overwritten upon next upgrade.') % (__name__, name, __name__)) - return False - - if os.name == 'nt': - tortoise_plink = self.find_binary('TortoisePlink.exe') - if tortoise_plink: - os.environ.setdefault('GIT_SSH', tortoise_plink) - return binary - - def run(self): - """ - Updates the repository with remote changes - - :return: False or error, or True on success - """ - - binary = self.retrieve_binary() - if not binary: - return False - args = [binary] - args.extend(self.update_command) - self.execute(args, self.working_copy) - return True - - def incoming(self): - """:return: bool if remote revisions are available""" - - cache_key = self.working_copy + '.incoming' - working_copy_cache = _channel_repository_cache.get(cache_key) - if working_copy_cache and working_copy_cache.get('time') > \ - time.time(): - return working_copy_cache.get('data') - - binary = self.retrieve_binary() - if not binary: - return False - self.execute([binary, 'fetch'], self.working_copy) - args = [binary, 'log'] - args.append('..' + '/'.join(self.update_command[-2:])) - output = self.execute(args, self.working_copy) - incoming = len(output) > 0 - - _channel_repository_cache[cache_key] = { - 'time': time.time() + self.cache_length, - 'data': incoming - } - return incoming - - -class HgUpgrader(VcsUpgrader): - """ - Allows upgrading a local mercurial-repository-based package - """ - - vcs_type = 'hg' - - def retrieve_binary(self): - """ - Returns the path to the hg executable - - :return: The string path to the executable or False on error - """ - - name = 'hg' - if os.name == 'nt': - name += '.exe' - binary = self.find_binary(name) - if binary and os.path.isdir(binary): - full_path = os.path.join(binary, name) - if os.path.exists(full_path): - binary = full_path - if not binary: - sublime.error_message(('%s: Unable to find %s. ' + - 'Please set the hg_binary setting by accessing the ' + - 'Preferences > Package Settings > %s > ' + - u'Settings – User menu entry. The Settings – Default entry ' + - 'can be used for reference, but changes to that will be ' + - 'overwritten upon next upgrade.') % (__name__, name, __name__)) - return False - return binary - - def run(self): - """ - Updates the repository with remote changes - - :return: False or error, or True on success - """ - - binary = self.retrieve_binary() - if not binary: - return False - args = [binary] - args.extend(self.update_command) - self.execute(args, self.working_copy) - return True - - def incoming(self): - """:return: bool if remote revisions are available""" - - cache_key = self.working_copy + '.incoming' - working_copy_cache = _channel_repository_cache.get(cache_key) - if working_copy_cache and working_copy_cache.get('time') > \ - time.time(): - return working_copy_cache.get('data') - - binary = self.retrieve_binary() - if not binary: - return False - args = [binary, 'in', '-q'] - args.append(self.update_command[-1]) - output = self.execute(args, self.working_copy) - incoming = len(output) > 0 - - _channel_repository_cache[cache_key] = { - 'time': time.time() + self.cache_length, - 'data': incoming - } - return incoming - - -def clear_directory(directory, ignore_paths=None): - was_exception = False - for root, dirs, files in os.walk(directory, topdown=False): - paths = [os.path.join(root, f) for f in files] - paths.extend([os.path.join(root, d) for d in dirs]) - - for path in paths: - try: - # Don't delete the metadata file, that way we have it - # when the reinstall happens, and the appropriate - # usage info can be sent back to the server - if ignore_paths and path in ignore_paths: - continue - if os.path.isdir(path): - os.rmdir(path) - else: - os.remove(path) - except (OSError, IOError) as (e): - was_exception = True - - return not was_exception - - - -class PackageManager(): - """ - Allows downloading, creating, installing, upgrading, and deleting packages - - Delegates metadata retrieval to the _channel_providers and - _package_providers classes. Uses VcsUpgrader-based classes for handling - git and hg repositories in the Packages folder. Downloader classes are - utilized to fetch contents of URLs. - - Also handles displaying package messaging, and sending usage information to - the usage server. - """ - - def __init__(self): - # Here we manually copy the settings since sublime doesn't like - # code accessing settings from threads - self.settings = {} - settings = sublime.load_settings(__name__ + '.sublime-settings') - for setting in ['timeout', 'repositories', 'repository_channels', - 'package_name_map', 'dirs_to_ignore', 'files_to_ignore', - 'package_destination', 'cache_length', 'auto_upgrade', - 'files_to_ignore_binary', 'files_to_keep', 'dirs_to_keep', - 'git_binary', 'git_update_command', 'hg_binary', - 'hg_update_command', 'http_proxy', 'https_proxy', - 'auto_upgrade_ignore', 'auto_upgrade_frequency', - 'submit_usage', 'submit_url', 'renamed_packages', - 'files_to_include', 'files_to_include_binary', 'certs', - 'ignore_vcs_packages', 'proxy_username', 'proxy_password', - 'debug', 'user_agent']: - if settings.get(setting) == None: - continue - self.settings[setting] = settings.get(setting) - - # https_proxy will inherit from http_proxy unless it is set to a - # string value or false - no_https_proxy = self.settings.get('https_proxy') in ["", None] - if no_https_proxy and self.settings.get('http_proxy'): - self.settings['https_proxy'] = self.settings.get('http_proxy') - if self.settings['https_proxy'] == False: - self.settings['https_proxy'] = '' - - self.settings['platform'] = sublime.platform() - self.settings['version'] = sublime.version() - - def compare_versions(self, version1, version2): - """ - Compares to version strings to see which is greater - - Date-based version numbers (used by GitHub and BitBucket providers) - are automatically pre-pended with a 0 so they are always less than - version 1.0. - - :return: - -1 if version1 is less than version2 - 0 if they are equal - 1 if version1 is greater than version2 - """ - - def date_compat(v): - # We prepend 0 to all date-based version numbers so that developers - # may switch to explicit versioning from GitHub/BitBucket - # versioning based on commit dates - date_match = re.match('(\d{4})\.(\d{2})\.(\d{2})\.(\d{2})\.(\d{2})\.(\d{2})$', v) - if date_match: - v = '0.%s.%s.%s.%s.%s.%s' % date_match.groups() - return v - - def semver_compat(v): - # When translating dates into semver, the way to get each date - # segment into the version is to treat the year and month as - # minor and patch, and then the rest as a numeric build version - # with four different parts. The result looks like: - # 0.2012.11+10.31.23.59 - date_match = re.match('(\d{4}(?:\.\d{2}){2})\.(\d{2}(?:\.\d{2}){3})$', v) - if date_match: - v = '%s+%s' % (date_match.group(1), date_match.group(2)) - - # Semver must have major, minor, patch - elif re.match('^\d+$', v): - v += '.0.0' - elif re.match('^\d+\.\d+$', v): - v += '.0' - return v - - def cmp_compat(v): - return [int(x) for x in re.sub(r'(\.0+)*$', '', v).split(".")] - - version1 = date_compat(version1) - version2 = date_compat(version2) - try: - return semver.compare(semver_compat(version1), semver_compat(version2)) - except (ValueError): - return cmp(cmp_compat(version1), cmp_compat(version2)) - - def download_url(self, url, error_message): - """ - Downloads a URL and returns the contents - - :param url: - The string URL to download - - :param error_message: - The error message to include if the download fails - - :return: - The string contents of the URL, or False on error - """ - - has_ssl = 'ssl' in sys.modules and hasattr(urllib2, 'HTTPSHandler') - is_ssl = re.search('^https://', url) != None - downloader = None - - if (is_ssl and has_ssl) or not is_ssl: - downloader = UrlLib2Downloader(self.settings) - else: - for downloader_class in [CurlDownloader, WgetDownloader]: - try: - downloader = downloader_class(self.settings) - break - except (BinaryNotFoundError): - pass - - if not downloader: - sublime.error_message(('%s: Unable to download %s due to no ' + - 'ssl module available and no capable program found. Please ' + - 'install curl or wget.') % (__name__, url)) - return False - - url = url.replace(' ', '%20') - hostname = urlparse.urlparse(url).hostname.lower() - timeout = self.settings.get('timeout', 3) - - rate_limited_cache = _channel_repository_cache.get('rate_limited_domains', {}) - if rate_limited_cache.get('time') and rate_limited_cache.get('time') > time.time(): - rate_limited_domains = rate_limited_cache.get('data', []) - else: - rate_limited_domains = [] - - if self.settings.get('debug'): - try: - ip = socket.gethostbyname(hostname) - except (socket.gaierror) as (e): - ip = unicode_from_os(e) - - print u"%s: Download Debug" % __name__ - print u" URL: %s" % url - print u" Resolved IP: %s" % ip - print u" Timeout: %s" % str(timeout) - - if hostname in rate_limited_domains: - if self.settings.get('debug'): - print u" Skipping due to hitting rate limit for %s" % hostname - return False - - try: - return downloader.download(url, error_message, timeout, 3) - except (RateLimitException) as (e): - - rate_limited_domains.append(hostname) - _channel_repository_cache['rate_limited_domains'] = { - 'data': rate_limited_domains, - 'time': time.time() + self.settings.get('cache_length', - 300) - } - - print ('%s: Hit rate limit of %s for %s, skipping all futher ' + - 'download requests for this domain') % (__name__, - e.limit, e.host) - return False - - def get_metadata(self, package): - """ - Returns the package metadata for an installed package - - :return: - A dict with the keys: - version - url - description - or an empty dict on error - """ - - metadata_filename = os.path.join(self.get_package_dir(package), - 'package-metadata.json') - if os.path.exists(metadata_filename): - with open(metadata_filename) as f: - try: - return json.load(f) - except (ValueError): - return {} - return {} - - def list_repositories(self): - """ - Returns a master list of all repositories pulled from all sources - - These repositories come from the channels specified in the - "repository_channels" setting, plus any repositories listed in the - "repositories" setting. - - :return: - A list of all available repositories - """ - - repositories = self.settings.get('repositories') - repository_channels = self.settings.get('repository_channels') - for channel in repository_channels: - channel = channel.strip() - - channel_repositories = None - - # Caches various info from channels for performance - cache_key = channel + '.repositories' - repositories_cache = _channel_repository_cache.get(cache_key) - if repositories_cache and repositories_cache.get('time') > \ - time.time(): - channel_repositories = repositories_cache.get('data') - - name_map_cache_key = channel + '.package_name_map' - name_map_cache = _channel_repository_cache.get( - name_map_cache_key) - if name_map_cache and name_map_cache.get('time') > \ - time.time(): - name_map = name_map_cache.get('data') - name_map.update(self.settings.get('package_name_map', {})) - self.settings['package_name_map'] = name_map - - renamed_cache_key = channel + '.renamed_packages' - renamed_cache = _channel_repository_cache.get( - renamed_cache_key) - if renamed_cache and renamed_cache.get('time') > \ - time.time(): - renamed_packages = renamed_cache.get('data') - renamed_packages.update(self.settings.get('renamed_packages', - {})) - self.settings['renamed_packages'] = renamed_packages - - unavailable_cache_key = channel + '.unavailable_packages' - unavailable_cache = _channel_repository_cache.get( - unavailable_cache_key) - if unavailable_cache and unavailable_cache.get('time') > \ - time.time(): - unavailable_packages = unavailable_cache.get('data') - unavailable_packages.extend(self.settings.get('unavailable_packages', - [])) - self.settings['unavailable_packages'] = unavailable_packages - - certs_cache_key = channel + '.certs' - certs_cache = _channel_repository_cache.get(certs_cache_key) - if certs_cache and certs_cache.get('time') > time.time(): - certs = self.settings.get('certs', {}) - certs.update(certs_cache.get('data')) - self.settings['certs'] = certs - - # If any of the info was not retrieved from the cache, we need to - # grab the channel to get it - if channel_repositories == None or \ - self.settings.get('package_name_map') == None or \ - self.settings.get('renamed_packages') == None: - - for provider_class in _channel_providers: - provider = provider_class(channel, self) - if provider.match_url(): - break - - channel_repositories = provider.get_repositories() - - if channel_repositories == False: - continue - _channel_repository_cache[cache_key] = { - 'time': time.time() + self.settings.get('cache_length', - 300), - 'data': channel_repositories - } - - for repo in channel_repositories: - if provider.get_packages(repo) == False: - continue - packages_cache_key = repo + '.packages' - _channel_repository_cache[packages_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', - 300), - 'data': provider.get_packages(repo) - } - - # Have the local name map override the one from the channel - name_map = provider.get_name_map() - name_map.update(self.settings.get('package_name_map', {})) - self.settings['package_name_map'] = name_map - _channel_repository_cache[name_map_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', - 300), - 'data': name_map - } - - renamed_packages = provider.get_renamed_packages() - _channel_repository_cache[renamed_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', - 300), - 'data': renamed_packages - } - if renamed_packages: - self.settings['renamed_packages'] = self.settings.get( - 'renamed_packages', {}) - self.settings['renamed_packages'].update(renamed_packages) - - unavailable_packages = provider.get_unavailable_packages() - _channel_repository_cache[unavailable_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', - 300), - 'data': unavailable_packages - } - if unavailable_packages: - self.settings['unavailable_packages'] = self.settings.get( - 'unavailable_packages', []) - self.settings['unavailable_packages'].extend(unavailable_packages) - - certs = provider.get_certs() - _channel_repository_cache[certs_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', - 300), - 'data': certs - } - if certs: - self.settings['certs'] = self.settings.get('certs', {}) - self.settings['certs'].update(certs) - - repositories.extend(channel_repositories) - return [repo.strip() for repo in repositories] - - def list_available_packages(self): - """ - Returns a master list of every available package from all sources - - :return: - A dict in the format: - { - 'Package Name': { - # Package details - see example-packages.json for format - }, - ... - } - """ - - repositories = self.list_repositories() - packages = {} - downloaders = [] - grouped_downloaders = {} - - # Repositories are run in reverse order so that the ones first - # on the list will overwrite those last on the list - for repo in repositories[::-1]: - repository_packages = None - - cache_key = repo + '.packages' - packages_cache = _channel_repository_cache.get(cache_key) - if packages_cache and packages_cache.get('time') > \ - time.time(): - repository_packages = packages_cache.get('data') - packages.update(repository_packages) - - if repository_packages == None: - downloader = RepositoryDownloader(self, - self.settings.get('package_name_map', {}), repo) - domain = re.sub('^https?://[^/]*?(\w+\.\w+)($|/.*$)', '\\1', - repo) - - # downloaders are grouped by domain so that multiple can - # be run in parallel without running into API access limits - if not grouped_downloaders.get(domain): - grouped_downloaders[domain] = [] - grouped_downloaders[domain].append(downloader) - - # Allows creating a separate named function for each downloader - # delay. Not having this contained in a function causes all of the - # schedules to reference the same downloader.start() - def schedule(downloader, delay): - downloader.has_started = False - - def inner(): - downloader.start() - downloader.has_started = True - sublime.set_timeout(inner, delay) - - # Grabs every repo grouped by domain and delays the start - # of each download from that domain by a fixed amount - for domain_downloaders in grouped_downloaders.values(): - for i in range(len(domain_downloaders)): - downloader = domain_downloaders[i] - downloaders.append(downloader) - schedule(downloader, i * 150) - - complete = [] - - # Wait for all of the downloaders to finish - while downloaders: - downloader = downloaders.pop() - if downloader.has_started: - downloader.join() - complete.append(downloader) - else: - downloaders.insert(0, downloader) - - # Grabs the results and stuff if all in the cache - for downloader in complete: - repository_packages = downloader.packages - if repository_packages == False: - continue - cache_key = downloader.repo + '.packages' - _channel_repository_cache[cache_key] = { - 'time': time.time() + self.settings.get('cache_length', 300), - 'data': repository_packages - } - packages.update(repository_packages) - - renamed_packages = downloader.renamed_packages - if renamed_packages == False: - continue - renamed_cache_key = downloader.repo + '.renamed_packages' - _channel_repository_cache[renamed_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', 300), - 'data': renamed_packages - } - if renamed_packages: - self.settings['renamed_packages'] = self.settings.get( - 'renamed_packages', {}) - self.settings['renamed_packages'].update(renamed_packages) - - unavailable_packages = downloader.unavailable_packages - unavailable_cache_key = downloader.repo + '.unavailable_packages' - _channel_repository_cache[unavailable_cache_key] = { - 'time': time.time() + self.settings.get('cache_length', 300), - 'data': unavailable_packages - } - if unavailable_packages: - self.settings['unavailable_packages'] = self.settings.get( - 'unavailable_packages', []) - self.settings['unavailable_packages'].extend(unavailable_packages) - - return packages - - def list_packages(self): - """ :return: A list of all installed, non-default, package names""" - - package_names = os.listdir(sublime.packages_path()) - package_names = [path for path in package_names if - os.path.isdir(os.path.join(sublime.packages_path(), path))] - - # Ignore things to be deleted - ignored = [] - for package in package_names: - cleanup_file = os.path.join(sublime.packages_path(), package, - 'package-control.cleanup') - if os.path.exists(cleanup_file): - ignored.append(package) - - packages = list(set(package_names) - set(ignored) - - set(self.list_default_packages())) - packages = sorted(packages, key=lambda s: s.lower()) - - return packages - - def list_all_packages(self): - """ :return: A list of all installed package names, including default packages""" - - packages = os.listdir(sublime.packages_path()) - packages = sorted(packages, key=lambda s: s.lower()) - return packages - - def list_default_packages(self): - """ :return: A list of all default package names""" - - files = os.listdir(os.path.join(os.path.dirname( - sublime.packages_path()), 'Pristine Packages')) - files = list(set(files) - set(os.listdir( - sublime.installed_packages_path()))) - packages = [file.replace('.sublime-package', '') for file in files] - packages = sorted(packages, key=lambda s: s.lower()) - return packages - - def get_package_dir(self, package): - """:return: The full filesystem path to the package directory""" - - return os.path.join(sublime.packages_path(), package) - - def get_mapped_name(self, package): - """:return: The name of the package after passing through mapping rules""" - - return self.settings.get('package_name_map', {}).get(package, package) - - def create_package(self, package_name, package_destination, - binary_package=False): - """ - Creates a .sublime-package file from the running Packages directory - - :param package_name: - The package to create a .sublime-package file for - - :param package_destination: - The full filesystem path of the directory to save the new - .sublime-package file in. - - :param binary_package: - If the created package should follow the binary package include/ - exclude patterns from the settings. These normally include a setup - to exclude .py files and include .pyc files, but that can be - changed via settings. - - :return: bool if the package file was successfully created - """ - - package_dir = self.get_package_dir(package_name) + '/' - - if not os.path.exists(package_dir): - sublime.error_message(('%s: The folder for the package name ' + - 'specified, %s, does not exist in %s') % - (__name__, package_name, sublime.packages_path())) - return False - - package_filename = package_name + '.sublime-package' - package_path = os.path.join(package_destination, - package_filename) - - if not os.path.exists(sublime.installed_packages_path()): - os.mkdir(sublime.installed_packages_path()) - - if os.path.exists(package_path): - os.remove(package_path) - - try: - package_file = zipfile.ZipFile(package_path, "w", - compression=zipfile.ZIP_DEFLATED) - except (OSError, IOError) as (exception): - sublime.error_message(('%s: An error occurred creating the ' + - 'package file %s in %s. %s') % (__name__, package_filename, - package_destination, unicode_from_os(exception))) - return False - - dirs_to_ignore = self.settings.get('dirs_to_ignore', []) - if not binary_package: - files_to_ignore = self.settings.get('files_to_ignore', []) - files_to_include = self.settings.get('files_to_include', []) - else: - files_to_ignore = self.settings.get('files_to_ignore_binary', []) - files_to_include = self.settings.get('files_to_include_binary', []) - - package_dir_regex = re.compile('^' + re.escape(package_dir)) - for root, dirs, files in os.walk(package_dir): - [dirs.remove(dir) for dir in dirs if dir in dirs_to_ignore] - paths = dirs - paths.extend(files) - for path in paths: - full_path = os.path.join(root, path) - relative_path = re.sub(package_dir_regex, '', full_path) - - ignore_matches = [fnmatch(relative_path, p) for p in files_to_ignore] - include_matches = [fnmatch(relative_path, p) for p in files_to_include] - if any(ignore_matches) and not any(include_matches): - continue - - if os.path.isdir(full_path): - continue - package_file.write(full_path, relative_path) - - package_file.close() - - return True - - def install_package(self, package_name): - """ - Downloads and installs (or upgrades) a package - - Uses the self.list_available_packages() method to determine where to - retrieve the package file from. - - The install process consists of: - - 1. Finding the package - 2. Downloading the .sublime-package/.zip file - 3. Extracting the package file - 4. Showing install/upgrade messaging - 5. Submitting usage info - 6. Recording that the package is installed - - :param package_name: - The package to download and install - - :return: bool if the package was successfully installed - """ - - packages = self.list_available_packages() - - if package_name in self.settings.get('unavailable_packages', []): - print ('%s: The package "%s" is not available ' + - 'on this platform.') % (__name__, package_name) - return False - - if package_name not in packages.keys(): - sublime.error_message(('%s: The package specified, %s, is ' + - 'not available.') % (__name__, package_name)) - return False - - download = packages[package_name]['downloads'][0] - url = download['url'] - - package_filename = package_name + \ - '.sublime-package' - package_path = os.path.join(sublime.installed_packages_path(), - package_filename) - pristine_package_path = os.path.join(os.path.dirname( - sublime.packages_path()), 'Pristine Packages', package_filename) - - package_dir = self.get_package_dir(package_name) - - package_metadata_file = os.path.join(package_dir, - 'package-metadata.json') - - if os.path.exists(os.path.join(package_dir, '.git')): - if self.settings.get('ignore_vcs_packages'): - sublime.error_message(('%s: Skipping git package %s since ' + - 'the setting ignore_vcs_packages is set to true') % - (__name__, package_name)) - return False - return GitUpgrader(self.settings['git_binary'], - self.settings['git_update_command'], package_dir, - self.settings['cache_length'], self.settings['debug']).run() - elif os.path.exists(os.path.join(package_dir, '.hg')): - if self.settings.get('ignore_vcs_packages'): - sublime.error_message(('%s: Skipping hg package %s since ' + - 'the setting ignore_vcs_packages is set to true') % - (__name__, package_name)) - return False - return HgUpgrader(self.settings['hg_binary'], - self.settings['hg_update_command'], package_dir, - self.settings['cache_length'], self.settings['debug']).run() - - is_upgrade = os.path.exists(package_metadata_file) - old_version = None - if is_upgrade: - old_version = self.get_metadata(package_name).get('version') - - package_bytes = self.download_url(url, 'Error downloading package.') - if package_bytes == False: - return False - with open(package_path, "wb") as package_file: - package_file.write(package_bytes) - - if not os.path.exists(package_dir): - os.mkdir(package_dir) - - # We create a backup copy incase something was edited - else: - try: - backup_dir = os.path.join(os.path.dirname( - sublime.packages_path()), 'Backup', - datetime.datetime.now().strftime('%Y%m%d%H%M%S')) - if not os.path.exists(backup_dir): - os.makedirs(backup_dir) - package_backup_dir = os.path.join(backup_dir, package_name) - shutil.copytree(package_dir, package_backup_dir) - except (OSError, IOError) as (exception): - sublime.error_message(('%s: An error occurred while trying ' + - 'to backup the package directory for %s. %s') % - (__name__, package_name, unicode_from_os(exception))) - shutil.rmtree(package_backup_dir) - return False - - try: - package_zip = zipfile.ZipFile(package_path, 'r') - except (zipfile.BadZipfile): - sublime.error_message(('%s: An error occurred while ' + - 'trying to unzip the package file for %s. Please try ' + - 'installing the package again.') % (__name__, package_name)) - return False - - root_level_paths = [] - last_path = None - for path in package_zip.namelist(): - last_path = path - if path.find('/') in [len(path) - 1, -1]: - root_level_paths.append(path) - if path[0] == '/' or path.find('../') != -1 or path.find('..\\') != -1: - sublime.error_message(('%s: The package specified, %s, ' + - 'contains files outside of the package dir and cannot ' + - 'be safely installed.') % (__name__, package_name)) - return False - - if last_path and len(root_level_paths) == 0: - root_level_paths.append(last_path[0:last_path.find('/') + 1]) - - os.chdir(package_dir) - - overwrite_failed = False - - # Here we don’t use .extractall() since it was having issues on OS X - skip_root_dir = len(root_level_paths) == 1 and \ - root_level_paths[0].endswith('/') - extracted_paths = [] - for path in package_zip.namelist(): - dest = path - try: - if not isinstance(dest, unicode): - dest = unicode(dest, 'utf-8', 'strict') - except (UnicodeDecodeError): - dest = unicode(dest, 'cp1252', 'replace') - - if os.name == 'nt': - regex = ':|\*|\?|"|<|>|\|' - if re.search(regex, dest) != None: - print ('%s: Skipping file from package named %s due to ' + - 'an invalid filename') % (__name__, path) - continue - - # If there was only a single directory in the package, we remove - # that folder name from the paths as we extract entries - if skip_root_dir: - dest = dest[len(root_level_paths[0]):] - - if os.name == 'nt': - dest = dest.replace('/', '\\') - else: - dest = dest.replace('\\', '/') - - dest = os.path.join(package_dir, dest) - - def add_extracted_dirs(dir): - while dir not in extracted_paths: - extracted_paths.append(dir) - dir = os.path.dirname(dir) - if dir == package_dir: - break - - if path.endswith('/'): - if not os.path.exists(dest): - os.makedirs(dest) - add_extracted_dirs(dest) - else: - dest_dir = os.path.dirname(dest) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) - add_extracted_dirs(dest_dir) - extracted_paths.append(dest) - try: - open(dest, 'wb').write(package_zip.read(path)) - except (IOError) as (e): - message = unicode_from_os(e) - if re.search('[Ee]rrno 13', message): - overwrite_failed = True - break - print ('%s: Skipping file from package named %s due to ' + - 'an invalid filename') % (__name__, path) - - except (UnicodeDecodeError): - print ('%s: Skipping file from package named %s due to ' + - 'an invalid filename') % (__name__, path) - package_zip.close() - - - # If upgrading failed, queue the package to upgrade upon next start - if overwrite_failed: - reinstall_file = os.path.join(package_dir, 'package-control.reinstall') - open(reinstall_file, 'w').close() - - # Don't delete the metadata file, that way we have it - # when the reinstall happens, and the appropriate - # usage info can be sent back to the server - clear_directory(package_dir, [reinstall_file, package_metadata_file]) - - sublime.error_message(('%s: An error occurred while trying to ' + - 'upgrade %s. Please restart Sublime Text to finish the ' + - 'upgrade.') % (__name__, package_name)) - return False - - - # Here we clean out any files that were not just overwritten. It is ok - # if there is an error removing a file. The next time there is an - # upgrade, it should be cleaned out successfully then. - clear_directory(package_dir, extracted_paths) - - - self.print_messages(package_name, package_dir, is_upgrade, old_version) - - with open(package_metadata_file, 'w') as f: - metadata = { - "version": packages[package_name]['downloads'][0]['version'], - "url": packages[package_name]['url'], - "description": packages[package_name]['description'] - } - json.dump(metadata, f) - - # Submit install and upgrade info - if is_upgrade: - params = { - 'package': package_name, - 'operation': 'upgrade', - 'version': packages[package_name]['downloads'][0]['version'], - 'old_version': old_version - } - else: - params = { - 'package': package_name, - 'operation': 'install', - 'version': packages[package_name]['downloads'][0]['version'] - } - self.record_usage(params) - - # Record the install in the settings file so that you can move - # settings across computers and have the same packages installed - def save_package(): - settings = sublime.load_settings(__name__ + '.sublime-settings') - installed_packages = settings.get('installed_packages', []) - if not installed_packages: - installed_packages = [] - installed_packages.append(package_name) - installed_packages = list(set(installed_packages)) - installed_packages = sorted(installed_packages, - key=lambda s: s.lower()) - settings.set('installed_packages', installed_packages) - sublime.save_settings(__name__ + '.sublime-settings') - sublime.set_timeout(save_package, 1) - - # Here we delete the package file from the installed packages directory - # since we don't want to accidentally overwrite user changes - os.remove(package_path) - # We have to remove the pristine package too or else Sublime Text 2 - # will silently delete the package - if os.path.exists(pristine_package_path): - os.remove(pristine_package_path) - - os.chdir(sublime.packages_path()) - return True - - def print_messages(self, package, package_dir, is_upgrade, old_version): - """ - Prints out package install and upgrade messages - - The functionality provided by this allows package maintainers to - show messages to the user when a package is installed, or when - certain version upgrade occur. - - :param package: - The name of the package the message is for - - :param package_dir: - The full filesystem path to the package directory - - :param is_upgrade: - If the install was actually an upgrade - - :param old_version: - The string version of the package before the upgrade occurred - """ - - messages_file = os.path.join(package_dir, 'messages.json') - if not os.path.exists(messages_file): - return - - messages_fp = open(messages_file, 'r') - try: - message_info = json.load(messages_fp) - except (ValueError): - print '%s: Error parsing messages.json for %s' % (__name__, package) - return - messages_fp.close() - - output = '' - if not is_upgrade and message_info.get('install'): - install_messages = os.path.join(package_dir, - message_info.get('install')) - message = '\n\n%s:\n%s\n\n ' % (package, - ('-' * len(package))) - with open(install_messages, 'r') as f: - message += unicode(f.read(), 'utf-8', errors='replace').replace('\n', '\n ') - output += message + '\n' - - elif is_upgrade and old_version: - upgrade_messages = list(set(message_info.keys()) - - set(['install'])) - upgrade_messages = sorted(upgrade_messages, - cmp=self.compare_versions, reverse=True) - for version in upgrade_messages: - if self.compare_versions(old_version, version) >= 0: - break - if not output: - message = '\n\n%s:\n%s\n' % (package, - ('-' * len(package))) - output += message - upgrade_messages = os.path.join(package_dir, - message_info.get(version)) - message = '\n ' - with open(upgrade_messages, 'r') as f: - message += unicode(f.read(), 'utf-8', errors='replace').replace('\n', '\n ') - output += message + '\n' - - if not output: - return - - def print_to_panel(): - window = sublime.active_window() - - views = window.views() - view = None - for _view in views: - if _view.name() == 'Package Control Messages': - view = _view - break - - if not view: - view = window.new_file() - view.set_name('Package Control Messages') - view.set_scratch(True) - - def write(string): - edit = view.begin_edit() - view.insert(edit, view.size(), string) - view.end_edit(edit) - - if not view.size(): - view.settings().set("word_wrap", True) - write('Package Control Messages\n' + - '========================') - - write(output) - sublime.set_timeout(print_to_panel, 1) - - def remove_package(self, package_name): - """ - Deletes a package - - The deletion process consists of: - - 1. Deleting the directory (or marking it for deletion if deletion fails) - 2. Submitting usage info - 3. Removing the package from the list of installed packages - - :param package_name: - The package to delete - - :return: bool if the package was successfully deleted - """ - - installed_packages = self.list_packages() - - if package_name not in installed_packages: - sublime.error_message(('%s: The package specified, %s, is not ' + - 'installed.') % (__name__, package_name)) - return False - - os.chdir(sublime.packages_path()) - - # Give Sublime Text some time to ignore the package - time.sleep(1) - - package_filename = package_name + '.sublime-package' - package_path = os.path.join(sublime.installed_packages_path(), - package_filename) - installed_package_path = os.path.join(os.path.dirname( - sublime.packages_path()), 'Installed Packages', package_filename) - pristine_package_path = os.path.join(os.path.dirname( - sublime.packages_path()), 'Pristine Packages', package_filename) - package_dir = self.get_package_dir(package_name) - - version = self.get_metadata(package_name).get('version') - - try: - if os.path.exists(package_path): - os.remove(package_path) - except (OSError, IOError) as (exception): - sublime.error_message(('%s: An error occurred while trying to ' + - 'remove the package file for %s. %s') % (__name__, - package_name, unicode_from_os(exception))) - return False - - try: - if os.path.exists(installed_package_path): - os.remove(installed_package_path) - except (OSError, IOError) as (exception): - sublime.error_message(('%s: An error occurred while trying to ' + - 'remove the installed package file for %s. %s') % (__name__, - package_name, unicode_from_os(exception))) - return False - - try: - if os.path.exists(pristine_package_path): - os.remove(pristine_package_path) - except (OSError, IOError) as (exception): - sublime.error_message(('%s: An error occurred while trying to ' + - 'remove the pristine package file for %s. %s') % (__name__, - package_name, unicode_from_os(exception))) - return False - - # We don't delete the actual package dir immediately due to a bug - # in sublime_plugin.py - can_delete_dir = True - if not clear_directory(package_dir): - # If there is an error deleting now, we will mark it for - # cleanup the next time Sublime Text starts - open(os.path.join(package_dir, 'package-control.cleanup'), - 'w').close() - can_delete_dir = False - - params = { - 'package': package_name, - 'operation': 'remove', - 'version': version - } - self.record_usage(params) - - # Remove the package from the installed packages list - def clear_package(): - settings = sublime.load_settings('%s.sublime-settings' % __name__) - installed_packages = settings.get('installed_packages', []) - if not installed_packages: - installed_packages = [] - installed_packages.remove(package_name) - settings.set('installed_packages', installed_packages) - sublime.save_settings('%s.sublime-settings' % __name__) - sublime.set_timeout(clear_package, 1) - - if can_delete_dir: - os.rmdir(package_dir) - - return True - - def record_usage(self, params): - """ - Submits install, upgrade and delete actions to a usage server - - The usage information is currently displayed on the Package Control - community package list at http://wbond.net/sublime_packages/community - - :param params: - A dict of the information to submit - """ - - if not self.settings.get('submit_usage'): - return - params['package_control_version'] = \ - self.get_metadata('Package Control').get('version') - params['sublime_platform'] = self.settings.get('platform') - params['sublime_version'] = self.settings.get('version') - url = self.settings.get('submit_url') + '?' + urllib.urlencode(params) - - result = self.download_url(url, 'Error submitting usage information.') - if result == False: - return - - try: - result = json.loads(result) - if result['result'] != 'success': - raise ValueError() - except (ValueError): - print '%s: Error submitting usage information for %s' % (__name__, - params['package']) - - -class PackageCreator(): - """ - Abstract class for commands that create .sublime-package files - """ - - def show_panel(self): - """ - Shows a list of packages that can be turned into a .sublime-package file - """ - - self.manager = PackageManager() - self.packages = self.manager.list_packages() - if not self.packages: - sublime.error_message(('%s: There are no packages available to ' + - 'be packaged.') % (__name__)) - return - self.window.show_quick_panel(self.packages, self.on_done) - - def get_package_destination(self): - """ - Retrieves the destination for .sublime-package files - - :return: - A string - the path to the folder to save .sublime-package files in - """ - - destination = self.manager.settings.get('package_destination') - - # We check destination via an if statement instead of using - # the dict.get() method since the key may be set, but to a blank value - if not destination: - destination = os.path.join(os.path.expanduser('~'), 'Desktop') - - return destination - - -class CreatePackageCommand(sublime_plugin.WindowCommand, PackageCreator): - """ - Command to create a regular .sublime-package file - """ - - def run(self): - self.show_panel() - - def on_done(self, picked): - """ - Quick panel user selection handler - processes the user package - selection and create the package file - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - package_name = self.packages[picked] - package_destination = self.get_package_destination() - - if self.manager.create_package(package_name, package_destination): - self.window.run_command('open_dir', {"dir": - package_destination, "file": package_name + - '.sublime-package'}) - - -class CreateBinaryPackageCommand(sublime_plugin.WindowCommand, PackageCreator): - """ - Command to create a binary .sublime-package file. Binary packages in - general exclude the .py source files and instead include the .pyc files. - Actual included and excluded files are controlled by settings. - """ - - def run(self): - self.show_panel() - - def on_done(self, picked): - """ - Quick panel user selection handler - processes the user package - selection and create the package file - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - package_name = self.packages[picked] - package_destination = self.get_package_destination() - - if self.manager.create_package(package_name, package_destination, - binary_package=True): - self.window.run_command('open_dir', {"dir": - package_destination, "file": package_name + - '.sublime-package'}) - - -class PackageRenamer(): - """ - Class to handle renaming packages via the renamed_packages setting - gathered from channels and repositories. - """ - - def load_settings(self): - """ - Loads the list of installed packages from the - Package Control.sublime-settings file. - """ - - self.settings_file = '%s.sublime-settings' % __name__ - self.settings = sublime.load_settings(self.settings_file) - self.installed_packages = self.settings.get('installed_packages', []) - if not isinstance(self.installed_packages, list): - self.installed_packages = [] - - def rename_packages(self, installer): - """ - Renames any installed packages that the user has installed. - - :param installer: - An instance of :class:`PackageInstaller` - """ - - # Fetch the packages since that will pull in the renamed packages list - installer.manager.list_available_packages() - renamed_packages = installer.manager.settings.get('renamed_packages', {}) - if not renamed_packages: - renamed_packages = {} - - # These are packages that have been tracked as installed - installed_pkgs = self.installed_packages - # There are the packages actually present on the filesystem - present_packages = installer.manager.list_packages() - - # Rename directories for packages that have changed names - for package_name in renamed_packages: - package_dir = os.path.join(sublime.packages_path(), package_name) - metadata_path = os.path.join(package_dir, 'package-metadata.json') - if not os.path.exists(metadata_path): - continue - - new_package_name = renamed_packages[package_name] - new_package_dir = os.path.join(sublime.packages_path(), - new_package_name) - - changing_case = package_name.lower() == new_package_name.lower() - case_insensitive_fs = sublime.platform() in ['windows', 'osx'] - - # Since Windows and OSX use case-insensitive filesystems, we have to - # scan through the list of installed packages if the rename of the - # package is just changing the case of it. If we don't find the old - # name for it, we continue the loop since os.path.exists() will return - # true due to the case-insensitive nature of the filesystems. - if case_insensitive_fs and changing_case: - has_old = False - for present_package_name in present_packages: - if present_package_name == package_name: - has_old = True - break - if not has_old: - continue - - if not os.path.exists(new_package_dir) or (case_insensitive_fs and changing_case): - - # Windows will not allow you to rename to the same name with - # a different case, so we work around that with a temporary name - if os.name == 'nt' and changing_case: - temp_package_name = '__' + new_package_name - temp_package_dir = os.path.join(sublime.packages_path(), - temp_package_name) - os.rename(package_dir, temp_package_dir) - package_dir = temp_package_dir - - os.rename(package_dir, new_package_dir) - installed_pkgs.append(new_package_name) - - print '%s: Renamed %s to %s' % (__name__, package_name, - new_package_name) - - else: - installer.manager.remove_package(package_name) - print ('%s: Removed %s since package with new name (%s) ' + - 'already exists') % (__name__, package_name, - new_package_name) - - try: - installed_pkgs.remove(package_name) - except (ValueError): - pass - - sublime.set_timeout(lambda: self.save_packages(installed_pkgs), 10) - - def save_packages(self, installed_packages): - """ - Saves the list of installed packages (after having been appropriately - renamed) - - :param installed_packages: - The new list of installed packages - """ - - installed_packages = list(set(installed_packages)) - installed_packages = sorted(installed_packages, - key=lambda s: s.lower()) - - if installed_packages != self.installed_packages: - self.settings.set('installed_packages', installed_packages) - sublime.save_settings(self.settings_file) - - -class PackageInstaller(): - """ - Provides helper functionality related to installing packages - """ - - def __init__(self): - self.manager = PackageManager() - - def make_package_list(self, ignore_actions=[], override_action=None, - ignore_packages=[]): - """ - Creates a list of packages and what operation would be performed for - each. Allows filtering by the applicable action or package name. - Returns the information in a format suitable for displaying in the - quick panel. - - :param ignore_actions: - A list of actions to ignore packages by. Valid actions include: - `install`, `upgrade`, `downgrade`, `reinstall`, `overwrite`, - `pull` and `none`. `pull` andd `none` are for Git and Hg - repositories. `pull` is present when incoming changes are detected, - where as `none` is selected if no commits are available. `overwrite` - is for packages that do not include version information via the - `package-metadata.json` file. - - :param override_action: - A string action name to override the displayed action for all listed - packages. - - :param ignore_packages: - A list of packages names that should not be returned in the list - - :return: - A list of lists, each containing three strings: - 0 - package name - 1 - package description - 2 - action; [extra info;] package url - """ - - packages = self.manager.list_available_packages() - installed_packages = self.manager.list_packages() - - package_list = [] - for package in sorted(packages.iterkeys(), key=lambda s: s.lower()): - if ignore_packages and package in ignore_packages: - continue - package_entry = [package] - info = packages[package] - download = info['downloads'][0] - - if package in installed_packages: - installed = True - metadata = self.manager.get_metadata(package) - if metadata.get('version'): - installed_version = metadata['version'] - else: - installed_version = None - else: - installed = False - - installed_version_name = 'v' + installed_version if \ - installed and installed_version else 'unknown version' - new_version = 'v' + download['version'] - - vcs = None - package_dir = self.manager.get_package_dir(package) - settings = self.manager.settings - - if override_action: - action = override_action - extra = '' - - else: - if os.path.exists(os.path.join(sublime.packages_path(), - package, '.git')): - if settings.get('ignore_vcs_packages'): - continue - vcs = 'git' - incoming = GitUpgrader(settings.get('git_binary'), - settings.get('git_update_command'), package_dir, - settings.get('cache_length'), settings.get('debug') - ).incoming() - elif os.path.exists(os.path.join(sublime.packages_path(), - package, '.hg')): - if settings.get('ignore_vcs_packages'): - continue - vcs = 'hg' - incoming = HgUpgrader(settings.get('hg_binary'), - settings.get('hg_update_command'), package_dir, - settings.get('cache_length'), settings.get('debug') - ).incoming() - - if installed: - if not installed_version: - if vcs: - if incoming: - action = 'pull' - extra = ' with ' + vcs - else: - action = 'none' - extra = '' - else: - action = 'overwrite' - extra = ' %s with %s' % (installed_version_name, - new_version) - else: - res = self.manager.compare_versions( - installed_version, download['version']) - if res < 0: - action = 'upgrade' - extra = ' to %s from %s' % (new_version, - installed_version_name) - elif res > 0: - action = 'downgrade' - extra = ' to %s from %s' % (new_version, - installed_version_name) - else: - action = 'reinstall' - extra = ' %s' % new_version - else: - action = 'install' - extra = ' %s' % new_version - extra += ';' - - if action in ignore_actions: - continue - - description = info.get('description') - if not description: - description = 'No description provided' - package_entry.append(description) - package_entry.append(action + extra + ' ' + - re.sub('^https?://', '', info['url'])) - package_list.append(package_entry) - return package_list - - def disable_package(self, package): - """ - Disables a package before installing or upgrading to prevent errors - where Sublime Text tries to read files that no longer exist, or read a - half-written file. - - :param package: The string package name - """ - - # Don't disable Package Control so it does not get stuck disabled - if package == 'Package Control': - return False - - settings = sublime.load_settings(preferences_filename()) - ignored = settings.get('ignored_packages') - if not ignored: - ignored = [] - if not package in ignored: - ignored.append(package) - settings.set('ignored_packages', ignored) - sublime.save_settings(preferences_filename()) - return True - return False - - def reenable_package(self, package): - """ - Re-enables a package after it has been installed or upgraded - - :param package: The string package name - """ - - settings = sublime.load_settings(preferences_filename()) - ignored = settings.get('ignored_packages') - if not ignored: - return - if package in ignored: - settings.set('ignored_packages', - list(set(ignored) - set([package]))) - sublime.save_settings(preferences_filename()) - - def on_done(self, picked): - """ - Quick panel user selection handler - disables a package, installs or - upgrades it, then re-enables the package - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - name = self.package_list[picked][0] - - if self.disable_package(name): - on_complete = lambda: self.reenable_package(name) - else: - on_complete = None - - thread = PackageInstallerThread(self.manager, name, on_complete) - thread.start() - ThreadProgress(thread, 'Installing package %s' % name, - 'Package %s successfully %s' % (name, self.completion_type)) - - -class PackageInstallerThread(threading.Thread): - """ - A thread to run package install/upgrade operations in so that the main - Sublime Text thread does not get blocked and freeze the UI - """ - - def __init__(self, manager, package, on_complete): - """ - :param manager: - An instance of :class:`PackageManager` - - :param package: - The string package name to install/upgrade - - :param on_complete: - A callback to run after installing/upgrading the package - """ - - self.package = package - self.manager = manager - self.on_complete = on_complete - threading.Thread.__init__(self) - - def run(self): - try: - self.result = self.manager.install_package(self.package) - finally: - if self.on_complete: - sublime.set_timeout(self.on_complete, 1) - - -class InstallPackageCommand(sublime_plugin.WindowCommand): - """ - A command that presents the list of available packages and allows the - user to pick one to install. - """ - - def run(self): - thread = InstallPackageThread(self.window) - thread.start() - ThreadProgress(thread, 'Loading repositories', '') - - -class InstallPackageThread(threading.Thread, PackageInstaller): - """ - A thread to run the action of retrieving available packages in. Uses the - default PackageInstaller.on_done quick panel handler. - """ - - def __init__(self, window): - """ - :param window: - An instance of :class:`sublime.Window` that represents the Sublime - Text window to show the available package list in. - """ - - self.window = window - self.completion_type = 'installed' - threading.Thread.__init__(self) - PackageInstaller.__init__(self) - - def run(self): - self.package_list = self.make_package_list(['upgrade', 'downgrade', - 'reinstall', 'pull', 'none']) - - def show_quick_panel(): - if not self.package_list: - sublime.error_message(('%s: There are no packages ' + - 'available for installation.') % __name__) - return - self.window.show_quick_panel(self.package_list, self.on_done) - sublime.set_timeout(show_quick_panel, 10) - - -class DiscoverPackagesCommand(sublime_plugin.WindowCommand): - """ - A command that opens the community package list webpage - """ - - def run(self): - self.window.run_command('open_url', - {'url': 'http://wbond.net/sublime_packages/community'}) - - -class UpgradePackageCommand(sublime_plugin.WindowCommand): - """ - A command that presents the list of installed packages that can be upgraded. - """ - - def run(self): - package_renamer = PackageRenamer() - package_renamer.load_settings() - - thread = UpgradePackageThread(self.window, package_renamer) - thread.start() - ThreadProgress(thread, 'Loading repositories', '') - - -class UpgradePackageThread(threading.Thread, PackageInstaller): - """ - A thread to run the action of retrieving upgradable packages in. - """ - - def __init__(self, window, package_renamer): - """ - :param window: - An instance of :class:`sublime.Window` that represents the Sublime - Text window to show the list of upgradable packages in. - - :param package_renamer: - An instance of :class:`PackageRenamer` - """ - self.window = window - self.package_renamer = package_renamer - self.completion_type = 'upgraded' - threading.Thread.__init__(self) - PackageInstaller.__init__(self) - - def run(self): - self.package_renamer.rename_packages(self) - - self.package_list = self.make_package_list(['install', 'reinstall', - 'none']) - - def show_quick_panel(): - if not self.package_list: - sublime.error_message(('%s: There are no packages ' + - 'ready for upgrade.') % __name__) - return - self.window.show_quick_panel(self.package_list, self.on_done) - sublime.set_timeout(show_quick_panel, 10) - - def on_done(self, picked): - """ - Quick panel user selection handler - disables a package, upgrades it, - then re-enables the package - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - name = self.package_list[picked][0] - - if self.disable_package(name): - on_complete = lambda: self.reenable_package(name) - else: - on_complete = None - - thread = PackageInstallerThread(self.manager, name, on_complete) - thread.start() - ThreadProgress(thread, 'Upgrading package %s' % name, - 'Package %s successfully %s' % (name, self.completion_type)) - - -class UpgradeAllPackagesCommand(sublime_plugin.WindowCommand): - """ - A command to automatically upgrade all installed packages that are - upgradable. - """ - - def run(self): - package_renamer = PackageRenamer() - package_renamer.load_settings() - - thread = UpgradeAllPackagesThread(self.window, package_renamer) - thread.start() - ThreadProgress(thread, 'Loading repositories', '') - - -class UpgradeAllPackagesThread(threading.Thread, PackageInstaller): - """ - A thread to run the action of retrieving upgradable packages in. - """ - - def __init__(self, window, package_renamer): - self.window = window - self.package_renamer = package_renamer - self.completion_type = 'upgraded' - threading.Thread.__init__(self) - PackageInstaller.__init__(self) - - def run(self): - self.package_renamer.rename_packages(self) - package_list = self.make_package_list(['install', 'reinstall', 'none']) - - disabled_packages = {} - - def do_upgrades(): - # Pause so packages can be disabled - time.sleep(0.5) - - # We use a function to generate the on-complete lambda because if - # we don't, the lambda will bind to info at the current scope, and - # thus use the last value of info from the loop - def make_on_complete(name): - return lambda: self.reenable_package(name) - - for info in package_list: - if disabled_packages.get(info[0]): - on_complete = make_on_complete(info[0]) - else: - on_complete = None - thread = PackageInstallerThread(self.manager, info[0], on_complete) - thread.start() - ThreadProgress(thread, 'Upgrading package %s' % info[0], - 'Package %s successfully %s' % (info[0], self.completion_type)) - - # Disabling a package means changing settings, which can only be done - # in the main thread. We then create a new background thread so that - # the upgrade process does not block the UI. - def disable_packages(): - for info in package_list: - disabled_packages[info[0]] = self.disable_package(info[0]) - threading.Thread(target=do_upgrades).start() - - sublime.set_timeout(disable_packages, 1) - - -class ExistingPackagesCommand(): - """ - Allows listing installed packages and their current version - """ - - def __init__(self): - self.manager = PackageManager() - - def make_package_list(self, action=''): - """ - Returns a list of installed packages suitable for displaying in the - quick panel. - - :param action: - An action to display at the beginning of the third element of the - list returned for each package - - :return: - A list of lists, each containing three strings: - 0 - package name - 1 - package description - 2 - [action] installed version; package url - """ - - packages = self.manager.list_packages() - - if action: - action += ' ' - - package_list = [] - for package in sorted(packages, key=lambda s: s.lower()): - package_entry = [package] - metadata = self.manager.get_metadata(package) - package_dir = os.path.join(sublime.packages_path(), package) - - description = metadata.get('description') - if not description: - description = 'No description provided' - package_entry.append(description) - - version = metadata.get('version') - if not version and os.path.exists(os.path.join(package_dir, - '.git')): - installed_version = 'git repository' - elif not version and os.path.exists(os.path.join(package_dir, - '.hg')): - installed_version = 'hg repository' - else: - installed_version = 'v' + version if version else \ - 'unknown version' - - url = metadata.get('url') - if url: - url = '; ' + re.sub('^https?://', '', url) - else: - url = '' - - package_entry.append(action + installed_version + url) - package_list.append(package_entry) - - return package_list - - -class ListPackagesCommand(sublime_plugin.WindowCommand): - """ - A command that shows a list of all installed packages in the quick panel - """ - - def run(self): - ListPackagesThread(self.window).start() - - -class ListPackagesThread(threading.Thread, ExistingPackagesCommand): - """ - A thread to prevent the listing of existing packages from freezing the UI - """ - - def __init__(self, window): - """ - :param window: - An instance of :class:`sublime.Window` that represents the Sublime - Text window to show the list of installed packages in. - """ - - self.window = window - threading.Thread.__init__(self) - ExistingPackagesCommand.__init__(self) - - def run(self): - self.package_list = self.make_package_list() - - def show_quick_panel(): - if not self.package_list: - sublime.error_message(('%s: There are no packages ' + - 'to list.') % __name__) - return - self.window.show_quick_panel(self.package_list, self.on_done) - sublime.set_timeout(show_quick_panel, 10) - - def on_done(self, picked): - """ - Quick panel user selection handler - opens the homepage for any - selected package in the user's browser - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - package_name = self.package_list[picked][0] - - def open_dir(): - self.window.run_command('open_dir', - {"dir": os.path.join(sublime.packages_path(), package_name)}) - sublime.set_timeout(open_dir, 10) - - -class RemovePackageCommand(sublime_plugin.WindowCommand, - ExistingPackagesCommand): - """ - A command that presents a list of installed packages, allowing the user to - select one to remove - """ - - def __init__(self, window): - """ - :param window: - An instance of :class:`sublime.Window` that represents the Sublime - Text window to show the list of installed packages in. - """ - - self.window = window - ExistingPackagesCommand.__init__(self) - - def run(self): - self.package_list = self.make_package_list('remove') - if not self.package_list: - sublime.error_message(('%s: There are no packages ' + - 'that can be removed.') % __name__) - return - self.window.show_quick_panel(self.package_list, self.on_done) - - def on_done(self, picked): - """ - Quick panel user selection handler - deletes the selected package - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - package = self.package_list[picked][0] - - # Don't disable Package Control so it does not get stuck disabled - if package != 'Package Control': - settings = sublime.load_settings(preferences_filename()) - ignored = settings.get('ignored_packages') - if not ignored: - ignored = [] - if not package in ignored: - ignored.append(package) - settings.set('ignored_packages', ignored) - sublime.save_settings(preferences_filename()) - - ignored.remove(package) - thread = RemovePackageThread(self.manager, package, - ignored) - thread.start() - ThreadProgress(thread, 'Removing package %s' % package, - 'Package %s successfully removed' % package) - - -class RemovePackageThread(threading.Thread): - """ - A thread to run the remove package operation in so that the Sublime Text - UI does not become frozen - """ - - def __init__(self, manager, package, ignored): - self.manager = manager - self.package = package - self.ignored = ignored - threading.Thread.__init__(self) - - def run(self): - self.result = self.manager.remove_package(self.package) - - def unignore_package(): - settings = sublime.load_settings(preferences_filename()) - settings.set('ignored_packages', self.ignored) - sublime.save_settings(preferences_filename()) - sublime.set_timeout(unignore_package, 10) - - -class AddRepositoryChannelCommand(sublime_plugin.WindowCommand): - """ - A command to add a new channel (list of repositories) to the user's machine - """ - - def run(self): - self.window.show_input_panel('Channel JSON URL', '', - self.on_done, self.on_change, self.on_cancel) - - def on_done(self, input): - """ - Input panel handler - adds the provided URL as a channel - - :param input: - A string of the URL to the new channel - """ - - settings = sublime.load_settings('%s.sublime-settings' % __name__) - repository_channels = settings.get('repository_channels', []) - if not repository_channels: - repository_channels = [] - repository_channels.append(input) - settings.set('repository_channels', repository_channels) - sublime.save_settings('%s.sublime-settings' % __name__) - sublime.status_message(('Channel %s successfully ' + - 'added') % input) - - def on_change(self, input): - pass - - def on_cancel(self): - pass - - -class AddRepositoryCommand(sublime_plugin.WindowCommand): - """ - A command to add a new repository to the user's machine - """ - - def run(self): - self.window.show_input_panel('GitHub or BitBucket Web URL, or Custom' + - ' JSON Repository URL', '', self.on_done, - self.on_change, self.on_cancel) - - def on_done(self, input): - """ - Input panel handler - adds the provided URL as a repository - - :param input: - A string of the URL to the new repository - """ - - settings = sublime.load_settings('%s.sublime-settings' % __name__) - repositories = settings.get('repositories', []) - if not repositories: - repositories = [] - repositories.append(input) - settings.set('repositories', repositories) - sublime.save_settings('%s.sublime-settings' % __name__) - sublime.status_message('Repository %s successfully added' % input) - - def on_change(self, input): - pass - - def on_cancel(self): - pass - - -class DisablePackageCommand(sublime_plugin.WindowCommand): - """ - A command that adds a package to Sublime Text's ignored packages list - """ - - def run(self): - manager = PackageManager() - packages = manager.list_all_packages() - self.settings = sublime.load_settings(preferences_filename()) - ignored = self.settings.get('ignored_packages') - if not ignored: - ignored = [] - self.package_list = list(set(packages) - set(ignored)) - self.package_list.sort() - if not self.package_list: - sublime.error_message(('%s: There are no enabled packages' + - 'to disable.') % __name__) - return - self.window.show_quick_panel(self.package_list, self.on_done) - - def on_done(self, picked): - """ - Quick panel user selection handler - disables the selected package - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - package = self.package_list[picked] - ignored = self.settings.get('ignored_packages') - if not ignored: - ignored = [] - ignored.append(package) - self.settings.set('ignored_packages', ignored) - sublime.save_settings(preferences_filename()) - sublime.status_message(('Package %s successfully added to list of ' + - 'disabled packages - restarting Sublime Text may be required') % - package) - - -class EnablePackageCommand(sublime_plugin.WindowCommand): - """ - A command that removes a package from Sublime Text's ignored packages list - """ - - def run(self): - self.settings = sublime.load_settings(preferences_filename()) - self.disabled_packages = self.settings.get('ignored_packages') - self.disabled_packages.sort() - if not self.disabled_packages: - sublime.error_message(('%s: There are no disabled packages ' + - 'to enable.') % __name__) - return - self.window.show_quick_panel(self.disabled_packages, self.on_done) - - def on_done(self, picked): - """ - Quick panel user selection handler - enables the selected package - - :param picked: - An integer of the 0-based package name index from the presented - list. -1 means the user cancelled. - """ - - if picked == -1: - return - package = self.disabled_packages[picked] - ignored = self.settings.get('ignored_packages') - self.settings.set('ignored_packages', - list(set(ignored) - set([package]))) - sublime.save_settings(preferences_filename()) - sublime.status_message(('Package %s successfully removed from list ' + - 'of disabled packages - restarting Sublime Text may be required') % - package) - - -class AutomaticUpgrader(threading.Thread): - """ - Automatically checks for updated packages and installs them. controlled - by the `auto_upgrade`, `auto_upgrade_ignore`, `auto_upgrade_frequency` and - `auto_upgrade_last_run` settings. - """ - - def __init__(self, found_packages): - """ - :param found_packages: - A list of package names for the packages that were found to be - installed on the machine. - """ - - self.installer = PackageInstaller() - self.manager = self.installer.manager - - self.load_settings() - - self.package_renamer = PackageRenamer() - self.package_renamer.load_settings() - - self.auto_upgrade = self.settings.get('auto_upgrade') - self.auto_upgrade_ignore = self.settings.get('auto_upgrade_ignore') - - self.next_run = int(time.time()) - self.last_run = None - last_run_file = os.path.join(sublime.packages_path(), 'User', - 'Package Control.last-run') - - if os.path.isfile(last_run_file): - with open(last_run_file) as fobj: - try: - self.last_run = int(fobj.read()) - except ValueError: - pass - - frequency = self.settings.get('auto_upgrade_frequency') - if frequency: - if self.last_run: - self.next_run = int(self.last_run) + (frequency * 60 * 60) - else: - self.next_run = time.time() - - # Detect if a package is missing that should be installed - self.missing_packages = list(set(self.installed_packages) - - set(found_packages)) - - if self.auto_upgrade and self.next_run <= time.time(): - with open(last_run_file, 'w') as fobj: - fobj.write(str(int(time.time()))) - - threading.Thread.__init__(self) - - def load_settings(self): - """ - Loads the list of installed packages from the - Package Control.sublime-settings file - """ - - self.settings_file = '%s.sublime-settings' % __name__ - self.settings = sublime.load_settings(self.settings_file) - self.installed_packages = self.settings.get('installed_packages', []) - self.should_install_missing = self.settings.get('install_missing') - if not isinstance(self.installed_packages, list): - self.installed_packages = [] - - def run(self): - self.install_missing() - - if self.next_run > time.time(): - self.print_skip() - return - - self.upgrade_packages() - - def install_missing(self): - """ - Installs all packages that were listed in the list of - `installed_packages` from Package Control.sublime-settings but were not - found on the filesystem and passed as `found_packages`. - """ - - if not self.missing_packages or not self.should_install_missing: - return - - print '%s: Installing %s missing packages' % \ - (__name__, len(self.missing_packages)) - for package in self.missing_packages: - if self.installer.manager.install_package(package): - print '%s: Installed missing package %s' % \ - (__name__, package) - - def print_skip(self): - """ - Prints a notice in the console if the automatic upgrade is skipped - due to already having been run in the last `auto_upgrade_frequency` - hours. - """ - - last_run = datetime.datetime.fromtimestamp(self.last_run) - next_run = datetime.datetime.fromtimestamp(self.next_run) - date_format = '%Y-%m-%d %H:%M:%S' - print ('%s: Skipping automatic upgrade, last run at ' + - '%s, next run at %s or after') % (__name__, - last_run.strftime(date_format), next_run.strftime(date_format)) - - def upgrade_packages(self): - """ - Upgrades all packages that are not currently upgraded to the lastest - version. Also renames any installed packages to their new names. - """ - - if not self.auto_upgrade: - return - - self.package_renamer.rename_packages(self.installer) - - packages = self.installer.make_package_list(['install', - 'reinstall', 'downgrade', 'overwrite', 'none'], - ignore_packages=self.auto_upgrade_ignore) - - # If Package Control is being upgraded, just do that and restart - for package in packages: - if package[0] != __name__: - continue - - def reset_last_run(): - settings = sublime.load_settings(self.settings_file) - settings.set('auto_upgrade_last_run', None) - sublime.save_settings(self.settings_file) - sublime.set_timeout(reset_last_run, 1) - packages = [package] - break - - if not packages: - print '%s: No updated packages' % __name__ - return - - print '%s: Installing %s upgrades' % (__name__, len(packages)) - for package in packages: - self.installer.manager.install_package(package[0]) - version = re.sub('^.*?(v[\d\.]+).*?$', '\\1', package[2]) - if version == package[2] and version.find('pull with') != -1: - vcs = re.sub('^pull with (\w+).*?$', '\\1', version) - version = 'latest %s commit' % vcs - print '%s: Upgraded %s to %s' % (__name__, package[0], version) - - -class PackageCleanup(threading.Thread, PackageRenamer): - """ - Cleans up folders for packages that were removed, but that still have files - in use. - """ - - def __init__(self): - self.manager = PackageManager() - self.load_settings() - threading.Thread.__init__(self) - - def run(self): - found_pkgs = [] - installed_pkgs = self.installed_packages - for package_name in os.listdir(sublime.packages_path()): - package_dir = os.path.join(sublime.packages_path(), package_name) - metadata_path = os.path.join(package_dir, 'package-metadata.json') - - # Cleanup packages that could not be removed due to in-use files - cleanup_file = os.path.join(package_dir, 'package-control.cleanup') - if os.path.exists(cleanup_file): - try: - shutil.rmtree(package_dir) - print '%s: Removed old directory for package %s' % \ - (__name__, package_name) - except (OSError) as (e): - if not os.path.exists(cleanup_file): - open(cleanup_file, 'w').close() - print ('%s: Unable to remove old directory for package ' + - '%s - deferring until next start: %s') % (__name__, - package_name, unicode_from_os(e)) - - # Finish reinstalling packages that could not be upgraded due to - # in-use files - reinstall = os.path.join(package_dir, 'package-control.reinstall') - if os.path.exists(reinstall): - if not clear_directory(package_dir, [metadata_path]): - if not os.path.exists(reinstall): - open(reinstall, 'w').close() - # Assigning this here prevents the callback from referencing the value - # of the "package_name" variable when it is executed - restart_message = ('%s: An error occurred while trying to ' + - 'finish the upgrade of %s. You will most likely need to ' + - 'restart your computer to complete the upgrade.') % ( - __name__, package_name) - def show_still_locked(): - sublime.error_message(restart_message) - sublime.set_timeout(show_still_locked, 10) - else: - self.manager.install_package(package_name) - - # This adds previously installed packages from old versions of PC - if os.path.exists(metadata_path) and \ - package_name not in self.installed_packages: - installed_pkgs.append(package_name) - params = { - 'package': package_name, - 'operation': 'install', - 'version': \ - self.manager.get_metadata(package_name).get('version') - } - self.manager.record_usage(params) - - found_pkgs.append(package_name) - - sublime.set_timeout(lambda: self.finish(installed_pkgs, found_pkgs), 10) - - def finish(self, installed_pkgs, found_pkgs): - """ - A callback that can be run the main UI thread to perform saving of the - Package Control.sublime-settings file. Also fires off the - :class:`AutomaticUpgrader`. - - :param installed_pkgs: - A list of the string package names of all "installed" packages, - even ones that do not appear to be in the filesystem. - - :param found_pkgs: - A list of the string package names of all packages that are - currently installed on the filesystem. - """ - - self.save_packages(installed_pkgs) - AutomaticUpgrader(found_pkgs).start() - - -# Start shortly after Sublime starts so package renames don't cause errors -# with keybindings, settings, etc disappearing in the middle of parsing -sublime.set_timeout(lambda: PackageCleanup().start(), 2000) +import sublime +import sys +import os +import locale + + +st_version = 2 + +# Warn about out-dated versions of ST3 +if sublime.version() == '': + st_version = 3 + print('Package Control: Please upgrade to Sublime Text 3 build 3012 or newer') + +elif int(sublime.version()) > 3000: + st_version = 3 + + +if st_version == 3: + installed_dir, _ = __name__.split('.') +elif st_version == 2: + installed_dir = os.path.basename(os.getcwd()) + + +# Ensure the user has installed Package Control properly +if installed_dir != 'Package Control': + message = (u"Package Control\n\nThis package appears to be installed " + + u"incorrectly.\n\nIt should be installed as \"Package Control\", " + + u"but seems to be installed as \"%s\".\n\n" % installed_dir) + # If installed unpacked + if os.path.exists(os.path.join(sublime.packages_path(), installed_dir)): + message += (u"Please use the Preferences > Browse Packages... menu " + + u"entry to open the \"Packages/\" folder and rename" + + u"\"%s/\" to \"Package Control/\" " % installed_dir) + # If installed as a .sublime-package file + else: + message += (u"Please use the Preferences > Browse Packages... menu " + + u"entry to open the \"Packages/\" folder, then browse up a " + + u"folder and into the \"Installed Packages/\" folder.\n\n" + + u"Inside of \"Installed Packages/\", rename " + + u"\"%s.sublime-package\" to " % installed_dir + + u"\"Package Control.sublime-package\" ") + message += u"and restart Sublime Text." + sublime.error_message(message) + +# Normal execution will finish setting up the package +else: + reloader_name = 'package_control.reloader' + + # ST3 loads each package as a module, so it needs an extra prefix + if st_version == 3: + reloader_name = 'Package Control.' + reloader_name + from imp import reload + + # Make sure all dependencies are reloaded on upgrade + if reloader_name in sys.modules: + reload(sys.modules[reloader_name]) + + + try: + # Python 3 + from .package_control import reloader + + from .package_control.commands import * + from .package_control.package_cleanup import PackageCleanup + + except (ValueError): + # Python 2 + from package_control import reloader + from package_control import sys_path + + from package_control.commands import * + from package_control.package_cleanup import PackageCleanup + + + def plugin_loaded(): + # Make sure the user's locale can handle non-ASCII. A whole bunch of + # work was done to try and make Package Control work even if the locale + # was poorly set, by manually encoding all file paths, but it ended up + # being a fool's errand since the package loading code built into + # Sublime Text is not written to work that way, and although packages + # could be installed, they could not be loaded properly. + try: + os.path.exists(os.path.join(sublime.packages_path(), u"fran\u00e7ais")) + except (UnicodeEncodeError) as e: + message = (u"Package Control\n\nYour system's locale is set to a " + + u"value that can not handle non-ASCII characters. Package " + + u"Control can not properly work unless this is fixed.\n\n" + + u"On Linux, please reference your distribution's docs for " + + u"information on properly setting the LANG environmental " + + u"variable. As a temporary work-around, you can launch " + + u"Sublime Text from the terminal with:\n\n" + + u"LANG=en_US.UTF-8 sublime_text") + sublime.error_message(message) + return + + # Start shortly after Sublime starts so package renames don't cause errors + # with keybindings, settings, etc disappearing in the middle of parsing + sublime.set_timeout(lambda: PackageCleanup().start(), 2000) + + if st_version == 2: + plugin_loaded() diff --git a/sublime/Packages/Package Control/Package Control.sublime-settings b/sublime/Packages/Package Control/Package Control.sublime-settings index 7008abe..03e1594 100644 --- a/sublime/Packages/Package Control/Package Control.sublime-settings +++ b/sublime/Packages/Package Control/Package Control.sublime-settings @@ -1,138 +1,166 @@ -{ - // A list of URLs that each contain a JSON file with a list of repositories. - // The repositories from these channels are placed in order after the - // repositories from the "repositories" setting - "repository_channels": [ - "https://sublime.wbond.net/repositories.json" - ], - - // A list of URLs that contain a packages JSON file. These repositories - // are placed in order before repositories from the "repository_channels" - // setting - "repositories": [], - - // A list of CA certs needed for domains. The default channel provides a - // list of domains and an identifier (the md5 hash) for the CA cert(s) - // necessary for each. - // - // If a custom cert is required for a proxy or for an alternate channel - // or repository domain name, it should be added in one of the two forms: - // - // "*": ["my_identifier", "https://example.com/url/of/cert_file"] - // "*": ["my_identifier_2", "/absolute/filesystem/path/to/cert_file"] - // - // In both cases the literal "*" means the cert will be checked to ensure - // it is present for accessing any URL. This is necessary for proxy - // connections, but also useful if you want to provide you own - // ca-bundle.crt file. - // - // The "my_identifier" and "my_identifier_2" can be any unique string - // that Package Control can use as a filename, and ensures that it has - // merged the cert file with the ca-bundle.crt file in the certs/ directory - // since that is what is passed to the downloaders. - "certs": { - "api.bitbucket.org": ["d867a7b2aecc46f9c31afc4f2f50de05", ""], - "api.github.com": ["1c5282418e2cb4989cd6beddcdbab0b5", ""], - "bitbucket.org": ["897abe0b41fd2f64e9e2e351cbc36d76", ""], - "nodeload.github.com": ["1c5282418e2cb4989cd6beddcdbab0b5", ""], - "raw.github.com": ["1c5282418e2cb4989cd6beddcdbab0b5", ""], - "sublime.wbond.net": ["7f4f8622b4fd001c7f648e09aae7edaa", ""] - }, - - // If debugging information for HTTP/HTTPS connections should be printed - // to the Sublime Text console - "debug": false, - - // This helps solve naming issues where a repository it not named the - // same as the package should be. This is primarily only useful for - // GitHub and BitBucket repositories. This mapping will override the - // mapping that is retrieved from the repository channels. - "package_name_map": {}, - - // If package install, upgrade and removal info should be submitted to - // the channel for aggregated statistics - "submit_usage": true, - - // The URL to post install, upgrade and removal notices to - "submit_url": "https://sublime.wbond.net/submit", - - // If packages should be automatically upgraded when ST2 starts - "auto_upgrade": true, - - // If missing packages should be automatically installed when ST2 starts - "install_missing": true, - - // The minimum frequency in hours in which to check for automatic upgrades, - // setting this to 0 will always check for automatic upgrades - "auto_upgrade_frequency": 1, - - // Packages to not auto upgrade - "auto_upgrade_ignore": [], - - // Timeout for downloading channels, repositories and packages - "timeout": 30, - - // The number of seconds to cache repository and package info for - "cache_length": 300, - - // An HTTP proxy server to use for requests - "http_proxy": "", - // An HTTPS proxy server to use for requests - this will inherit from - // http_proxy if it is set to "" or null and http_proxy has a value. You - // can set this to false to prevent inheriting from http_proxy. - "https_proxy": "", - - // Username and password for both http_proxy and https_proxy - "proxy_username": "", - "proxy_password": "", - - // User agent for HTTP requests - "user_agent": "Sublime Package Control", - - // Setting this to true will cause Package Control to ignore all git - // and hg repositories - this may help if trying to list packages to install - // hangs - "ignore_vcs_packages": false, - - // Custom paths to VCS binaries for when they can't be automatically - // found on the system and a package includes a VCS metadata directory - "git_binary": "", - "git_update_command": ["pull", "origin", "master", "--ff", "--commit"], - - "hg_binary": "", - - // Be sure to keep the remote name as the last argument - "hg_update_command": ["pull", "--update", "default"], - - // Directories to ignore when creating a package - "dirs_to_ignore": [ - ".hg", ".git", ".svn", "_darcs", "CVS" - ], - - // Files to ignore when creating a package - "files_to_ignore": [ - ".hgignore", ".gitignore", ".bzrignore", "*.pyc", "*.sublime-project", - "*.sublime-workspace", "*.tmTheme.cache" - ], - - // Files to include, even if they match a pattern in files_to_ignore - "files_to_include": [], - - // Files to ignore when creating a binary package. By default binary - // packages ship with .pyc files instead of .py files. If an __init__.py - // file exists, it will always be included, even if it matches one of - // these patterns. - "files_to_ignore_binary": [ - ".hgignore", ".gitignore", ".bzrignore", "*.py", "*.sublime-project", - "*.sublime-workspace", "*.tmTheme.cache" - ], - - // Files to include for a binary package, even if they match a pattern i - // files_to_ignore_binary - "files_to_include_binary": [ - "__init__.py" - ], - - // When a package is created, copy it to this folder - defaults to Desktop - "package_destination": "" -} \ No newline at end of file +{ + // A list of URLs that each contain a JSON file with a list of repositories. + // The repositories from these channels are placed in order after the + // repositories from the "repositories" setting + "channels": [ + "https://sublime.wbond.net/channel.json" + ], + + // A list of URLs that contain a packages JSON file. These repositories + // are placed in order before repositories from the "channels" + // setting + "repositories": [], + + // A list of CA certs needed for domains. The default channel provides a + // list of domains and an identifier (the md5 hash) for the CA cert(s) + // necessary for each. Not used on Windows since the system CA cert list + // is automatically used via WinINet. + // + // If a custom cert is required for a proxy or for an alternate channel + // or repository domain name, it should be added in one of the two forms: + // + // "*": ["my_identifier", "https://example.com/url/of/cert_file"] + // "*": ["my_identifier_2", "/absolute/filesystem/path/to/cert_file"] + // + // In both cases the literal "*" means the cert will be checked to ensure + // it is present for accessing any URL. This is necessary for proxy + // connections, but also useful if you want to provide you own + // Pckage Control.ca-bundle file. + // + // The "my_identifier" and "my_identifier_2" can be any unique string + // that Package Control can use as a filename, and ensures that it has + // merged the cert file with the ca-bundle.crt file in the certs/ directory + // since that is what is passed to the downloaders. + "certs": { + "api.bitbucket.org": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "api.github.com": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "bitbucket.org": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "codeload.github.com": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "downloads.sourceforge.net": ["221e907bdfff70d71cea42361ae209d5", "https://sublime.wbond.net/certs/221e907bdfff70d71cea42361ae209d5"], + "github.com": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "nodeload.github.com": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "raw.github.com": ["7d0986b90061d60c8c02aa3b1cf23850", "https://sublime.wbond.net/certs/7d0986b90061d60c8c02aa3b1cf23850"], + "sublime.wbond.net": ["221e907bdfff70d71cea42361ae209d5", "https://sublime.wbond.net/certs/221e907bdfff70d71cea42361ae209d5"] + }, + + // Install pre-release versions of packages. If this is false, versions + // under 1.0.0 will still be installed. Only packages using the SemVer + // -prerelease suffixes will be ignored. + "install_prereleases": false, + + // If debugging information for HTTP/HTTPS connections should be printed + // to the Sublime Text console + "debug": false, + + // This helps solve naming issues where a repository it not named the + // same as the package should be. This is primarily only useful for + // GitHub and BitBucket repositories. This mapping will override the + // mapping that is retrieved from the repository channels. + "package_name_map": {}, + + // If package install, upgrade and removal info should be submitted to + // the channel for aggregated statistics + "submit_usage": true, + + // The URL to post install, upgrade and removal notices to + "submit_url": "https://sublime.wbond.net/submit", + + // If packages should be automatically upgraded when ST2 starts + "auto_upgrade": true, + + // If missing packages should be automatically installed when ST2 starts + "install_missing": true, + + // The minimum frequency in hours in which to check for automatic upgrades, + // setting this to 0 will always check for automatic upgrades + "auto_upgrade_frequency": 1, + + // Packages to not auto upgrade + "auto_upgrade_ignore": [], + + // Timeout for downloading channels, repositories and packages. Doesn't + // have an effect on Windows due to a bug in WinINet. + "timeout": 30, + + // The number of seconds to cache repository and package info for + "cache_length": 300, + + // An HTTP proxy server to use for requests. Not used on Windows since the + // system proxy configuration is utilized via WinINet. + "http_proxy": "", + // An HTTPS proxy server to use for requests - this will inherit from + // http_proxy if it is set to "" or null and http_proxy has a value. You + // can set this to false to prevent inheriting from http_proxy. Not used on + // Windows since the system proxy configuration is utilized via WinINet. + "https_proxy": "", + + // Username and password for both http_proxy and https_proxy. May be used + // with WinINet to set credentials for system-level proxy config. + "proxy_username": "", + "proxy_password": "", + + // If HTTP responses should be cached to disk + "http_cache": true, + + // Number of seconds to cache HTTP responses for, defaults to one week + "http_cache_length": 604800, + + // User agent for HTTP requests. If "%s" is present, will be replaced + // with the current version. + "user_agent": "Sublime Package Control v%s", + + // Setting this to true will cause Package Control to ignore all git + // and hg repositories - this may help if trying to list packages to install + // hangs + "ignore_vcs_packages": false, + + // Custom paths to VCS binaries for when they can't be automatically + // found on the system and a package includes a VCS metadata directory + "git_binary": "", + + // This should NOT contain the name of the remote or branch - that will + // be automatically determined. + "git_update_command": ["pull", "--ff", "--commit"], + + "hg_binary": "", + + // For HG repositories, be sure to use "default" as the remote URL. + // This is the default behavior when cloning an HG repo. + "hg_update_command": ["pull", "--update"], + + // Full path to the openssl binary, if not found on your machine. This is + // only used when running the Grab CA Certs command. + "openssl_binary": "", + + // Directories to ignore when creating a package + "dirs_to_ignore": [ + ".hg", ".git", ".svn", "_darcs", "CVS" + ], + + // Files to ignore when creating a package + "files_to_ignore": [ + ".hgignore", ".gitignore", ".bzrignore", "*.pyc", "*.sublime-project", + "*.sublime-workspace", "*.tmTheme.cache" + ], + + // Files to include, even if they match a pattern in files_to_ignore + "files_to_include": [], + + // Files to ignore when creating a binary package. By default binary + // packages ship with .pyc files instead of .py files. If an __init__.py + // file exists, it will always be included, even if it matches one of + // these patterns. + "files_to_ignore_binary": [ + ".hgignore", ".gitignore", ".bzrignore", "*.py", "*.sublime-project", + "*.sublime-workspace", "*.tmTheme.cache" + ], + + // Files to include for a binary package, even if they match a pattern i + // files_to_ignore_binary + "files_to_include_binary": [ + "__init__.py" + ], + + // When a package is created, copy it to this folder - defaults to Desktop + "package_destination": "" +} diff --git a/sublime/Packages/Package Control/example-channel.json b/sublime/Packages/Package Control/example-channel.json new file mode 100644 index 0000000..75aeac3 --- /dev/null +++ b/sublime/Packages/Package Control/example-channel.json @@ -0,0 +1,64 @@ +{ + "schema_version": "2.0", + + // All repositories must be an HTTP or HTTPS URL. HTTPS is vastly superior + // since verification of the source server is performed on SSL certificates. + "repositories": [ + "http://sublime.wbond.net/packages.json", + "https://github.com/buymeasoda/soda-theme", + "https://github.com/SublimeText" + ], + + // The "packages_cache" is completely optional, but allows the + // channel to cache and deliver package data from multiple + // repositories in a single HTTP request, allowing for significantly + // improved performance. + "packages_cache": { + + // The first level keys are the repository URLs + "http://sublime.wbond.net/packages.json": [ + + // Each repository has an array of packages with their fully + // expanded info. This means that the "details" key must be expanded + // into the various keys it provides. + { + "name": "Alignment", + "description": "Multi-line and multiple selection alignment plugin", + "author": "wbond", + "homepage": "http://wbond.net/sublime_packages/alignment", + "releases": [ + { + "version": "2.0.0", + "url": "https://sublime.wbond.net/Alignment.sublime-package", + "date": "2011-09-18 20:12:41" + } + ] + } + ] + }, + + // Package Control ships with the SSL Certificate Authority (CA) cert for the + // SSL certificate that secures and identifies sublime.wbond.net. After this + // initial connection is made, the channel server provides a list of CA certs + // for the various URLs that Package Control need to connect to. This way the + // default channel (https://sublime.wbond.net/channel.json) can provide + // real-time updates to CA certs in the case that a CA is compromised. The + // CA certs are extracted from openssl, and the server runs on an LTS version + // of Ubuntu, which automatically applies security patches from the official + // Ubuntu repositories. This architecture helps to ensure that the packages + // being downloaded are from the source listed and that users are very + // unlikely to be the subject of the man-in-the-middle attack. + "certs": { + + // All certs have the domain they apply to as the key + "sublime.wbond.net": [ + // The value is an array of two elements, the first being an md5 + // hash of the contents of the certificate. This helps in detecting + // CA cert changes. The second element is the URL where the cert + // can be downloaded, if it is not already installed on the user’s + // copy of Sublime Text. + "7f4f8622b4fd001c7f648e09aae7edaa", + "https://sublime.wbond.net/certs/7f4f8622b4fd001c7f648e09aae7edaa" + ] + } +} \ No newline at end of file diff --git a/sublime/Packages/Package Control/example-repository.json b/sublime/Packages/Package Control/example-repository.json new file mode 100644 index 0000000..39fe43d --- /dev/null +++ b/sublime/Packages/Package Control/example-repository.json @@ -0,0 +1,275 @@ +{ + "schema_version": "2.0", + + // Packages can be specified with a simple URL to a GitHub or BitBucket + // repository, but details can be overridden for every field. It is + // also possible not utilize GitHub or BitBucket at all, but just to + // host your packages on any server with an SSL certificate. + "packages": [ + + // This is what most packages should aim to model. + // + // The majority of the information about a package ("name", + // "description", "author") are all pulled from the GitHub (or + // BitBucket) repository info. + // + // If the word "sublime" exists in the repository name, the name + // can be overridden by the "name" key. + // + // A release is created from the the tag that is the highest semantic + // versioning version number in the list of tags. + { + "name": "Alignment", + "details": "https://github.com/wbond/sublime_alignment", + "releases": [ + { + "details": "https://github.com/wbond/sublime_alignment/tags" + } + ] + }, + + // Here is an equivalent package being pulled from BitBucket + { + "name": "Alignment", + "details": "https://bitbucket.org/wbond/sublime_alignment", + "releases": [ + { + "details": "https://bitbucket.org/wbond/sublime_alignment#tags" + } + ] + }, + + // Pull most details from GitHub, releases from master branch. + // This form is discouraged because users will upgrade to every single + // commit you make to master. + { + "details": "https://github.com/wbond/sublime_alignment" + }, + + // Pull most details from a BitBucket repository and releases from + // the branch "default" or "master", depending on how your repository + // is configured. + // Similar to the above example, this form is discouraged because users + // will upgrade to every single commit you make to master. + { + "details": "https://bitbucket.org/wbond/sublime_alignment" + }, + + // Use a custom name instead of just the URL slug + { + "name": "Alignment", + "details": "https://github.com/wbond/sublime_alignment" + }, + + // You can also override the homepage and author + { + "name": "Alignment", + "details": "https://github.com/wbond/sublime_alignment", + "homepage": "http://wbond.net/sublime_packages/alignment", + "author": "wbond" + }, + + // It is possible to provide the URL to a readme file. This URL + // should be to the raw source of the file, not rendered HTML. + // GitHub and BitBucket repositories will automatically provide + // these. + // + // The following extensions will be rendered: + // + // .markdown, .mdown, .mkd, .md + // .texttile + // .creole + // .rst + // + // All others are treated as plaintext. + { + "details": "https://github.com/wbond/sublime_alignment", + "readme": "https://raw.github.com/wbond/sublime_alignment/master/readme.creole" + }, + + // If a package has a public bug tracker, the URL should be + // included via the "issues" key. Both GitHub and BitBucket + // repositories will automatically provide this if they have + // issues enabled. + { + "details": "https://github.com/wbond/sublime_alignment", + "issues": "https://github.com/wbond/sublime_alignment/issues" + }, + + // The URL to donate to support the development of a package. + // GitHub and BitBucket repositories will default to: + // + // https://www.gittip.com/{username}/ + // + // Other URLs with special integration include: + // + // https://flattr.com/profile/{username} + // https://www.dwolla.com/hub/{username} + // + // This may also contain a URL to another other donation-type site + // where users may support the author for their development of the + // package. + { + "details": "https://github.com/wbond/sublime_alignment", + "donate": "https://www.gittip.com/wbond/" + }, + + // The URL to purchase a license to the package + { + "details": "https://github.com/wbond/sublime_alignment", + "buy": "https://wbond.net/sublime_packages/alignment/buy" + }, + + // If you rename a package, you can provide the previous name(s) + // so that users with the old package name can be automatically + // upgraded to the new one. + { + "name": "Alignment", + "details": "https://github.com/wbond/sublime_alignment", + "previous_names": ["sublime_alignment"] + }, + + // Packages can be labelled for the purpose of creating a + // folksonomy so users may more easily find relevant packages. + // Labels should be all lower case and should use spaces instead + // of _ or - to separate words. + // + // Some suggested labels are listed below, however, anything can + // be used as a label: + // + // auto-complete + // browser integration + // build system + // code navigation + // code sharing + // color scheme + // deprecated + // diff/merge + // editor emulation + // file creation + // file navigation + // formatting + // ftp + // language syntax + // linting + // minification + // search + // snippets + // terminal/shell/repl + // testing + // text manipulation + // text navigation + // theme + // todo + // vcs + { + "details": "https://github.com/wbond/sublime_alignment", + "labels": ["text manipulation", "formatting"] + }, + + // In addition to the recommendation above of pulling releases + // from tags that are semantic version numbers, releases can also + // comefrom a custom branch. + { + "details": "https://github.com/wbond/sublime_alignment", + "releases": [ + { + "details": "https://github.com/wbond/sublime_alignment/tree/custom_branch" + } + ] + }, + + // An equivalent package being pulled from BitBucket. + { + "details": "https://bitbucket.org/wbond/sublime_alignment", + "releases": [ + { + "details": "https://bitbucket.org/wbond/sublime_alignment/src/custom_branch" + } + ] + }, + + // If your package is only compatible with specific builds of + // Sublime Text, this will cause the package to be hidden from + // users with incompatible versions. + { + "details": "https://github.com/wbond/sublime_alignment", + "releases": [ + { + // Could also be >2999 for ST3. Leaving this out indicates + // the package works with both ST2 and ST3. + "sublime_text": "<3000", + "details": "https://github.com/wbond/sublime_alignment" + } + ] + }, + + // The "platforms" key allows specifying what platform(s) the release + // is valid for. As shown, there can be multiple releases of a package + // at any given time. However, only the latest version for any given + // platform/arch will be shown to the user. + // + // The "platforms" key allows specifying a single platform, or a list + // of platforms. Valid platform indentifiers include: + // + // "*" + // "windows", "windows-x64", "windows-x32" + // "osx", "osx-x64" + // "linux", "linux-x32", "linux-x64" + { + "details": "https://github.com/wbond/sublime_alignment", + "releases": [ + { + // Defaults to "*", or all platforms. + "platforms": ["osx", "linux"], + "details": "https://github.com/wbond/sublime_alignment/tree/posix" + }, + { + "platforms": "windows", + "details": "https://github.com/wbond/sublime_alignment/tree/win32" + } + ] + }, + + // If you don't use a "details" key for a "releases" entry, you need to + // specify the "version", "url" and "date" manually. + { + "details": "https://github.com/wbond/sublime_alignment", + "releases": [ + { + // The version number needs to be a semantic version number per + // http://semver.org 2.x.x + "version": "2.0.0", + + // The URL needs to be a zip file containing the package. It is permissible + // for the zip file to contain a single root folder with any name. All + // file will be extracted out of this single root folder. This allows + // zip files from GitHub and BitBucket to be used a sources. + "url": "https://codeload.github.com/wbond/sublime_alignment/zip/v2.0.0", + + // The date MUST be in the form "YYYY-MM-DD HH:MM:SS" and SHOULD be UTC + "date": "2011-09-18 20:12:41" + } + ] + } + ], + + // If you need/want to split your repository up into multiple smaller files + // for the sake of organization, the "includes" key allows you to enter + // URL paths that will be combined together and dynamically inserted + // into the "packages" key. These URLs these can be relative or absolute. + "includes": [ + + // Here is an example of how relative paths work for URLs. If this file + // was loaded from: + // "https://sublime.wbond.net/example-repository.json" + // then the following files would be loaded from: + // "https://sublime.wbond.net/repository/0-9.json" + // "https://sublime.wbond.net/repository/a.json" + "./repository/0-9.json", + "./repository/a.json", + + // An example of an absolute URL + "https://sublime.wbond.net/repository/b.json" + ] +} diff --git a/sublime/Packages/Package Control/messages.json b/sublime/Packages/Package Control/messages.json index 00340ab..8c26db4 100644 --- a/sublime/Packages/Package Control/messages.json +++ b/sublime/Packages/Package Control/messages.json @@ -1,4 +1,5 @@ { "1.3.0": "messages/1.3.0.txt", - "1.6.0": "messages/1.6.0.txt" -} \ No newline at end of file + "1.6.0": "messages/1.6.0.txt", + "2.0.0": "messages/2.0.0.txt" +} diff --git a/sublime/Packages/Package Control/messages/2.0.0.txt b/sublime/Packages/Package Control/messages/2.0.0.txt new file mode 100644 index 0000000..59524ea --- /dev/null +++ b/sublime/Packages/Package Control/messages/2.0.0.txt @@ -0,0 +1,64 @@ +Package Control 2.0.0 Changelog: + + +Today I'd like to announce two big milestones: + + - Package Control 2.0 for ST2 and ST3 + - A new Package Control website at https://sublime.wbond.net + +The full announcement about the PC 2.0 release is available on the news page at +https://sublime.wbond.net/news. + +If you are running the "testing" version of Package Control (1.6.9 - 1.6.11), +you will likely need to restart Sublime Text before Package Control will work +properly. + + +Giving Back + +Part of the new Package Control website is in-depth information about each +package. The new package pages include a link where you can give a tip to the +developer/maintainer of your favorite packages. + +The donate links go to https://www.gittip.com, which is building an excellent, +and open platform for users to say "thank you" to open source developers. It +is possible to give a small amount each week, such as $0.25, however these small +amounts multiplied by the large size of the community can be a big thank you! + +One of the less glamorous jobs involved with making Package Control happen is +reviewing and giving package developers feedback before adding their packages +to the default channel. The follow contributors deserve a big thank you: + +FichteFoll - https://www.gittip.com/FichteFoll/ +joneshf - https://www.gittip.com/on/github/joneshf/ +sentience - https://www.gittip.com/on/github/sentience/ + +Finally, I'm looking to raise some money to obtain a Mac Mini for the purposes +of supporting ST3 on OS X and a license for a Windows 8 VM. If you are inclined +to donate to those, or want to just buy me a beer, check out: + +https://sublime.wbond.net/say_thanks + + +Notable Features + + - A new Windows downloader that uses WinINet and should provide much better + proxy support + + - Using operating system-supplied SSL CA certs on all platforms, with a + deprecated fallback to certificates served through the channel + + - Proxy server fixes for OS X + + - A completely revamped channel and repository system with support for more + information about packages including labels; readme, issues, donate and buy + URLs; tag-based releases; platform targetting without a custom packages.json + file; and Sublime Text version targetting + + - Support for installing via .sublime-package files in ST3, which allows users + to easily override specific files from the package. Package developers who + need a loose folder of files may include a .no-sublime-package file in their + repo. + + - In the coming days the new Package Control website will be released as open + source on GitHub diff --git a/sublime/Packages/Package Control/package-metadata.json b/sublime/Packages/Package Control/package-metadata.json index adff7f1..c8258ac 100644 --- a/sublime/Packages/Package Control/package-metadata.json +++ b/sublime/Packages/Package Control/package-metadata.json @@ -1 +1,5 @@ -{"url": "http://wbond.net/sublime_packages/package_control", "version": "1.6.3", "description": "A full-featured package manager"} \ No newline at end of file +{ + "version": "2.0.0", + "url": "https://sublime.wbond.net", + "description": "A full-featured package manager" +} diff --git a/sublime/Packages/Package Control/package_control/__init__.py b/sublime/Packages/Package Control/package_control/__init__.py new file mode 100644 index 0000000..b541c64 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/__init__.py @@ -0,0 +1,2 @@ +__version__ = "2.0.0" +__version_info__ = (2, 0, 0) diff --git a/sublime/Packages/Package Control/package_control/automatic_upgrader.py b/sublime/Packages/Package Control/package_control/automatic_upgrader.py new file mode 100644 index 0000000..bbebd8a --- /dev/null +++ b/sublime/Packages/Package Control/package_control/automatic_upgrader.py @@ -0,0 +1,215 @@ +import threading +import re +import os +import datetime +import time + +import sublime + +from .console_write import console_write +from .package_installer import PackageInstaller +from .package_renamer import PackageRenamer +from .open_compat import open_compat, read_compat + + +class AutomaticUpgrader(threading.Thread): + """ + Automatically checks for updated packages and installs them. controlled + by the `auto_upgrade`, `auto_upgrade_ignore`, and `auto_upgrade_frequency` + settings. + """ + + def __init__(self, found_packages): + """ + :param found_packages: + A list of package names for the packages that were found to be + installed on the machine. + """ + + self.installer = PackageInstaller() + self.manager = self.installer.manager + + self.load_settings() + + self.package_renamer = PackageRenamer() + self.package_renamer.load_settings() + + self.auto_upgrade = self.settings.get('auto_upgrade') + self.auto_upgrade_ignore = self.settings.get('auto_upgrade_ignore') + + self.load_last_run() + self.determine_next_run() + + # Detect if a package is missing that should be installed + self.missing_packages = list(set(self.installed_packages) - + set(found_packages)) + + if self.auto_upgrade and self.next_run <= time.time(): + self.save_last_run(time.time()) + + threading.Thread.__init__(self) + + def load_last_run(self): + """ + Loads the last run time from disk into memory + """ + + self.last_run = None + + self.last_run_file = os.path.join(sublime.packages_path(), 'User', + 'Package Control.last-run') + + if os.path.isfile(self.last_run_file): + with open_compat(self.last_run_file) as fobj: + try: + self.last_run = int(read_compat(fobj)) + except ValueError: + pass + + def determine_next_run(self): + """ + Figure out when the next run should happen + """ + + self.next_run = int(time.time()) + + frequency = self.settings.get('auto_upgrade_frequency') + if frequency: + if self.last_run: + self.next_run = int(self.last_run) + (frequency * 60 * 60) + else: + self.next_run = time.time() + + def save_last_run(self, last_run): + """ + Saves a record of when the last run was + + :param last_run: + The unix timestamp of when to record the last run as + """ + + with open_compat(self.last_run_file, 'w') as fobj: + fobj.write(str(int(last_run))) + + + def load_settings(self): + """ + Loads the list of installed packages from the + Package Control.sublime-settings file + """ + + self.settings_file = 'Package Control.sublime-settings' + self.settings = sublime.load_settings(self.settings_file) + self.installed_packages = self.settings.get('installed_packages', []) + self.should_install_missing = self.settings.get('install_missing') + if not isinstance(self.installed_packages, list): + self.installed_packages = [] + + def run(self): + self.install_missing() + + if self.next_run > time.time(): + self.print_skip() + return + + self.upgrade_packages() + + def install_missing(self): + """ + Installs all packages that were listed in the list of + `installed_packages` from Package Control.sublime-settings but were not + found on the filesystem and passed as `found_packages`. + """ + + if not self.missing_packages or not self.should_install_missing: + return + + console_write(u'Installing %s missing packages' % len(self.missing_packages), True) + for package in self.missing_packages: + if self.installer.manager.install_package(package): + console_write(u'Installed missing package %s' % package, True) + + def print_skip(self): + """ + Prints a notice in the console if the automatic upgrade is skipped + due to already having been run in the last `auto_upgrade_frequency` + hours. + """ + + last_run = datetime.datetime.fromtimestamp(self.last_run) + next_run = datetime.datetime.fromtimestamp(self.next_run) + date_format = '%Y-%m-%d %H:%M:%S' + message_string = u'Skipping automatic upgrade, last run at %s, next run at %s or after' % ( + last_run.strftime(date_format), next_run.strftime(date_format)) + console_write(message_string, True) + + def upgrade_packages(self): + """ + Upgrades all packages that are not currently upgraded to the lastest + version. Also renames any installed packages to their new names. + """ + + if not self.auto_upgrade: + return + + self.package_renamer.rename_packages(self.installer) + + package_list = self.installer.make_package_list(['install', + 'reinstall', 'downgrade', 'overwrite', 'none'], + ignore_packages=self.auto_upgrade_ignore) + + # If Package Control is being upgraded, just do that and restart + for package in package_list: + if package[0] != 'Package Control': + continue + + def reset_last_run(): + # Re-save the last run time so it runs again after PC has + # been updated + self.save_last_run(self.last_run) + sublime.set_timeout(reset_last_run, 1) + package_list = [package] + break + + if not package_list: + console_write(u'No updated packages', True) + return + + console_write(u'Installing %s upgrades' % len(package_list), True) + + disabled_packages = [] + + def do_upgrades(): + # Wait so that the ignored packages can be "unloaded" + time.sleep(0.5) + + # We use a function to generate the on-complete lambda because if + # we don't, the lambda will bind to info at the current scope, and + # thus use the last value of info from the loop + def make_on_complete(name): + return lambda: self.installer.reenable_package(name) + + for info in package_list: + if info[0] in disabled_packages: + on_complete = make_on_complete(info[0]) + else: + on_complete = None + + self.installer.manager.install_package(info[0]) + + version = re.sub('^.*?(v[\d\.]+).*?$', '\\1', info[2]) + if version == info[2] and version.find('pull with') != -1: + vcs = re.sub('^pull with (\w+).*?$', '\\1', version) + version = 'latest %s commit' % vcs + message_string = u'Upgraded %s to %s' % (info[0], version) + console_write(message_string, True) + if on_complete: + sublime.set_timeout(on_complete, 1) + + # Disabling a package means changing settings, which can only be done + # in the main thread. We then create a new background thread so that + # the upgrade process does not block the UI. + def disable_packages(): + disabled_packages.extend(self.installer.disable_packages([info[0] for info in package_list])) + threading.Thread(target=do_upgrades).start() + sublime.set_timeout(disable_packages, 1) diff --git a/sublime/Packages/Package Control/package_control/ca_certs.py b/sublime/Packages/Package Control/package_control/ca_certs.py new file mode 100644 index 0000000..d29d2e0 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/ca_certs.py @@ -0,0 +1,378 @@ +import hashlib +import os +import re +import time +import sys + +from .cmd import Cli +from .console_write import console_write +from .open_compat import open_compat, read_compat + + +# Have somewhere to store the CA bundle, even when not running in Sublime Text +try: + import sublime + ca_bundle_dir = None +except (ImportError): + ca_bundle_dir = os.path.join(os.path.expanduser('~'), '.package_control') + if not os.path.exists(ca_bundle_dir): + os.mkdir(ca_bundle_dir) + + +def find_root_ca_cert(settings, domain): + runner = OpensslCli(settings.get('openssl_binary'), settings.get('debug')) + binary = runner.retrieve_binary() + + args = [binary, 's_client', '-showcerts', '-connect', domain + ':443'] + result = runner.execute(args, os.path.dirname(binary)) + + certs = [] + temp = [] + + in_block = False + for line in result.splitlines(): + if line.find('BEGIN CERTIFICATE') != -1: + in_block = True + if in_block: + temp.append(line) + if line.find('END CERTIFICATE') != -1: + in_block = False + certs.append(u"\n".join(temp)) + temp = [] + + # Remove the cert for the domain itself, just leaving the + # chain cert and the CA cert + certs.pop(0) + + # Look for the "parent" root CA cert + subject = openssl_get_cert_subject(settings, certs[-1]) + issuer = openssl_get_cert_issuer(settings, certs[-1]) + + cert = get_ca_cert_by_subject(settings, issuer) + cert_hash = hashlib.md5(cert.encode('utf-8')).hexdigest() + + return [cert, cert_hash] + + + +def get_system_ca_bundle_path(settings): + """ + Get the filesystem path to the system CA bundle. On Linux it looks in a + number of predefined places, however on OS X it has to be programatically + exported from the SystemRootCertificates.keychain. Windows does not ship + with a CA bundle, but also we use WinINet on Windows, so we don't need to + worry about CA certs. + + :param settings: + A dict to look in for `debug` and `openssl_binary` keys + + :return: + The full filesystem path to the .ca-bundle file, or False on error + """ + + # If the sublime module is available, we bind this value at run time + # since the sublime.packages_path() is not available at import time + global ca_bundle_dir + + platform = sys.platform + debug = settings.get('debug') + + ca_path = False + + if platform == 'win32': + console_write(u"Unable to get system CA cert path since Windows does not ship with them", True) + return False + + # OS X + if platform == 'darwin': + if not ca_bundle_dir: + ca_bundle_dir = os.path.join(sublime.packages_path(), 'User') + ca_path = os.path.join(ca_bundle_dir, 'Package Control.system-ca-bundle') + + exists = os.path.exists(ca_path) + # The bundle is old if it is a week or more out of date + is_old = exists and os.stat(ca_path).st_mtime < time.time() - 604800 + + if not exists or is_old: + if debug: + console_write(u"Generating new CA bundle from system keychain", True) + _osx_create_ca_bundle(settings, ca_path) + if debug: + console_write(u"Finished generating new CA bundle at %s" % ca_path, True) + elif debug: + console_write(u"Found previously exported CA bundle at %s" % ca_path, True) + + # Linux + else: + # Common CA cert paths + paths = [ + '/usr/lib/ssl/certs/ca-certificates.crt', + '/etc/ssl/certs/ca-certificates.crt', + '/etc/pki/tls/certs/ca-bundle.crt', + '/etc/ssl/ca-bundle.pem' + ] + for path in paths: + if os.path.exists(path): + ca_path = path + break + + if debug and ca_path: + console_write(u"Found system CA bundle at %s" % ca_path, True) + + return ca_path + + +def get_ca_cert_by_subject(settings, subject): + bundle_path = get_system_ca_bundle_path(settings) + + with open_compat(bundle_path, 'r') as f: + contents = read_compat(f) + + temp = [] + + in_block = False + for line in contents.splitlines(): + if line.find('BEGIN CERTIFICATE') != -1: + in_block = True + + if in_block: + temp.append(line) + + if line.find('END CERTIFICATE') != -1: + in_block = False + cert = u"\n".join(temp) + temp = [] + + if openssl_get_cert_subject(settings, cert) == subject: + return cert + + return False + + +def openssl_get_cert_issuer(settings, cert): + """ + Uses the openssl command line client to extract the issuer of an x509 + certificate. + + :param settings: + A dict to look in for `debug` and `openssl_binary` keys + + :param cert: + A string containing the PEM-encoded x509 certificate to extract the + issuer from + + :return: + The cert issuer + """ + + runner = OpensslCli(settings.get('openssl_binary'), settings.get('debug')) + binary = runner.retrieve_binary() + args = [binary, 'x509', '-noout', '-issuer'] + output = runner.execute(args, os.path.dirname(binary), cert) + return re.sub('^issuer=\s*', '', output) + + +def openssl_get_cert_name(settings, cert): + """ + Uses the openssl command line client to extract the name of an x509 + certificate. If the commonName is set, that is used, otherwise the first + organizationalUnitName is used. This mirrors what OS X uses for storing + trust preferences. + + :param settings: + A dict to look in for `debug` and `openssl_binary` keys + + :param cert: + A string containing the PEM-encoded x509 certificate to extract the + name from + + :return: + The cert subject name, which is the commonName (if available), or the + first organizationalUnitName + """ + + runner = OpensslCli(settings.get('openssl_binary'), settings.get('debug')) + + binary = runner.retrieve_binary() + + args = [binary, 'x509', '-noout', '-subject', '-nameopt', + 'sep_multiline,lname,utf8'] + result = runner.execute(args, os.path.dirname(binary), cert) + + # First look for the common name + cn = None + # If there is no common name for the cert, the trust prefs use the first + # orginizational unit name + first_ou = None + + for line in result.splitlines(): + match = re.match('^\s+commonName=(.*)$', line) + if match: + cn = match.group(1) + break + match = re.match('^\s+organizationalUnitName=(.*)$', line) + if first_ou is None and match: + first_ou = match.group(1) + continue + + # This is the name of the cert that would be used in the trust prefs + return cn or first_ou + + +def openssl_get_cert_subject(settings, cert): + """ + Uses the openssl command line client to extract the subject of an x509 + certificate. + + :param settings: + A dict to look in for `debug` and `openssl_binary` keys + + :param cert: + A string containing the PEM-encoded x509 certificate to extract the + subject from + + :return: + The cert subject + """ + + runner = OpensslCli(settings.get('openssl_binary'), settings.get('debug')) + binary = runner.retrieve_binary() + args = [binary, 'x509', '-noout', '-subject'] + output = runner.execute(args, os.path.dirname(binary), cert) + return re.sub('^subject=\s*', '', output) + + +def _osx_create_ca_bundle(settings, destination): + """ + Uses the OS X `security` command line tool to export the system's list of + CA certs from /System/Library/Keychains/SystemRootCertificates.keychain. + Checks the cert names against the trust preferences, ensuring that + distrusted certs are not exported. + + :param settings: + A dict to look in for `debug` and `openssl_binary` keys + + :param destination: + The full filesystem path to the destination .ca-bundle file + """ + + distrusted_certs = _osx_get_distrusted_certs(settings) + + # Export the root certs + args = ['/usr/bin/security', 'export', '-k', + '/System/Library/Keychains/SystemRootCertificates.keychain', '-t', + 'certs', '-p'] + result = Cli(None, settings.get('debug')).execute(args, '/usr/bin') + + certs = [] + temp = [] + + in_block = False + for line in result.splitlines(): + if line.find('BEGIN CERTIFICATE') != -1: + in_block = True + + if in_block: + temp.append(line) + + if line.find('END CERTIFICATE') != -1: + in_block = False + cert = u"\n".join(temp) + temp = [] + + if distrusted_certs: + # If it is a distrusted cert, we move on to the next + cert_name = openssl_get_cert_name(settings, cert) + if cert_name in distrusted_certs: + if settings.get('debug'): + console_write(u'Skipping root certficate %s because it is distrusted' % cert_name, True) + continue + + certs.append(cert) + + with open_compat(destination, 'w') as f: + f.write(u"\n".join(certs)) + + +def _osx_get_distrusted_certs(settings): + """ + Uses the OS X `security` binary to get a list of admin trust settings, + which is what is set when a user changes the trust setting on a root + certificate. By looking at the SSL policy, we can properly exclude + distrusted certs from out export. + + Tested on OS X 10.6 and 10.8 + + :param settings: + A dict to look in for `debug` key + + :return: + A list of CA cert names, where the name is the commonName (if + available), or the first organizationalUnitName + """ + + args = ['/usr/bin/security', 'dump-trust-settings', '-d'] + result = Cli(None, settings.get('debug')).execute(args, '/usr/bin') + + distrusted_certs = [] + cert_name = None + ssl_policy = False + for line in result.splitlines(): + if line == '': + continue + + # Reset for each cert + match = re.match('Cert\s+\d+:\s+(.*)$', line) + if match: + cert_name = match.group(1) + continue + + # Reset for each trust setting + if re.match('^\s+Trust\s+Setting\s+\d+:', line): + ssl_policy = False + continue + + # We are only interested in SSL policies + if re.match('^\s+Policy\s+OID\s+:\s+SSL', line): + ssl_policy = True + continue + + distrusted = re.match('^\s+Result\s+Type\s+:\s+kSecTrustSettingsResultDeny', line) + if ssl_policy and distrusted and cert_name not in distrusted_certs: + if settings.get('debug'): + console_write(u'Found SSL distrust setting for root certificate %s' % cert_name, True) + distrusted_certs.append(cert_name) + + return distrusted_certs + + +class OpensslCli(Cli): + + cli_name = 'openssl' + + def retrieve_binary(self): + """ + Returns the path to the openssl executable + + :return: The string path to the executable or False on error + """ + + name = 'openssl' + if os.name == 'nt': + name += '.exe' + + binary = self.find_binary(name) + if binary and os.path.isdir(binary): + full_path = os.path.join(binary, name) + if os.path.exists(full_path): + binary = full_path + + if not binary: + show_error((u'Unable to find %s. Please set the openssl_binary ' + + u'setting by accessing the Preferences > Package Settings > ' + + u'Package Control > Settings \u2013 User menu entry. The ' + + u'Settings \u2013 Default entry can be used for reference, ' + + u'but changes to that will be overwritten upon next upgrade.') % name) + return False + + return binary diff --git a/sublime/Packages/Package Control/package_control/cache.py b/sublime/Packages/Package Control/package_control/cache.py new file mode 100644 index 0000000..4b8021f --- /dev/null +++ b/sublime/Packages/Package Control/package_control/cache.py @@ -0,0 +1,168 @@ +import time + + +# A cache of channel and repository info to allow users to install multiple +# packages without having to wait for the metadata to be downloaded more +# than once. The keys are managed locally by the utilizing code. +_channel_repository_cache = {} + + +def clear_cache(): + global _channel_repository_cache + _channel_repository_cache = {} + + +def get_cache(key, default=None): + """ + Gets an in-memory cache value + + :param key: + The string key + + :param default: + The value to return if the key has not been set, or the ttl expired + + :return: + The cached value, or default + """ + + struct = _channel_repository_cache.get(key, {}) + expires = struct.get('expires') + if expires and expires > time.time(): + return struct.get('data') + return default + + +def merge_cache_over_settings(destination, setting, key_prefix): + """ + Take the cached value of `key` and put it into the key `setting` of + the destination.settings dict. Merge the values by overlaying the + cached setting over the existing info. + + :param destination: + An object that has a `.settings` attribute that is a dict + + :param setting: + The dict key to use when pushing the value into the settings dict + + :param key_prefix: + The string to prefix to `setting` to make the cache key + """ + + existing = destination.settings.get(setting, {}) + value = get_cache(key_prefix + '.' + setting, {}) + if value: + existing.update(value) + destination.settings[setting] = existing + + +def merge_cache_under_settings(destination, setting, key_prefix, list_=False): + """ + Take the cached value of `key` and put it into the key `setting` of + the destination.settings dict. Merge the values by overlaying the + existing setting value over the cached info. + + :param destination: + An object that has a `.settings` attribute that is a dict + + :param setting: + The dict key to use when pushing the value into the settings dict + + :param key_prefix: + The string to prefix to `setting` to make the cache key + + :param list_: + If a list should be used instead of a dict + """ + + default = {} if not list_ else [] + existing = destination.settings.get(setting) + value = get_cache(key_prefix + '.' + setting, default) + if value: + if existing: + if list_: + value.extend(existing) + else: + value.update(existing) + destination.settings[setting] = value + + +def set_cache(key, data, ttl=300): + """ + Sets an in-memory cache value + + :param key: + The string key + + :param data: + The data to cache + + :param ttl: + The integer number of second to cache the data for + """ + + _channel_repository_cache[key] = { + 'data': data, + 'expires': time.time() + ttl + } + + +def set_cache_over_settings(destination, setting, key_prefix, value, ttl): + """ + Take the value passed, and merge it over the current `setting`. Once + complete, take the value and set the cache `key` and destination.settings + `setting` to that value, using the `ttl` for set_cache(). + + :param destination: + An object that has a `.settings` attribute that is a dict + + :param setting: + The dict key to use when pushing the value into the settings dict + + :param key_prefix: + The string to prefix to `setting` to make the cache key + + :param value: + The value to set + + :param ttl: + The cache ttl to use + """ + + existing = destination.settings.get(setting, {}) + existing.update(value) + set_cache(key_prefix + '.' + setting, value, ttl) + destination.settings[setting] = value + + +def set_cache_under_settings(destination, setting, key_prefix, value, ttl, list_=False): + """ + Take the value passed, and merge the current `setting` over it. Once + complete, take the value and set the cache `key` and destination.settings + `setting` to that value, using the `ttl` for set_cache(). + + :param destination: + An object that has a `.settings` attribute that is a dict + + :param setting: + The dict key to use when pushing the value into the settings dict + + :param key_prefix: + The string to prefix to `setting` to make the cache key + + :param value: + The value to set + + :param ttl: + The cache ttl to use + """ + + default = {} if not list_ else [] + existing = destination.settings.get(setting, default) + if value: + if list_: + value.extend(existing) + else: + value.update(existing) + set_cache(key_prefix + '.' + setting, value, ttl) + destination.settings[setting] = value diff --git a/sublime/Packages/Package Control/package_control/clear_directory.py b/sublime/Packages/Package Control/package_control/clear_directory.py new file mode 100644 index 0000000..4ddfc07 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/clear_directory.py @@ -0,0 +1,37 @@ +import os + + +def clear_directory(directory, ignore_paths=None): + """ + Tries to delete all files and folders from a directory + + :param directory: + The string directory path + + :param ignore_paths: + An array of paths to ignore while deleting files + + :return: + If all of the files and folders were successfully deleted + """ + + was_exception = False + for root, dirs, files in os.walk(directory, topdown=False): + paths = [os.path.join(root, f) for f in files] + paths.extend([os.path.join(root, d) for d in dirs]) + + for path in paths: + try: + # Don't delete the metadata file, that way we have it + # when the reinstall happens, and the appropriate + # usage info can be sent back to the server + if ignore_paths and path in ignore_paths: + continue + if os.path.isdir(path): + os.rmdir(path) + else: + os.remove(path) + except (OSError, IOError): + was_exception = True + + return not was_exception diff --git a/sublime/Packages/Package Control/package_control/clients/__init__.py b/sublime/Packages/Package Control/package_control/clients/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sublime/Packages/Package Control/package_control/clients/bitbucket_client.py b/sublime/Packages/Package Control/package_control/clients/bitbucket_client.py new file mode 100644 index 0000000..d76b019 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/clients/bitbucket_client.py @@ -0,0 +1,249 @@ +import re + +from ..versions import version_sort, version_filter +from .json_api_client import JSONApiClient + + +# A predefined list of readme filenames to look for +_readme_filenames = [ + 'readme', + 'readme.txt', + 'readme.md', + 'readme.mkd', + 'readme.mdown', + 'readme.markdown', + 'readme.textile', + 'readme.creole', + 'readme.rst' +] + + +class BitBucketClient(JSONApiClient): + + def download_info(self, url): + """ + Retrieve information about downloading a package + + :param url: + The URL of the repository, in one of the forms: + https://bitbucket.org/{user}/{repo} + https://bitbucket.org/{user}/{repo}/src/{branch} + https://bitbucket.org/{user}/{repo}/#tags + If the last option, grabs the info from the newest + tag that is a valid semver version. + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, False if no commit, or a dict with the following keys: + `version` - the version number of the download + `url` - the download URL of a zip file of the package + `date` - the ISO-8601 timestamp string when the version was published + """ + + commit_info = self._commit_info(url) + if not commit_info: + return commit_info + + commit_date = commit_info['timestamp'][0:19] + + return { + 'version': re.sub('[\-: ]', '.', commit_date), + 'url': 'https://bitbucket.org/%s/get/%s.zip' % (commit_info['user_repo'], commit_info['commit']), + 'date': commit_date + } + + def repo_info(self, url): + """ + Retrieve general information about a repository + + :param url: + The URL to the repository, in one of the forms: + https://bitbucket.org/{user}/{repo} + https://bitbucket.org/{user}/{repo}/src/{branch} + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, or a dict with the following keys: + `name` + `description` + `homepage` - URL of the homepage + `author` + `readme` - URL of the readme + `issues` - URL of bug tracker + `donate` - URL of a donate page + """ + + user_repo, branch = self._user_repo_branch(url) + if not user_repo: + return user_repo + + api_url = self._make_api_url(user_repo) + + info = self.fetch_json(api_url) + + issues_url = u'https://bitbucket.org/%s/issues' % user_repo + + return { + 'name': info['name'], + 'description': info['description'] or 'No description provided', + 'homepage': info['website'] or url, + 'author': info['owner'], + 'donate': u'https://www.gittip.com/on/bitbucket/%s/' % info['owner'], + 'readme': self._readme_url(user_repo, branch), + 'issues': issues_url if info['has_issues'] else None + } + + def _commit_info(self, url): + """ + Fetches info about the latest commit to a repository + + :param url: + The URL to the repository, in one of the forms: + https://bitbucket.org/{user}/{repo} + https://bitbucket.org/{user}/{repo}/src/{branch} + https://bitbucket.org/{user}/{repo}/#tags + If the last option, grabs the info from the newest + tag that is a valid semver version. + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, False if no commit, or a dict with the following keys: + `user_repo` - the user/repo name + `timestamp` - the ISO-8601 UTC timestamp string + `commit` - the branch or tag name + """ + + tags_match = re.match('https?://bitbucket.org/([^/]+/[^#/]+)/?#tags$', url) + + if tags_match: + user_repo = tags_match.group(1) + tags_url = self._make_api_url(user_repo, '/tags') + tags_list = self.fetch_json(tags_url) + tags = version_filter(tags_list.keys(), self.settings.get('install_prereleases')) + tags = version_sort(tags, reverse=True) + if not tags: + return False + commit = tags[0] + + else: + user_repo, commit = self._user_repo_branch(url) + if not user_repo: + return user_repo + + changeset_url = self._make_api_url(user_repo, '/changesets/%s' % commit) + commit_info = self.fetch_json(changeset_url) + + return { + 'user_repo': user_repo, + 'timestamp': commit_info['timestamp'], + 'commit': commit + } + + def _main_branch_name(self, user_repo): + """ + Fetch the name of the default branch + + :param user_repo: + The user/repo name to get the main branch for + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + The name of the main branch - `master` or `default` + """ + + main_branch_url = self._make_api_url(user_repo, '/main-branch') + main_branch_info = self.fetch_json(main_branch_url, True) + return main_branch_info['name'] + + def _make_api_url(self, user_repo, suffix=''): + """ + Generate a URL for the BitBucket API + + :param user_repo: + The user/repo of the repository + + :param suffix: + The extra API path info to add to the URL + + :return: + The API URL + """ + + return 'https://api.bitbucket.org/1.0/repositories/%s%s' % (user_repo, suffix) + + def _readme_url(self, user_repo, branch, prefer_cached=False): + """ + Parse the root directory listing for the repo and return the URL + to any file that looks like a readme + + :param user_repo: + The user/repo string + + :param branch: + The branch to fetch the readme from + + :param prefer_cached: + If a cached directory listing should be used instead of a new HTTP request + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + The URL to the readme file, or None + """ + + listing_url = self._make_api_url(user_repo, '/src/%s/' % branch) + root_dir_info = self.fetch_json(listing_url, prefer_cached) + + for entry in root_dir_info['files']: + if entry['path'].lower() in _readme_filenames: + return 'https://bitbucket.org/%s/raw/%s/%s' % (user_repo, + branch, entry['path']) + + return None + + def _user_repo_branch(self, url): + """ + Extract the username/repo and branch name from the URL + + :param url: + The URL to extract the info from, in one of the forms: + https://bitbucket.org/{user}/{repo} + https://bitbucket.org/{user}/{repo}/src/{branch} + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + A tuple of (user/repo, branch name) or (None, None) if not matching + """ + + repo_match = re.match('https?://bitbucket.org/([^/]+/[^/]+)/?$', url) + branch_match = re.match('https?://bitbucket.org/([^/]+/[^/]+)/src/([^/]+)/?$', url) + + if repo_match: + user_repo = repo_match.group(1) + branch = self._main_branch_name(user_repo) + + elif branch_match: + user_repo = branch_match.group(1) + branch = branch_match.group(2) + + else: + return (None, None) + + return (user_repo, branch) diff --git a/sublime/Packages/Package Control/package_control/clients/client_exception.py b/sublime/Packages/Package Control/package_control/clients/client_exception.py new file mode 100644 index 0000000..fb8dd72 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/clients/client_exception.py @@ -0,0 +1,5 @@ +class ClientException(Exception): + """If a client could not fetch information""" + + def __str__(self): + return self.args[0] diff --git a/sublime/Packages/Package Control/package_control/clients/github_client.py b/sublime/Packages/Package Control/package_control/clients/github_client.py new file mode 100644 index 0000000..9c1fd61 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/clients/github_client.py @@ -0,0 +1,284 @@ +import re + +try: + # Python 3 + from urllib.parse import urlencode, quote +except (ImportError): + # Python 2 + from urllib import urlencode, quote + +from ..versions import version_sort, version_filter +from .json_api_client import JSONApiClient +from ..downloaders.downloader_exception import DownloaderException + + +class GitHubClient(JSONApiClient): + + def download_info(self, url): + """ + Retrieve information about downloading a package + + :param url: + The URL of the repository, in one of the forms: + https://github.com/{user}/{repo} + https://github.com/{user}/{repo}/tree/{branch} + https://github.com/{user}/{repo}/tags + If the last option, grabs the info from the newest + tag that is a valid semver version. + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, False if no commit, or a dict with the following keys: + `version` - the version number of the download + `url` - the download URL of a zip file of the package + `date` - the ISO-8601 timestamp string when the version was published + """ + + commit_info = self._commit_info(url) + if not commit_info: + return commit_info + + commit_date = commit_info['timestamp'][0:19].replace('T', ' ') + + return { + 'version': re.sub('[\-: ]', '.', commit_date), + # We specifically use codeload.github.com here because the download + # URLs all redirect there, and some of the downloaders don't follow + # HTTP redirect headers + 'url': 'https://codeload.github.com/%s/zip/%s' % (commit_info['user_repo'], quote(commit_info['commit'])), + 'date': commit_date + } + + def repo_info(self, url): + """ + Retrieve general information about a repository + + :param url: + The URL to the repository, in one of the forms: + https://github.com/{user}/{repo} + https://github.com/{user}/{repo}/tree/{branch} + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, or a dict with the following keys: + `name` + `description` + `homepage` - URL of the homepage + `author` + `readme` - URL of the readme + `issues` - URL of bug tracker + `donate` - URL of a donate page + """ + + user_repo, branch = self._user_repo_branch(url) + if not user_repo: + return user_repo + + api_url = self._make_api_url(user_repo) + + info = self.fetch_json(api_url) + + output = self._extract_repo_info(info) + output['readme'] = None + + readme_info = self._readme_info(user_repo, branch) + if not readme_info: + return output + + output['readme'] = 'https://raw.github.com/%s/%s/%s' % (user_repo, + branch, readme_info['path']) + return output + + def user_info(self, url): + """ + Retrieve general information about all repositories that are + part of a user/organization. + + :param url: + The URL to the user/organization, in the following form: + https://github.com/{user} + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, or am list of dicts with the following keys: + `name` + `description` + `homepage` - URL of the homepage + `author` + `readme` - URL of the readme + `issues` - URL of bug tracker + `donate` - URL of a donate page + """ + + user_match = re.match('https?://github.com/([^/]+)/?$', url) + if user_match == None: + return None + + user = user_match.group(1) + api_url = self._make_api_url(user) + + repos_info = self.fetch_json(api_url) + + output = [] + for info in repos_info: + output.append(self._extract_repo_info(info)) + return output + + def _commit_info(self, url): + """ + Fetches info about the latest commit to a repository + + :param url: + The URL to the repository, in one of the forms: + https://github.com/{user}/{repo} + https://github.com/{user}/{repo}/tree/{branch} + https://github.com/{user}/{repo}/tags + If the last option, grabs the info from the newest + tag that is a valid semver version. + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + None if no match, False is no commit, or a dict with the following keys: + `user_repo` - the user/repo name + `timestamp` - the ISO-8601 UTC timestamp string + `commit` - the branch or tag name + """ + + tags_match = re.match('https?://github.com/([^/]+/[^/]+)/tags/?$', url) + + if tags_match: + user_repo = tags_match.group(1) + tags_url = self._make_api_url(user_repo, '/tags') + tags_list = self.fetch_json(tags_url) + tags = [tag['name'] for tag in tags_list] + tags = version_filter(tags, self.settings.get('install_prereleases')) + tags = version_sort(tags, reverse=True) + if not tags: + return False + commit = tags[0] + + else: + user_repo, commit = self._user_repo_branch(url) + if not user_repo: + return user_repo + + query_string = urlencode({'sha': commit, 'per_page': 1}) + commit_url = self._make_api_url(user_repo, '/commits?%s' % query_string) + commit_info = self.fetch_json(commit_url) + + return { + 'user_repo': user_repo, + 'timestamp': commit_info[0]['commit']['committer']['date'], + 'commit': commit + } + + def _extract_repo_info(self, result): + """ + Extracts information about a repository from the API result + + :param result: + A dict representing the data returned from the GitHub API + + :return: + A dict with the following keys: + `name` + `description` + `homepage` - URL of the homepage + `author` + `issues` - URL of bug tracker + `donate` - URL of a donate page + """ + + issues_url = u'https://github.com/%s/%s/issues' % (result['owner']['login'], result['name']) + + return { + 'name': result['name'], + 'description': result['description'] or 'No description provided', + 'homepage': result['homepage'] or result['html_url'], + 'author': result['owner']['login'], + 'issues': issues_url if result['has_issues'] else None, + 'donate': u'https://www.gittip.com/on/github/%s/' % result['owner']['login'] + } + + def _make_api_url(self, user_repo, suffix=''): + """ + Generate a URL for the BitBucket API + + :param user_repo: + The user/repo of the repository + + :param suffix: + The extra API path info to add to the URL + + :return: + The API URL + """ + + return 'https://api.github.com/repos/%s%s' % (user_repo, suffix) + + def _readme_info(self, user_repo, branch, prefer_cached=False): + """ + Fetches the raw GitHub API information about a readme + + :param user_repo: + The user/repo of the repository + + :param branch: + The branch to pull the readme from + + :param prefer_cached: + If a cached version of the info should be returned instead of making a new HTTP request + + :raises: + DownloaderException: when there is an error downloading + ClientException: when there is an error parsing the response + + :return: + A dict containing all of the info from the GitHub API, or None if no readme exists + """ + + query_string = urlencode({'ref': branch}) + readme_url = self._make_api_url(user_repo, '/readme?%s' % query_string) + try: + return self.fetch_json(readme_url, prefer_cached) + except (DownloaderException) as e: + if str(e).find('HTTP error 404') != -1: + return None + raise + + def _user_repo_branch(self, url): + """ + Extract the username/repo and branch name from the URL + + :param url: + The URL to extract the info from, in one of the forms: + https://github.com/{user}/{repo} + https://github.com/{user}/{repo}/tree/{branch} + + :return: + A tuple of (user/repo, branch name) or (None, None) if no match + """ + + branch = 'master' + branch_match = re.match('https?://github.com/[^/]+/[^/]+/tree/([^/]+)/?$', url) + if branch_match != None: + branch = branch_match.group(1) + + repo_match = re.match('https?://github.com/([^/]+/[^/]+)($|/.*$)', url) + if repo_match == None: + return (None, None) + + user_repo = repo_match.group(1) + return (user_repo, branch) diff --git a/sublime/Packages/Package Control/package_control/clients/json_api_client.py b/sublime/Packages/Package Control/package_control/clients/json_api_client.py new file mode 100644 index 0000000..a847302 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/clients/json_api_client.py @@ -0,0 +1,64 @@ +import json + +try: + # Python 3 + from urllib.parse import urlencode, urlparse +except (ImportError): + # Python 2 + from urllib import urlencode + from urlparse import urlparse + +from .client_exception import ClientException +from ..download_manager import downloader + + +class JSONApiClient(): + def __init__(self, settings): + self.settings = settings + + def fetch(self, url, prefer_cached=False): + """ + Retrieves the contents of a URL + + :param url: + The URL to download the content from + + :param prefer_cached: + If a cached copy of the content is preferred + + :return: The bytes/string + """ + + # If there are extra params for the domain name, add them + extra_params = self.settings.get('query_string_params') + domain_name = urlparse(url).netloc + if extra_params and domain_name in extra_params: + params = urlencode(extra_params[domain_name]) + joiner = '?%s' if url.find('?') == -1 else '&%s' + url += joiner % params + + with downloader(url, self.settings) as manager: + content = manager.fetch(url, 'Error downloading repository.', + prefer_cached) + return content + + def fetch_json(self, url, prefer_cached=False): + """ + Retrieves and parses the JSON from a URL + + :param url: + The URL to download the JSON from + + :param prefer_cached: + If a cached copy of the JSON is preferred + + :return: A dict or list from the JSON + """ + + repository_json = self.fetch(url, prefer_cached) + + try: + return json.loads(repository_json.decode('utf-8')) + except (ValueError): + error_string = u'Error parsing JSON from URL %s.' % url + raise ClientException(error_string) diff --git a/sublime/Packages/Package Control/package_control/clients/readme_client.py b/sublime/Packages/Package Control/package_control/clients/readme_client.py new file mode 100644 index 0000000..47e2a7b --- /dev/null +++ b/sublime/Packages/Package Control/package_control/clients/readme_client.py @@ -0,0 +1,83 @@ +import re +import os +import base64 + +try: + # Python 3 + from urllib.parse import urlencode +except (ImportError): + # Python 2 + from urllib import urlencode + +from .json_api_client import JSONApiClient +from ..downloaders.downloader_exception import DownloaderException + + +# Used to map file extensions to formats +_readme_formats = { + '.md': 'markdown', + '.mkd': 'markdown', + '.mdown': 'markdown', + '.markdown': 'markdown', + '.textile': 'textile', + '.creole': 'creole', + '.rst': 'rst' +} + + +class ReadmeClient(JSONApiClient): + + def readme_info(self, url): + """ + Retrieve the readme and info about it + + :param url: + The URL of the readme file + + :raises: + DownloaderException: if there is an error downloading the readme + ClientException: if there is an error parsing the response + + :return: + A dict with the following keys: + `filename` + `format` - `markdown`, `textile`, `creole`, `rst` or `txt` + `contents` - contents of the readme as str/unicode + """ + + contents = None + + # Try to grab the contents of a GitHub-based readme by grabbing the cached + # content of the readme API call + github_match = re.match('https://raw.github.com/([^/]+/[^/]+)/([^/]+)/readme(\.(md|mkd|mdown|markdown|textile|creole|rst|txt))?$', url, re.I) + if github_match: + user_repo = github_match.group(1) + branch = github_match.group(2) + + query_string = urlencode({'ref': branch}) + readme_json_url = 'https://api.github.com/repos/%s/readme?%s' % (user_repo, query_string) + try: + info = self.fetch_json(readme_json_url, prefer_cached=True) + contents = base64.b64decode(info['content']) + except (ValueError) as e: + pass + + if not contents: + contents = self.fetch(url) + + basename, ext = os.path.splitext(url) + format = 'txt' + ext = ext.lower() + if ext in _readme_formats: + format = _readme_formats[ext] + + try: + contents = contents.decode('utf-8') + except (UnicodeDecodeError) as e: + contents = contents.decode('cp1252', errors='replace') + + return { + 'filename': os.path.basename(url), + 'format': format, + 'contents': contents + } diff --git a/sublime/Packages/Package Control/package_control/cmd.py b/sublime/Packages/Package Control/package_control/cmd.py new file mode 100644 index 0000000..0d5c999 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/cmd.py @@ -0,0 +1,167 @@ +import os +import subprocess +import re + +if os.name == 'nt': + from ctypes import windll, create_unicode_buffer + +from .console_write import console_write +from .unicode import unicode_from_os +from .show_error import show_error + +try: + # Python 2 + str_cls = unicode +except (NameError): + # Python 3 + str_cls = str + + +def create_cmd(args, basename_binary=False): + """ + Takes an array of strings to be passed to subprocess.Popen and creates + a string that can be pasted into a terminal + + :param args: + The array containing the binary name/path and all arguments + + :param basename_binary: + If only the basename of the binary should be disabled instead of the full path + + :return: + The command string + """ + + if basename_binary: + args[0] = os.path.basename(args[0]) + + if os.name == 'nt': + return subprocess.list2cmdline(args) + else: + escaped_args = [] + for arg in args: + if re.search('^[a-zA-Z0-9/_^\\-\\.:=]+$', arg) == None: + arg = u"'" + arg.replace(u"'", u"'\\''") + u"'" + escaped_args.append(arg) + return u' '.join(escaped_args) + + +class Cli(object): + """ + Base class for running command line apps + + :param binary: + The full filesystem path to the executable for the version control + system. May be set to None to allow the code to try and find it. + """ + + cli_name = None + + def __init__(self, binary, debug): + self.binary = binary + self.debug = debug + + def execute(self, args, cwd, input=None): + """ + Creates a subprocess with the executable/args + + :param args: + A list of the executable path and all arguments to it + + :param cwd: + The directory in which to run the executable + + :param input: + The input text to send to the program + + :return: A string of the executable output + """ + + startupinfo = None + if os.name == 'nt': + startupinfo = subprocess.STARTUPINFO() + startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW + + # Make sure the cwd is ascii + try: + cwd.encode('ascii') + except UnicodeEncodeError: + buf = create_unicode_buffer(512) + if windll.kernel32.GetShortPathNameW(cwd, buf, len(buf)): + cwd = buf.value + + if self.debug: + console_write(u"Trying to execute command %s" % create_cmd(args), True) + + try: + proc = subprocess.Popen(args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + startupinfo=startupinfo, cwd=cwd) + + if input and isinstance(input, str_cls): + input = input.encode('utf-8') + output, _ = proc.communicate(input) + output = output.decode('utf-8') + output = output.replace('\r\n', '\n').rstrip(' \n\r') + + return output + + except (OSError) as e: + cmd = create_cmd(args) + error = unicode_from_os(e) + message = u"Error executing: %s\n%s\n\nTry checking your \"%s_binary\" setting?" % (cmd, error, self.cli_name) + show_error(message) + return False + + def find_binary(self, name): + """ + Locates the executable by looking in the PATH and well-known directories + + :param name: + The string filename of the executable + + :return: The filesystem path to the executable, or None if not found + """ + + if self.binary: + if self.debug: + error_string = u"Using \"%s_binary\" from settings \"%s\"" % ( + self.cli_name, self.binary) + console_write(error_string, True) + return self.binary + + # Try the path first + for dir_ in os.environ['PATH'].split(os.pathsep): + path = os.path.join(dir_, name) + if os.path.exists(path): + if self.debug: + console_write(u"Found %s at \"%s\"" % (self.cli_name, path), True) + return path + + # This is left in for backwards compatibility and for windows + # users who may have the binary, albeit in a common dir that may + # not be part of the PATH + if os.name == 'nt': + dirs = ['C:\\Program Files\\Git\\bin', + 'C:\\Program Files (x86)\\Git\\bin', + 'C:\\Program Files\\TortoiseGit\\bin', + 'C:\\Program Files\\Mercurial', + 'C:\\Program Files (x86)\\Mercurial', + 'C:\\Program Files (x86)\\TortoiseHg', + 'C:\\Program Files\\TortoiseHg', + 'C:\\cygwin\\bin'] + else: + # ST seems to launch with a minimal set of environmental variables + # on OS X, so we add some common paths for it + dirs = ['/usr/local/git/bin', '/usr/local/bin'] + + for dir_ in dirs: + path = os.path.join(dir_, name) + if os.path.exists(path): + if self.debug: + console_write(u"Found %s at \"%s\"" % (self.cli_name, path), True) + return path + + if self.debug: + console_write(u"Could not find %s on your machine" % self.cli_name, True) + return None diff --git a/sublime/Packages/Package Control/package_control/commands/__init__.py b/sublime/Packages/Package Control/package_control/commands/__init__.py new file mode 100644 index 0000000..dde03d4 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/__init__.py @@ -0,0 +1,39 @@ +import os + +from .add_channel_command import AddChannelCommand +from .add_repository_command import AddRepositoryCommand +from .create_binary_package_command import CreateBinaryPackageCommand +from .create_package_command import CreatePackageCommand +from .disable_package_command import DisablePackageCommand +from .discover_packages_command import DiscoverPackagesCommand +from .enable_package_command import EnablePackageCommand +from .grab_certs_command import GrabCertsCommand +from .install_package_command import InstallPackageCommand +from .list_packages_command import ListPackagesCommand +from .remove_package_command import RemovePackageCommand +from .upgrade_all_packages_command import UpgradeAllPackagesCommand +from .upgrade_package_command import UpgradePackageCommand +from .package_message_command import PackageMessageCommand + + +__all__ = [ + 'AddChannelCommand', + 'AddRepositoryCommand', + 'CreateBinaryPackageCommand', + 'CreatePackageCommand', + 'DisablePackageCommand', + 'DiscoverPackagesCommand', + 'EnablePackageCommand', + 'InstallPackageCommand', + 'ListPackagesCommand', + 'RemovePackageCommand', + 'UpgradeAllPackagesCommand', + 'UpgradePackageCommand', + 'PackageMessageCommand' +] + +# Windows uses the wininet downloader, so it doesn't use the CA cert bundle +# and thus does not need the ability to grab to CA certs. Additionally, +# there is no openssl.exe on Windows. +if os.name != 'nt': + __all__.append('GrabCertsCommand') diff --git a/sublime/Packages/Package Control/package_control/commands/add_channel_command.py b/sublime/Packages/Package Control/package_control/commands/add_channel_command.py new file mode 100644 index 0000000..5e1b8d1 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/add_channel_command.py @@ -0,0 +1,46 @@ +import re + +import sublime +import sublime_plugin + +from ..show_error import show_error + + +class AddChannelCommand(sublime_plugin.WindowCommand): + """ + A command to add a new channel (list of repositories) to the user's machine + """ + + def run(self): + self.window.show_input_panel('Channel JSON URL', '', + self.on_done, self.on_change, self.on_cancel) + + def on_done(self, input): + """ + Input panel handler - adds the provided URL as a channel + + :param input: + A string of the URL to the new channel + """ + + input = input.strip() + + if re.match('https?://', input, re.I) == None: + show_error(u"Unable to add the channel \"%s\" since it does not appear to be served via HTTP (http:// or https://)." % input) + return + + settings = sublime.load_settings('Package Control.sublime-settings') + channels = settings.get('channels', []) + if not channels: + channels = [] + channels.append(input) + settings.set('channels', channels) + sublime.save_settings('Package Control.sublime-settings') + sublime.status_message(('Channel %s successfully ' + + 'added') % input) + + def on_change(self, input): + pass + + def on_cancel(self): + pass diff --git a/sublime/Packages/Package Control/package_control/commands/add_repository_command.py b/sublime/Packages/Package Control/package_control/commands/add_repository_command.py new file mode 100644 index 0000000..3d04323 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/add_repository_command.py @@ -0,0 +1,46 @@ +import re + +import sublime +import sublime_plugin + +from ..show_error import show_error + + +class AddRepositoryCommand(sublime_plugin.WindowCommand): + """ + A command to add a new repository to the user's machine + """ + + def run(self): + self.window.show_input_panel('GitHub or BitBucket Web URL, or Custom' + + ' JSON Repository URL', '', self.on_done, + self.on_change, self.on_cancel) + + def on_done(self, input): + """ + Input panel handler - adds the provided URL as a repository + + :param input: + A string of the URL to the new repository + """ + + input = input.strip() + + if re.match('https?://', input, re.I) == None: + show_error(u"Unable to add the repository \"%s\" since it does not appear to be served via HTTP (http:// or https://)." % input) + return + + settings = sublime.load_settings('Package Control.sublime-settings') + repositories = settings.get('repositories', []) + if not repositories: + repositories = [] + repositories.append(input) + settings.set('repositories', repositories) + sublime.save_settings('Package Control.sublime-settings') + sublime.status_message('Repository %s successfully added' % input) + + def on_change(self, input): + pass + + def on_cancel(self): + pass diff --git a/sublime/Packages/Package Control/package_control/commands/create_binary_package_command.py b/sublime/Packages/Package Control/package_control/commands/create_binary_package_command.py new file mode 100644 index 0000000..491dd1c --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/create_binary_package_command.py @@ -0,0 +1,35 @@ +import sublime_plugin + +from ..package_creator import PackageCreator + + +class CreateBinaryPackageCommand(sublime_plugin.WindowCommand, PackageCreator): + """ + Command to create a binary .sublime-package file. Binary packages in + general exclude the .py source files and instead include the .pyc files. + Actual included and excluded files are controlled by settings. + """ + + def run(self): + self.show_panel() + + def on_done(self, picked): + """ + Quick panel user selection handler - processes the user package + selection and create the package file + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + package_name = self.packages[picked] + package_destination = self.get_package_destination() + + if self.manager.create_package(package_name, package_destination, + binary_package=True): + self.window.run_command('open_dir', {"dir": + package_destination, "file": package_name + + '.sublime-package'}) diff --git a/sublime/Packages/Package Control/package_control/commands/create_package_command.py b/sublime/Packages/Package Control/package_control/commands/create_package_command.py new file mode 100644 index 0000000..8b0524a --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/create_package_command.py @@ -0,0 +1,32 @@ +import sublime_plugin + +from ..package_creator import PackageCreator + + +class CreatePackageCommand(sublime_plugin.WindowCommand, PackageCreator): + """ + Command to create a regular .sublime-package file + """ + + def run(self): + self.show_panel() + + def on_done(self, picked): + """ + Quick panel user selection handler - processes the user package + selection and create the package file + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + package_name = self.packages[picked] + package_destination = self.get_package_destination() + + if self.manager.create_package(package_name, package_destination): + self.window.run_command('open_dir', {"dir": + package_destination, "file": package_name + + '.sublime-package'}) diff --git a/sublime/Packages/Package Control/package_control/commands/disable_package_command.py b/sublime/Packages/Package Control/package_control/commands/disable_package_command.py new file mode 100644 index 0000000..d5ebd97 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/disable_package_command.py @@ -0,0 +1,48 @@ +import sublime +import sublime_plugin + +from ..show_error import show_error +from ..package_manager import PackageManager +from ..preferences_filename import preferences_filename + + +class DisablePackageCommand(sublime_plugin.WindowCommand): + """ + A command that adds a package to Sublime Text's ignored packages list + """ + + def run(self): + manager = PackageManager() + packages = manager.list_all_packages() + self.settings = sublime.load_settings(preferences_filename()) + ignored = self.settings.get('ignored_packages') + if not ignored: + ignored = [] + self.package_list = list(set(packages) - set(ignored)) + self.package_list.sort() + if not self.package_list: + show_error('There are no enabled packages to disable.') + return + self.window.show_quick_panel(self.package_list, self.on_done) + + def on_done(self, picked): + """ + Quick panel user selection handler - disables the selected package + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + package = self.package_list[picked] + ignored = self.settings.get('ignored_packages') + if not ignored: + ignored = [] + ignored.append(package) + self.settings.set('ignored_packages', ignored) + sublime.save_settings(preferences_filename()) + sublime.status_message(('Package %s successfully added to list of ' + + 'disabled packages - restarting Sublime Text may be required') % + package) diff --git a/sublime/Packages/Package Control/package_control/commands/discover_packages_command.py b/sublime/Packages/Package Control/package_control/commands/discover_packages_command.py new file mode 100644 index 0000000..78d9812 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/discover_packages_command.py @@ -0,0 +1,11 @@ +import sublime_plugin + + +class DiscoverPackagesCommand(sublime_plugin.WindowCommand): + """ + A command that opens the community package list webpage + """ + + def run(self): + self.window.run_command('open_url', + {'url': 'http://wbond.net/sublime_packages/community'}) diff --git a/sublime/Packages/Package Control/package_control/commands/enable_package_command.py b/sublime/Packages/Package Control/package_control/commands/enable_package_command.py new file mode 100644 index 0000000..2e5e6d1 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/enable_package_command.py @@ -0,0 +1,40 @@ +import sublime +import sublime_plugin + +from ..show_error import show_error +from ..preferences_filename import preferences_filename + + +class EnablePackageCommand(sublime_plugin.WindowCommand): + """ + A command that removes a package from Sublime Text's ignored packages list + """ + + def run(self): + self.settings = sublime.load_settings(preferences_filename()) + self.disabled_packages = self.settings.get('ignored_packages') + self.disabled_packages.sort() + if not self.disabled_packages: + show_error('There are no disabled packages to enable.') + return + self.window.show_quick_panel(self.disabled_packages, self.on_done) + + def on_done(self, picked): + """ + Quick panel user selection handler - enables the selected package + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + package = self.disabled_packages[picked] + ignored = self.settings.get('ignored_packages') + self.settings.set('ignored_packages', + list(set(ignored) - set([package]))) + sublime.save_settings(preferences_filename()) + sublime.status_message(('Package %s successfully removed from list ' + + 'of disabled packages - restarting Sublime Text may be required') % + package) diff --git a/sublime/Packages/Package Control/package_control/commands/existing_packages_command.py b/sublime/Packages/Package Control/package_control/commands/existing_packages_command.py new file mode 100644 index 0000000..78615d6 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/existing_packages_command.py @@ -0,0 +1,69 @@ +import os +import re + +import sublime + +from ..package_manager import PackageManager + + +class ExistingPackagesCommand(): + """ + Allows listing installed packages and their current version + """ + + def __init__(self): + self.manager = PackageManager() + + def make_package_list(self, action=''): + """ + Returns a list of installed packages suitable for displaying in the + quick panel. + + :param action: + An action to display at the beginning of the third element of the + list returned for each package + + :return: + A list of lists, each containing three strings: + 0 - package name + 1 - package description + 2 - [action] installed version; package url + """ + + packages = self.manager.list_packages() + + if action: + action += ' ' + + package_list = [] + for package in sorted(packages, key=lambda s: s.lower()): + package_entry = [package] + metadata = self.manager.get_metadata(package) + package_dir = os.path.join(sublime.packages_path(), package) + + description = metadata.get('description') + if not description: + description = 'No description provided' + package_entry.append(description) + + version = metadata.get('version') + if not version and os.path.exists(os.path.join(package_dir, + '.git')): + installed_version = 'git repository' + elif not version and os.path.exists(os.path.join(package_dir, + '.hg')): + installed_version = 'hg repository' + else: + installed_version = 'v' + version if version else \ + 'unknown version' + + url = metadata.get('url') + if url: + url = '; ' + re.sub('^https?://', '', url) + else: + url = '' + + package_entry.append(action + installed_version + url) + package_list.append(package_entry) + + return package_list diff --git a/sublime/Packages/Package Control/package_control/commands/grab_certs_command.py b/sublime/Packages/Package Control/package_control/commands/grab_certs_command.py new file mode 100644 index 0000000..4eb77e0 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/grab_certs_command.py @@ -0,0 +1,109 @@ +import os +import re +import socket +import threading + +try: + # Python 3 + from urllib.parse import urlparse +except (ImportError): + # Python 2 + from urlparse import urlparse + +import sublime +import sublime_plugin + +from ..show_error import show_error +from ..open_compat import open_compat +from ..ca_certs import find_root_ca_cert +from ..thread_progress import ThreadProgress +from ..package_manager import PackageManager + + +class GrabCertsCommand(sublime_plugin.WindowCommand): + """ + A command that extracts the CA certs for a domain name, allowing a user to + fetch packages from sources other than those used by the default channel + """ + + def run(self): + panel = self.window.show_input_panel('Domain Name', 'example.com', self.on_done, + None, None) + panel.sel().add(sublime.Region(0, panel.size())) + + def on_done(self, domain): + """ + Input panel handler - grabs the CA certs for the domain name presented + + :param domain: + A string of the domain name + """ + + domain = domain.strip() + + # Make sure the user enters something + if domain == '': + show_error(u"Please enter a domain name, or press cancel") + self.run() + return + + # If the user inputs a URL, extract the domain name + if domain.find('/') != -1: + parts = urlparse(domain) + if parts.hostname: + domain = parts.hostname + + # Allow _ even though it technically isn't valid, this is really + # just to try and prevent people from typing in gibberish + if re.match('^(?:[a-zA-Z0-9]+(?:[\-_]*[a-zA-Z0-9]+)*\.)+[a-zA-Z]{2,6}$', domain, re.I) == None: + show_error(u"Unable to get the CA certs for \"%s\" since it does not appear to be a validly formed domain name" % domain) + return + + # Make sure it is a real domain + try: + socket.gethostbyname(domain) + except (socket.gaierror) as e: + error = unicode_from_os(e) + show_error(u"Error trying to lookup \"%s\":\n\n%s" % (domain, error)) + return + + manager = PackageManager() + + thread = GrabCertsThread(manager.settings, domain) + thread.start() + ThreadProgress(thread, 'Grabbing CA certs for %s' % domain, + 'CA certs for %s added to settings' % domain) + + +class GrabCertsThread(threading.Thread): + """ + A thread to run openssl so that the Sublime Text UI does not become frozen + """ + + def __init__(self, settings, domain): + self.settings = settings + self.domain = domain + threading.Thread.__init__(self) + + def run(self): + cert, cert_hash = find_root_ca_cert(self.settings, self.domain) + + certs_dir = os.path.join(sublime.packages_path(), 'User', + 'Package Control.ca-certs') + if not os.path.exists(certs_dir): + os.mkdir(certs_dir) + + cert_path = os.path.join(certs_dir, self.domain + '-ca.crt') + with open_compat(cert_path, 'w') as f: + f.write(cert) + + def save_certs(): + settings = sublime.load_settings('Package Control.sublime-settings') + certs = settings.get('certs', {}) + if not certs: + certs = {} + certs[self.domain] = [cert_hash, cert_path] + settings.set('certs', certs) + sublime.save_settings('Package Control.sublime-settings') + + sublime.set_timeout(save_certs, 10) diff --git a/sublime/Packages/Package Control/package_control/commands/install_package_command.py b/sublime/Packages/Package Control/package_control/commands/install_package_command.py new file mode 100644 index 0000000..bbe9031 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/install_package_command.py @@ -0,0 +1,50 @@ +import threading + +import sublime +import sublime_plugin + +from ..show_error import show_error +from ..package_installer import PackageInstaller +from ..thread_progress import ThreadProgress + + +class InstallPackageCommand(sublime_plugin.WindowCommand): + """ + A command that presents the list of available packages and allows the + user to pick one to install. + """ + + def run(self): + thread = InstallPackageThread(self.window) + thread.start() + ThreadProgress(thread, 'Loading repositories', '') + + +class InstallPackageThread(threading.Thread, PackageInstaller): + """ + A thread to run the action of retrieving available packages in. Uses the + default PackageInstaller.on_done quick panel handler. + """ + + def __init__(self, window): + """ + :param window: + An instance of :class:`sublime.Window` that represents the Sublime + Text window to show the available package list in. + """ + + self.window = window + self.completion_type = 'installed' + threading.Thread.__init__(self) + PackageInstaller.__init__(self) + + def run(self): + self.package_list = self.make_package_list(['upgrade', 'downgrade', + 'reinstall', 'pull', 'none']) + + def show_quick_panel(): + if not self.package_list: + show_error('There are no packages available for installation') + return + self.window.show_quick_panel(self.package_list, self.on_done) + sublime.set_timeout(show_quick_panel, 10) diff --git a/sublime/Packages/Package Control/package_control/commands/list_packages_command.py b/sublime/Packages/Package Control/package_control/commands/list_packages_command.py new file mode 100644 index 0000000..84c57e4 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/list_packages_command.py @@ -0,0 +1,63 @@ +import threading +import os + +import sublime +import sublime_plugin + +from ..show_error import show_error +from .existing_packages_command import ExistingPackagesCommand + + +class ListPackagesCommand(sublime_plugin.WindowCommand): + """ + A command that shows a list of all installed packages in the quick panel + """ + + def run(self): + ListPackagesThread(self.window).start() + + +class ListPackagesThread(threading.Thread, ExistingPackagesCommand): + """ + A thread to prevent the listing of existing packages from freezing the UI + """ + + def __init__(self, window): + """ + :param window: + An instance of :class:`sublime.Window` that represents the Sublime + Text window to show the list of installed packages in. + """ + + self.window = window + threading.Thread.__init__(self) + ExistingPackagesCommand.__init__(self) + + def run(self): + self.package_list = self.make_package_list() + + def show_quick_panel(): + if not self.package_list: + show_error('There are no packages to list') + return + self.window.show_quick_panel(self.package_list, self.on_done) + sublime.set_timeout(show_quick_panel, 10) + + def on_done(self, picked): + """ + Quick panel user selection handler - opens the homepage for any + selected package in the user's browser + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + package_name = self.package_list[picked][0] + + def open_dir(): + self.window.run_command('open_dir', + {"dir": os.path.join(sublime.packages_path(), package_name)}) + sublime.set_timeout(open_dir, 10) diff --git a/sublime/Packages/Package Control/package_control/commands/package_message_command.py b/sublime/Packages/Package Control/package_control/commands/package_message_command.py new file mode 100644 index 0000000..6e083df --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/package_message_command.py @@ -0,0 +1,11 @@ +import sublime +import sublime_plugin + + +class PackageMessageCommand(sublime_plugin.TextCommand): + """ + A command to write a package message to the Package Control messaging buffer + """ + + def run(self, edit, string=''): + self.view.insert(edit, self.view.size(), string) diff --git a/sublime/Packages/Package Control/package_control/commands/remove_package_command.py b/sublime/Packages/Package Control/package_control/commands/remove_package_command.py new file mode 100644 index 0000000..df0350c --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/remove_package_command.py @@ -0,0 +1,88 @@ +import threading + +import sublime +import sublime_plugin + +from ..show_error import show_error +from .existing_packages_command import ExistingPackagesCommand +from ..preferences_filename import preferences_filename +from ..thread_progress import ThreadProgress + + +class RemovePackageCommand(sublime_plugin.WindowCommand, + ExistingPackagesCommand): + """ + A command that presents a list of installed packages, allowing the user to + select one to remove + """ + + def __init__(self, window): + """ + :param window: + An instance of :class:`sublime.Window` that represents the Sublime + Text window to show the list of installed packages in. + """ + + self.window = window + ExistingPackagesCommand.__init__(self) + + def run(self): + self.package_list = self.make_package_list('remove') + if not self.package_list: + show_error('There are no packages that can be removed.') + return + self.window.show_quick_panel(self.package_list, self.on_done) + + def on_done(self, picked): + """ + Quick panel user selection handler - deletes the selected package + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + package = self.package_list[picked][0] + + settings = sublime.load_settings(preferences_filename()) + ignored = settings.get('ignored_packages') + if not ignored: + ignored = [] + + # Don't disable Package Control so it does not get stuck disabled + if package != 'Package Control': + if not package in ignored: + ignored.append(package) + settings.set('ignored_packages', ignored) + sublime.save_settings(preferences_filename()) + ignored.remove(package) + + thread = RemovePackageThread(self.manager, package, + ignored) + thread.start() + ThreadProgress(thread, 'Removing package %s' % package, + 'Package %s successfully removed' % package) + + +class RemovePackageThread(threading.Thread): + """ + A thread to run the remove package operation in so that the Sublime Text + UI does not become frozen + """ + + def __init__(self, manager, package, ignored): + self.manager = manager + self.package = package + self.ignored = ignored + threading.Thread.__init__(self) + + def run(self): + self.result = self.manager.remove_package(self.package) + + def unignore_package(): + settings = sublime.load_settings(preferences_filename()) + settings.set('ignored_packages', self.ignored) + sublime.save_settings(preferences_filename()) + sublime.set_timeout(unignore_package, 10) diff --git a/sublime/Packages/Package Control/package_control/commands/upgrade_all_packages_command.py b/sublime/Packages/Package Control/package_control/commands/upgrade_all_packages_command.py new file mode 100644 index 0000000..a4a730d --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/upgrade_all_packages_command.py @@ -0,0 +1,77 @@ +import time +import threading + +import sublime +import sublime_plugin + +from ..thread_progress import ThreadProgress +from ..package_installer import PackageInstaller, PackageInstallerThread +from ..package_renamer import PackageRenamer + + +class UpgradeAllPackagesCommand(sublime_plugin.WindowCommand): + """ + A command to automatically upgrade all installed packages that are + upgradable. + """ + + def run(self): + package_renamer = PackageRenamer() + package_renamer.load_settings() + + thread = UpgradeAllPackagesThread(self.window, package_renamer) + thread.start() + ThreadProgress(thread, 'Loading repositories', '') + + +class UpgradeAllPackagesThread(threading.Thread, PackageInstaller): + """ + A thread to run the action of retrieving upgradable packages in. + """ + + def __init__(self, window, package_renamer): + self.window = window + self.package_renamer = package_renamer + self.completion_type = 'upgraded' + threading.Thread.__init__(self) + PackageInstaller.__init__(self) + + def run(self): + self.package_renamer.rename_packages(self) + package_list = self.make_package_list(['install', 'reinstall', 'none']) + + disabled_packages = [] + + def do_upgrades(): + # Pause so packages can be disabled + time.sleep(0.5) + + # We use a function to generate the on-complete lambda because if + # we don't, the lambda will bind to info at the current scope, and + # thus use the last value of info from the loop + def make_on_complete(name): + return lambda: self.reenable_package(name) + + for info in package_list: + if info[0] in disabled_packages: + on_complete = make_on_complete(info[0]) + else: + on_complete = None + thread = PackageInstallerThread(self.manager, info[0], + on_complete) + thread.start() + ThreadProgress(thread, 'Upgrading package %s' % info[0], + 'Package %s successfully %s' % (info[0], + self.completion_type)) + + # Disabling a package means changing settings, which can only be done + # in the main thread. We then create a new background thread so that + # the upgrade process does not block the UI. + def disable_packages(): + package_names = [] + for info in package_list: + package_names.append(info[0]) + disabled_packages.extend(self.disable_packages(package_names)) + threading.Thread(target=do_upgrades).start() + + sublime.set_timeout(disable_packages, 1) diff --git a/sublime/Packages/Package Control/package_control/commands/upgrade_package_command.py b/sublime/Packages/Package Control/package_control/commands/upgrade_package_command.py new file mode 100644 index 0000000..6c478e6 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/commands/upgrade_package_command.py @@ -0,0 +1,81 @@ +import threading + +import sublime +import sublime_plugin + +from ..show_error import show_error +from ..thread_progress import ThreadProgress +from ..package_installer import PackageInstaller, PackageInstallerThread +from ..package_renamer import PackageRenamer + + +class UpgradePackageCommand(sublime_plugin.WindowCommand): + """ + A command that presents the list of installed packages that can be upgraded + """ + + def run(self): + package_renamer = PackageRenamer() + package_renamer.load_settings() + + thread = UpgradePackageThread(self.window, package_renamer) + thread.start() + ThreadProgress(thread, 'Loading repositories', '') + + +class UpgradePackageThread(threading.Thread, PackageInstaller): + """ + A thread to run the action of retrieving upgradable packages in. + """ + + def __init__(self, window, package_renamer): + """ + :param window: + An instance of :class:`sublime.Window` that represents the Sublime + Text window to show the list of upgradable packages in. + + :param package_renamer: + An instance of :class:`PackageRenamer` + """ + self.window = window + self.package_renamer = package_renamer + self.completion_type = 'upgraded' + threading.Thread.__init__(self) + PackageInstaller.__init__(self) + + def run(self): + self.package_renamer.rename_packages(self) + + self.package_list = self.make_package_list(['install', 'reinstall', + 'none']) + + def show_quick_panel(): + if not self.package_list: + show_error('There are no packages ready for upgrade') + return + self.window.show_quick_panel(self.package_list, self.on_done) + sublime.set_timeout(show_quick_panel, 10) + + def on_done(self, picked): + """ + Quick panel user selection handler - disables a package, upgrades it, + then re-enables the package + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + name = self.package_list[picked][0] + + if name in self.disable_packages(name): + on_complete = lambda: self.reenable_package(name) + else: + on_complete = None + + thread = PackageInstallerThread(self.manager, name, on_complete) + thread.start() + ThreadProgress(thread, 'Upgrading package %s' % name, + 'Package %s successfully %s' % (name, self.completion_type)) diff --git a/sublime/Packages/Package Control/package_control/console_write.py b/sublime/Packages/Package Control/package_control/console_write.py new file mode 100644 index 0000000..5fb0796 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/console_write.py @@ -0,0 +1,20 @@ +import sys + + +def console_write(string, prefix=False): + """ + Writes a value to the Sublime Text console, encoding unicode to utf-8 first + + :param string: + The value to write + + :param prefix: + If the string "Package Control: " should be prefixed to the string + """ + + if sys.version_info < (3,): + if isinstance(string, unicode): + string = string.encode('UTF-8') + if prefix: + sys.stdout.write('Package Control: ') + print(string) diff --git a/sublime/Packages/Package Control/package_control/download_manager.py b/sublime/Packages/Package Control/package_control/download_manager.py new file mode 100644 index 0000000..a4d028d --- /dev/null +++ b/sublime/Packages/Package Control/package_control/download_manager.py @@ -0,0 +1,231 @@ +import sys +import re +import socket +from threading import Lock, Timer +from contextlib import contextmanager + +try: + # Python 3 + from urllib.parse import urlparse +except (ImportError): + # Python 2 + from urlparse import urlparse + +from . import __version__ + +from .show_error import show_error +from .console_write import console_write +from .cache import set_cache, get_cache +from .unicode import unicode_from_os + +from .downloaders import DOWNLOADERS +from .downloaders.binary_not_found_error import BinaryNotFoundError +from .downloaders.rate_limit_exception import RateLimitException +from .downloaders.no_ca_cert_exception import NoCaCertException +from .downloaders.downloader_exception import DownloaderException +from .http_cache import HttpCache + + +# A dict of domains - each points to a list of downloaders +_managers = {} + +# How many managers are currently checked out +_in_use = 0 + +# Make sure connection management doesn't run into threading issues +_lock = Lock() + +# A timer used to disconnect all managers after a period of no usage +_timer = None + + +@contextmanager +def downloader(url, settings): + try: + manager = _grab(url, settings) + yield manager + + finally: + _release(url, manager) + + +def _grab(url, settings): + global _managers, _lock, _in_use, _timer + + _lock.acquire() + try: + if _timer: + _timer.cancel() + _timer = None + + hostname = urlparse(url).hostname.lower() + if hostname not in _managers: + _managers[hostname] = [] + + if not _managers[hostname]: + _managers[hostname].append(DownloadManager(settings)) + + _in_use += 1 + + return _managers[hostname].pop() + + finally: + _lock.release() + + +def _release(url, manager): + global _managers, _lock, _in_use, _timer + + _lock.acquire() + try: + hostname = urlparse(url).hostname.lower() + _managers[hostname].insert(0, manager) + + _in_use -= 1 + + if _timer: + _timer.cancel() + _timer = None + + if _in_use == 0: + _timer = Timer(5.0, close_all_connections) + _timer.start() + + finally: + _lock.release() + + +def close_all_connections(): + global _managers, _lock, _in_use, _timer + + _lock.acquire() + try: + if _timer: + _timer.cancel() + _timer = None + + for domain, managers in _managers.items(): + for manager in managers: + manager.close() + _managers = {} + + finally: + _lock.release() + + +class DownloadManager(object): + def __init__(self, settings): + # Cache the downloader for re-use + self.downloader = None + + user_agent = settings.get('user_agent') + if user_agent and user_agent.find('%s') != -1: + settings['user_agent'] = user_agent % __version__ + + self.settings = settings + if settings.get('http_cache'): + cache_length = settings.get('http_cache_length', 604800) + self.settings['cache'] = HttpCache(cache_length) + + def close(self): + if self.downloader: + self.downloader.close() + self.downloader = None + + def fetch(self, url, error_message, prefer_cached=False): + """ + Downloads a URL and returns the contents + + :param url: + The string URL to download + + :param error_message: + The error message to include if the download fails + + :param prefer_cached: + If cached version of the URL content is preferred over a new request + + :raises: + DownloaderException: if there was an error downloading the URL + + :return: + The string contents of the URL + """ + + is_ssl = re.search('^https://', url) != None + + # Make sure we have a downloader, and it supports SSL if we need it + if not self.downloader or (is_ssl and not self.downloader.supports_ssl()): + for downloader_class in DOWNLOADERS: + try: + downloader = downloader_class(self.settings) + if is_ssl and not downloader.supports_ssl(): + continue + self.downloader = downloader + break + except (BinaryNotFoundError): + pass + + if not self.downloader: + error_string = u'Unable to download %s due to no ssl module available and no capable program found. Please install curl or wget.' % url + show_error(error_string) + raise DownloaderException(error_string) + + url = url.replace(' ', '%20') + hostname = urlparse(url).hostname + if hostname: + hostname = hostname.lower() + timeout = self.settings.get('timeout', 3) + + rate_limited_domains = get_cache('rate_limited_domains', []) + no_ca_cert_domains = get_cache('no_ca_cert_domains', []) + + if self.settings.get('debug'): + try: + ip = socket.gethostbyname(hostname) + except (socket.gaierror) as e: + ip = unicode_from_os(e) + except (TypeError) as e: + ip = None + + console_write(u"Download Debug", True) + console_write(u" URL: %s" % url) + console_write(u" Resolved IP: %s" % ip) + console_write(u" Timeout: %s" % str(timeout)) + + if hostname in rate_limited_domains: + error_string = u"Skipping due to hitting rate limit for %s" % hostname + if self.settings.get('debug'): + console_write(u" %s" % error_string) + raise DownloaderException(error_string) + + if hostname in no_ca_cert_domains: + error_string = u" Skipping since there are no CA certs for %s" % hostname + if self.settings.get('debug'): + console_write(u" %s" % error_string) + raise DownloaderException(error_string) + + try: + return self.downloader.download(url, error_message, timeout, 3, prefer_cached) + + except (RateLimitException) as e: + + rate_limited_domains.append(hostname) + set_cache('rate_limited_domains', rate_limited_domains, self.settings.get('cache_length')) + + error_string = (u'Hit rate limit of %s for %s, skipping all futher ' + + u'download requests for this domain') % (e.limit, e.domain) + console_write(error_string, True) + raise + + except (NoCaCertException) as e: + + no_ca_cert_domains.append(hostname) + set_cache('no_ca_cert_domains', no_ca_cert_domains, self.settings.get('cache_length')) + + error_string = (u'No CA certs available for %s, skipping all futher ' + + u'download requests for this domain. If you are on a trusted ' + + u'network, you can add the CA certs by running the "Grab ' + + u'CA Certs" command from the command palette.') % e.domain + console_write(error_string, True) + raise diff --git a/sublime/Packages/Package Control/package_control/downloaders/__init__.py b/sublime/Packages/Package Control/package_control/downloaders/__init__.py new file mode 100644 index 0000000..fb68aef --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/__init__.py @@ -0,0 +1,11 @@ +import os + +if os.name == 'nt': + from .wininet_downloader import WinINetDownloader + DOWNLOADERS = [WinINetDownloader] + +else: + from .urllib_downloader import UrlLibDownloader + from .curl_downloader import CurlDownloader + from .wget_downloader import WgetDownloader + DOWNLOADERS = [UrlLibDownloader, CurlDownloader, WgetDownloader] diff --git a/sublime/Packages/Package Control/package_control/downloaders/background_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/background_downloader.py new file mode 100644 index 0000000..250d2de --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/background_downloader.py @@ -0,0 +1,62 @@ +import threading + + +class BackgroundDownloader(threading.Thread): + """ + Downloads information from one or more URLs in the background. + Normal usage is to use one BackgroundDownloader per domain name. + + :param settings: + A dict containing at least the following fields: + `cache_length`, + `debug`, + `timeout`, + `user_agent`, + `http_proxy`, + `https_proxy`, + `proxy_username`, + `proxy_password` + + :param providers: + An array of providers that can download the URLs + """ + + def __init__(self, settings, providers): + self.settings = settings + self.urls = [] + self.providers = providers + self.used_providers = {} + threading.Thread.__init__(self) + + def add_url(self, url): + """ + Adds a URL to the list to download + + :param url: + The URL to download info about + """ + + self.urls.append(url) + + def get_provider(self, url): + """ + Returns the provider for the URL specified + + :param url: + The URL to return the provider for + + :return: + The provider object for the URL + """ + + return self.used_providers[url] + + def run(self): + for url in self.urls: + for provider_class in self.providers: + if provider_class.match_url(url): + provider = provider_class(url, self.settings) + break + + provider.prefetch() + self.used_providers[url] = provider diff --git a/sublime/Packages/Package Control/package_control/downloaders/binary_not_found_error.py b/sublime/Packages/Package Control/package_control/downloaders/binary_not_found_error.py new file mode 100644 index 0000000..a7955b9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/binary_not_found_error.py @@ -0,0 +1,4 @@ +class BinaryNotFoundError(Exception): + """If a necessary executable is not found in the PATH on the system""" + + pass diff --git a/sublime/Packages/Package Control/package_control/downloaders/caching_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/caching_downloader.py new file mode 100644 index 0000000..ab3d71f --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/caching_downloader.py @@ -0,0 +1,185 @@ +import sys +import re +import json +import hashlib + +from ..console_write import console_write + + +class CachingDownloader(object): + """ + A base downloader that will use a caching backend to cache HTTP requests + and make conditional requests. + """ + + def add_conditional_headers(self, url, headers): + """ + Add `If-Modified-Since` and `If-None-Match` headers to a request if a + cached copy exists + + :param headers: + A dict with the request headers + + :return: + The request headers dict, possibly with new headers added + """ + + if not self.settings.get('cache'): + return headers + + info_key = self.generate_key(url, '.info') + info_json = self.settings['cache'].get(info_key) + + if not info_json: + return headers + + # Make sure we have the cached content to use if we get a 304 + key = self.generate_key(url) + if not self.settings['cache'].has(key): + return headers + + try: + info = json.loads(info_json.decode('utf-8')) + except ValueError: + return headers + + etag = info.get('etag') + if etag: + headers['If-None-Match'] = etag + + last_modified = info.get('last-modified') + if last_modified: + headers['If-Modified-Since'] = last_modified + + return headers + + def cache_result(self, method, url, status, headers, content): + """ + Processes a request result, either caching the result, or returning + the cached version of the url. + + :param method: + The HTTP method used for the request + + :param url: + The url of the request + + :param status: + The numeric response status of the request + + :param headers: + A dict of reponse headers, with keys being lowercase + + :param content: + The response content + + :return: + The response content + """ + + debug = self.settings.get('debug', False) + + if not self.settings.get('cache'): + if debug: + console_write(u"Skipping cache since there is no cache object", True) + return content + + if method.lower() != 'get': + if debug: + console_write(u"Skipping cache since the HTTP method != GET", True) + return content + + status = int(status) + + # Don't do anything unless it was successful or not modified + if status not in [200, 304]: + if debug: + console_write(u"Skipping cache since the HTTP status code not one of: 200, 304", True) + return content + + key = self.generate_key(url) + + if status == 304: + cached_content = self.settings['cache'].get(key) + if cached_content: + if debug: + console_write(u"Using cached content for %s" % url, True) + return cached_content + + # If we got a 304, but did not have the cached content + # stop here so we don't cache an empty response + return content + + # If we got here, the status is 200 + + # Respect some basic cache control headers + cache_control = headers.get('cache-control', '') + if cache_control: + fields = re.split(',\s*', cache_control) + for field in fields: + if field == 'no-store': + return content + + # Don't ever cache zip/binary files for the sake of hard drive space + if headers.get('content-type') in ['application/zip', 'application/octet-stream']: + if debug: + console_write(u"Skipping cache since the response is a zip file", True) + return content + + etag = headers.get('etag') + last_modified = headers.get('last-modified') + + if not etag and not last_modified: + return content + + struct = {'etag': etag, 'last-modified': last_modified} + struct_json = json.dumps(struct, indent=4) + + info_key = self.generate_key(url, '.info') + if debug: + console_write(u"Caching %s in %s" % (url, key), True) + + self.settings['cache'].set(info_key, struct_json.encode('utf-8')) + self.settings['cache'].set(key, content) + + return content + + def generate_key(self, url, suffix=''): + """ + Generates a key to store the cache under + + :param url: + The URL being cached + + :param suffix: + A string to append to the key + + :return: + A string key for the URL + """ + + if sys.version_info >= (3,) or isinstance(url, unicode): + url = url.encode('utf-8') + + key = hashlib.md5(url).hexdigest() + return key + suffix + + def retrieve_cached(self, url): + """ + Tries to return the cached content for a URL + + :param url: + The URL to get the cached content for + + :return: + The cached content + """ + + key = self.generate_key(url) + if not self.settings['cache'].has(key): + return False + + if self.settings.get('debug'): + console_write(u"Using cached content for %s" % url, True) + + return self.settings['cache'].get(key) diff --git a/sublime/Packages/Package Control/package_control/downloaders/cert_provider.py b/sublime/Packages/Package Control/package_control/downloaders/cert_provider.py new file mode 100644 index 0000000..f8c8c3b --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/cert_provider.py @@ -0,0 +1,203 @@ +import os +import re +import json + +import sublime + +from ..console_write import console_write +from ..open_compat import open_compat, read_compat +from ..package_io import read_package_file +from ..cache import get_cache +from ..ca_certs import get_system_ca_bundle_path +from .no_ca_cert_exception import NoCaCertException +from .downloader_exception import DownloaderException + + +class CertProvider(object): + """ + A base downloader that provides access to a ca-bundle for validating + SSL certificates. + """ + + def check_certs(self, domain, timeout): + """ + Ensures that the SSL CA cert for a domain is present on the machine + + :param domain: + The domain to ensure there is a CA cert for + + :param timeout: + The int timeout for downloading the CA cert from the channel + + :raises: + NoCaCertException: when a suitable CA cert could not be found + + :return: + The CA cert bundle path + """ + + # Try to use the system CA bundle + ca_bundle_path = get_system_ca_bundle_path(self.settings) + if ca_bundle_path: + return ca_bundle_path + + # If the system bundle did not work, fall back to our CA distribution + # system. Hopefully this will be going away soon. + if self.settings.get('debug'): + console_write(u'Unable to find system CA cert bundle, falling back to certs provided by Package Control') + + cert_match = False + + certs_list = get_cache('*.certs', self.settings.get('certs', {})) + + ca_bundle_path = os.path.join(sublime.packages_path(), 'User', 'Package Control.ca-bundle') + if not os.path.exists(ca_bundle_path) or os.stat(ca_bundle_path).st_size == 0: + bundle_contents = read_package_file('Package Control', 'Package Control.ca-bundle', True) + if not bundle_contents: + raise NoCaCertException(u'Unable to copy distributed Package Control.ca-bundle', domain) + with open_compat(ca_bundle_path, 'wb') as f: + f.write(bundle_contents) + + cert_info = certs_list.get(domain) + if cert_info: + cert_match = self.locate_cert(cert_info[0], + cert_info[1], domain, timeout) + + wildcard_info = certs_list.get('*') + if wildcard_info: + cert_match = self.locate_cert(wildcard_info[0], + wildcard_info[1], domain, timeout) or cert_match + + if not cert_match: + raise NoCaCertException(u'No CA certs available for %s' % domain, domain) + + return ca_bundle_path + + def locate_cert(self, cert_id, location, domain, timeout): + """ + Makes sure the SSL cert specified has been added to the CA cert + bundle that is present on the machine + + :param cert_id: + The identifier for CA cert(s). For those provided by the channel + system, this will be an md5 of the contents of the cert(s). For + user-provided certs, this is something they provide. + + :param location: + An http(s) URL, or absolute filesystem path to the CA cert(s) + + :param domain: + The domain to ensure there is a CA cert for + + :param timeout: + The int timeout for downloading the CA cert from the channel + + :return: + If the cert specified (by cert_id) is present on the machine and + part of the Package Control.ca-bundle file in the User package folder + """ + + ca_list_path = os.path.join(sublime.packages_path(), 'User', 'Package Control.ca-list') + if not os.path.exists(ca_list_path) or os.stat(ca_list_path).st_size == 0: + list_contents = read_package_file('Package Control', 'Package Control.ca-list') + if not list_contents: + raise NoCaCertException(u'Unable to copy distributed Package Control.ca-list', domain) + with open_compat(ca_list_path, 'w') as f: + f.write(list_contents) + + ca_certs = [] + with open_compat(ca_list_path, 'r') as f: + ca_certs = json.loads(read_compat(f)) + + if not cert_id in ca_certs: + if str(location) != '': + if re.match('^https?://', location): + contents = self.download_cert(cert_id, location, domain, + timeout) + else: + contents = self.load_cert(cert_id, location, domain) + if contents: + self.save_cert(cert_id, contents) + return True + return False + return True + + def download_cert(self, cert_id, url, domain, timeout): + """ + Downloads CA cert(s) from a URL + + :param cert_id: + The identifier for CA cert(s). For those provided by the channel + system, this will be an md5 of the contents of the cert(s). For + user-provided certs, this is something they provide. + + :param url: + An http(s) URL to the CA cert(s) + + :param domain: + The domain to ensure there is a CA cert for + + :param timeout: + The int timeout for downloading the CA cert from the channel + + :return: + The contents of the CA cert(s) + """ + + cert_downloader = self.__class__(self.settings) + if self.settings.get('debug'): + console_write(u"Downloading CA cert for %s from \"%s\"" % (domain, url), True) + return cert_downloader.download(url, + 'Error downloading CA certs for %s.' % domain, timeout, 1) + + def load_cert(self, cert_id, path, domain): + """ + Copies CA cert(s) from a file path + + :param cert_id: + The identifier for CA cert(s). For those provided by the channel + system, this will be an md5 of the contents of the cert(s). For + user-provided certs, this is something they provide. + + :param path: + The absolute filesystem path to a file containing the CA cert(s) + + :param domain: + The domain name the cert is for + + :return: + The contents of the CA cert(s) + """ + + if os.path.exists(path): + if self.settings.get('debug'): + console_write(u"Copying CA cert for %s from \"%s\"" % (domain, path), True) + with open_compat(path, 'rb') as f: + return f.read() + else: + raise NoCaCertException(u"Unable to find CA cert for %s at \"%s\"" % (domain, path), domain) + + def save_cert(self, cert_id, contents): + """ + Saves CA cert(s) to the Package Control.ca-bundle + + :param cert_id: + The identifier for CA cert(s). For those provided by the channel + system, this will be an md5 of the contents of the cert(s). For + user-provided certs, this is something they provide. + + :param contents: + The contents of the CA cert(s) + """ + + + ca_bundle_path = os.path.join(sublime.packages_path(), 'User', 'Package Control.ca-bundle') + with open_compat(ca_bundle_path, 'ab') as f: + f.write(b"\n" + contents) + + ca_list_path = os.path.join(sublime.packages_path(), 'User', 'Package Control.ca-list') + with open_compat(ca_list_path, 'r') as f: + ca_certs = json.loads(read_compat(f)) + ca_certs.append(cert_id) + with open_compat(ca_list_path, 'w') as f: + f.write(json.dumps(ca_certs, indent=4)) diff --git a/sublime/Packages/Package Control/package_control/downloaders/cli_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/cli_downloader.py new file mode 100644 index 0000000..76c42dd --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/cli_downloader.py @@ -0,0 +1,81 @@ +import os +import subprocess + +from ..console_write import console_write +from ..cmd import create_cmd +from .non_clean_exit_error import NonCleanExitError +from .binary_not_found_error import BinaryNotFoundError + + +class CliDownloader(object): + """ + Base for downloaders that use a command line program + + :param settings: + A dict of the various Package Control settings. The Sublime Text + Settings API is not used because this code is run in a thread. + """ + + def __init__(self, settings): + self.settings = settings + + def clean_tmp_file(self): + if os.path.exists(self.tmp_file): + os.remove(self.tmp_file) + + def find_binary(self, name): + """ + Finds the given executable name in the system PATH + + :param name: + The exact name of the executable to find + + :return: + The absolute path to the executable + + :raises: + BinaryNotFoundError when the executable can not be found + """ + + dirs = os.environ['PATH'].split(os.pathsep) + if os.name != 'nt': + # This is mostly for OS X, which seems to launch ST with a + # minimal set of environmental variables + dirs.append('/usr/local/bin') + + for dir_ in dirs: + path = os.path.join(dir_, name) + if os.path.exists(path): + return path + + raise BinaryNotFoundError('The binary %s could not be located' % name) + + def execute(self, args): + """ + Runs the executable and args and returns the result + + :param args: + A list of the executable path and all arguments to be passed to it + + :return: + The text output of the executable + + :raises: + NonCleanExitError when the executable exits with an error + """ + + if self.settings.get('debug'): + console_write(u"Trying to execute command %s" % create_cmd(args), True) + + proc = subprocess.Popen(args, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + + output = proc.stdout.read() + self.stderr = proc.stderr.read() + returncode = proc.wait() + if returncode != 0: + error = NonCleanExitError(returncode) + error.stderr = self.stderr + error.stdout = output + raise error + return output diff --git a/sublime/Packages/Package Control/package_control/downloaders/curl_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/curl_downloader.py new file mode 100644 index 0000000..b09d448 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/curl_downloader.py @@ -0,0 +1,267 @@ +import tempfile +import re +import os + +from ..console_write import console_write +from ..open_compat import open_compat, read_compat +from .cli_downloader import CliDownloader +from .non_clean_exit_error import NonCleanExitError +from .rate_limit_exception import RateLimitException +from .downloader_exception import DownloaderException +from .cert_provider import CertProvider +from .limiting_downloader import LimitingDownloader +from .caching_downloader import CachingDownloader + + +class CurlDownloader(CliDownloader, CertProvider, LimitingDownloader, CachingDownloader): + """ + A downloader that uses the command line program curl + + :param settings: + A dict of the various Package Control settings. The Sublime Text + Settings API is not used because this code is run in a thread. + + :raises: + BinaryNotFoundError: when curl can not be found + """ + + def __init__(self, settings): + self.settings = settings + self.curl = self.find_binary('curl') + + def close(self): + """ + No-op for compatibility with UrllibDownloader and WinINetDownloader + """ + + pass + + def download(self, url, error_message, timeout, tries, prefer_cached=False): + """ + Downloads a URL and returns the contents + + :param url: + The URL to download + + :param error_message: + A string to include in the console error that is printed + when an error occurs + + :param timeout: + The int number of seconds to set the timeout to + + :param tries: + The int number of times to try and download the URL in the case of + a timeout or HTTP 503 error + + :param prefer_cached: + If a cached version should be returned instead of trying a new request + + :raises: + NoCaCertException: when no CA certs can be found for the url + RateLimitException: when a rate limit is hit + DownloaderException: when any other download error occurs + + :return: + The string contents of the URL + """ + + if prefer_cached: + cached = self.retrieve_cached(url) + if cached: + return cached + + self.tmp_file = tempfile.NamedTemporaryFile().name + command = [self.curl, '--user-agent', self.settings.get('user_agent'), + '--connect-timeout', str(int(timeout)), '-sSL', + # Don't be alarmed if the response from the server does not select + # one of these since the server runs a relatively new version of + # OpenSSL which supports compression on the SSL layer, and Apache + # will use that instead of HTTP-level encoding. + '--compressed', + # We have to capture the headers to check for rate limit info + '--dump-header', self.tmp_file] + + request_headers = self.add_conditional_headers(url, {}) + + for name, value in request_headers.items(): + command.extend(['--header', "%s: %s" % (name, value)]) + + secure_url_match = re.match('^https://([^/]+)', url) + if secure_url_match != None: + secure_domain = secure_url_match.group(1) + bundle_path = self.check_certs(secure_domain, timeout) + command.extend(['--cacert', bundle_path]) + + debug = self.settings.get('debug') + if debug: + command.append('-v') + + http_proxy = self.settings.get('http_proxy') + https_proxy = self.settings.get('https_proxy') + proxy_username = self.settings.get('proxy_username') + proxy_password = self.settings.get('proxy_password') + + if debug: + console_write(u"Curl Debug Proxy", True) + console_write(u" http_proxy: %s" % http_proxy) + console_write(u" https_proxy: %s" % https_proxy) + console_write(u" proxy_username: %s" % proxy_username) + console_write(u" proxy_password: %s" % proxy_password) + + if http_proxy or https_proxy: + command.append('--proxy-anyauth') + + if proxy_username or proxy_password: + command.extend(['-U', u"%s:%s" % (proxy_username, proxy_password)]) + + if http_proxy: + os.putenv('http_proxy', http_proxy) + if https_proxy: + os.putenv('HTTPS_PROXY', https_proxy) + + command.append(url) + + error_string = None + while tries > 0: + tries -= 1 + try: + output = self.execute(command) + + with open_compat(self.tmp_file, 'r') as f: + headers_str = read_compat(f) + self.clean_tmp_file() + + message = 'OK' + status = 200 + headers = {} + for header in headers_str.splitlines(): + if header[0:5] == 'HTTP/': + message = re.sub('^HTTP/\d\.\d\s+\d+\s*', '', header) + status = int(re.sub('^HTTP/\d\.\d\s+(\d+)(\s+.*)?$', '\\1', header)) + continue + if header.strip() == '': + continue + name, value = header.split(':', 1) + headers[name.lower()] = value.strip() + + if debug: + self.print_debug(self.stderr.decode('utf-8')) + + self.handle_rate_limit(headers, url) + + if status not in [200, 304]: + e = NonCleanExitError(22) + e.stderr = "%s %s" % (status, message) + raise e + + output = self.cache_result('get', url, status, headers, output) + + return output + + except (NonCleanExitError) as e: + # Stderr is used for both the error message and the debug info + # so we need to process it to extra the debug info + if self.settings.get('debug'): + if hasattr(e.stderr, 'decode'): + e.stderr = e.stderr.decode('utf-8') + e.stderr = self.print_debug(e.stderr) + + self.clean_tmp_file() + + if e.returncode == 22: + code = re.sub('^.*?(\d+)([\w\s]+)?$', '\\1', e.stderr) + if code == '503' and tries != 0: + # GitHub and BitBucket seem to rate limit via 503 + error_string = u'Downloading %s was rate limited' % url + if tries: + error_string += ', trying again' + if debug: + console_write(error_string, True) + continue + + download_error = u'HTTP error ' + code + + elif e.returncode == 6: + download_error = u'URL error host not found' + + elif e.returncode == 28: + # GitHub and BitBucket seem to time out a lot + error_string = u'Downloading %s timed out' % url + if tries: + error_string += ', trying again' + if debug: + console_write(error_string, True) + continue + + else: + download_error = e.stderr.rstrip() + + error_string = u'%s %s downloading %s.' % (error_message, download_error, url) + + break + + raise DownloaderException(error_string) + + def supports_ssl(self): + """ + Indicates if the object can handle HTTPS requests + + :return: + If the object supports HTTPS requests + """ + + return True + + def print_debug(self, string): + """ + Takes debug output from curl and groups and prints it + + :param string: + The complete debug output from curl + + :return: + A string containing any stderr output + """ + + section = 'General' + last_section = None + + output = '' + + for line in string.splitlines(): + # Placeholder for body of request + if line and line[0:2] == '{ ': + continue + if line and line[0:18] == '} [data not shown]': + continue + + if len(line) > 1: + subtract = 0 + if line[0:2] == '* ': + section = 'General' + subtract = 2 + elif line[0:2] == '> ': + section = 'Write' + subtract = 2 + elif line[0:2] == '< ': + section = 'Read' + subtract = 2 + line = line[subtract:] + + # If the line does not start with "* ", "< ", "> " or " " + # then it is a real stderr message + if subtract == 0 and line[0:2] != ' ': + output += line.rstrip() + ' ' + continue + + if line.strip() == '': + continue + + if section != last_section: + console_write(u"Curl HTTP Debug %s" % section, True) + + console_write(u' ' + line) + last_section = section + + return output.rstrip() diff --git a/sublime/Packages/Package Control/package_control/downloaders/decoding_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/decoding_downloader.py new file mode 100644 index 0000000..bc1acf3 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/decoding_downloader.py @@ -0,0 +1,24 @@ +import gzip +import zlib + +try: + # Python 3 + from io import BytesIO as StringIO +except (ImportError): + # Python 2 + from StringIO import StringIO + + +class DecodingDownloader(object): + """ + A base for downloaders that provides the ability to decode gzipped + or deflated content. + """ + + def decode_response(self, encoding, response): + if encoding == 'gzip': + return gzip.GzipFile(fileobj=StringIO(response)).read() + elif encoding == 'deflate': + decompresser = zlib.decompressobj(-zlib.MAX_WBITS) + return decompresser.decompress(response) + decompresser.flush() + return response diff --git a/sublime/Packages/Package Control/package_control/downloaders/downloader_exception.py b/sublime/Packages/Package Control/package_control/downloaders/downloader_exception.py new file mode 100644 index 0000000..7519d8f --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/downloader_exception.py @@ -0,0 +1,5 @@ +class DownloaderException(Exception): + """If a downloader could not download a URL""" + + def __str__(self): + return self.args[0] diff --git a/sublime/Packages/Package Control/package_control/downloaders/http_error.py b/sublime/Packages/Package Control/package_control/downloaders/http_error.py new file mode 100644 index 0000000..996e46d --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/http_error.py @@ -0,0 +1,9 @@ +class HttpError(Exception): + """If a downloader was able to download a URL, but the result was not a 200 or 304""" + + def __init__(self, message, code): + self.code = code + super(HttpError, self).__init__(message) + + def __str__(self): + return self.args[0] diff --git a/sublime/Packages/Package Control/package_control/downloaders/limiting_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/limiting_downloader.py new file mode 100644 index 0000000..10d2f1f --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/limiting_downloader.py @@ -0,0 +1,36 @@ +try: + # Python 3 + from urllib.parse import urlparse +except (ImportError): + # Python 2 + from urlparse import urlparse + +from .rate_limit_exception import RateLimitException + + +class LimitingDownloader(object): + """ + A base for downloaders that checks for rate limiting headers. + """ + + def handle_rate_limit(self, headers, url): + """ + Checks the headers of a response object to make sure we are obeying the + rate limit + + :param headers: + The dict-like object that contains lower-cased headers + + :param url: + The URL that was requested + + :raises: + RateLimitException when the rate limit has been hit + """ + + limit_remaining = headers.get('x-ratelimit-remaining', '1') + limit = headers.get('x-ratelimit-limit', '1') + + if str(limit_remaining) == '0': + hostname = urlparse(url).hostname + raise RateLimitException(hostname, limit) diff --git a/sublime/Packages/Package Control/package_control/downloaders/no_ca_cert_exception.py b/sublime/Packages/Package Control/package_control/downloaders/no_ca_cert_exception.py new file mode 100644 index 0000000..8452bd9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/no_ca_cert_exception.py @@ -0,0 +1,11 @@ +from .downloader_exception import DownloaderException + + +class NoCaCertException(DownloaderException): + """ + An exception for when there is no CA cert for a domain name + """ + + def __init__(self, message, domain): + self.domain = domain + super(NoCaCertException, self).__init__(message) diff --git a/sublime/Packages/Package Control/package_control/downloaders/non_clean_exit_error.py b/sublime/Packages/Package Control/package_control/downloaders/non_clean_exit_error.py new file mode 100644 index 0000000..a932363 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/non_clean_exit_error.py @@ -0,0 +1,13 @@ +class NonCleanExitError(Exception): + """ + When an subprocess does not exit cleanly + + :param returncode: + The command line integer return code of the subprocess + """ + + def __init__(self, returncode): + self.returncode = returncode + + def __str__(self): + return repr(self.returncode) diff --git a/sublime/Packages/Package Control/package_control/downloaders/non_http_error.py b/sublime/Packages/Package Control/package_control/downloaders/non_http_error.py new file mode 100644 index 0000000..8a45595 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/non_http_error.py @@ -0,0 +1,5 @@ +class NonHttpError(Exception): + """If a downloader had a non-clean exit, but it was not due to an HTTP error""" + + def __str__(self): + return self.args[0] diff --git a/sublime/Packages/Package Control/package_control/downloaders/rate_limit_exception.py b/sublime/Packages/Package Control/package_control/downloaders/rate_limit_exception.py new file mode 100644 index 0000000..18d2b9e --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/rate_limit_exception.py @@ -0,0 +1,13 @@ +from .downloader_exception import DownloaderException + + +class RateLimitException(DownloaderException): + """ + An exception for when the rate limit of an API has been exceeded. + """ + + def __init__(self, domain, limit): + self.domain = domain + self.limit = limit + message = u'Rate limit of %s exceeded for %s' % (limit, domain) + super(RateLimitException, self).__init__(message) diff --git a/sublime/Packages/Package Control/package_control/downloaders/urllib_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/urllib_downloader.py new file mode 100644 index 0000000..aa04d31 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/urllib_downloader.py @@ -0,0 +1,291 @@ +import re +import os +import sys + +from .. import http + +try: + # Python 3 + from http.client import HTTPException, BadStatusLine + from urllib.request import ProxyHandler, HTTPPasswordMgrWithDefaultRealm, ProxyBasicAuthHandler, ProxyDigestAuthHandler, build_opener, Request + from urllib.error import HTTPError, URLError + import urllib.request as urllib_compat +except (ImportError): + # Python 2 + from httplib import HTTPException, BadStatusLine + from urllib2 import ProxyHandler, HTTPPasswordMgrWithDefaultRealm, ProxyBasicAuthHandler, ProxyDigestAuthHandler, build_opener, Request + from urllib2 import HTTPError, URLError + import urllib2 as urllib_compat + +try: + # Python 3.3 + import ConnectionError +except (ImportError): + # Python 2.6-3.2 + from socket import error as ConnectionError + +from ..console_write import console_write +from ..unicode import unicode_from_os +from ..http.validating_https_handler import ValidatingHTTPSHandler +from ..http.debuggable_http_handler import DebuggableHTTPHandler +from .rate_limit_exception import RateLimitException +from .downloader_exception import DownloaderException +from .cert_provider import CertProvider +from .decoding_downloader import DecodingDownloader +from .limiting_downloader import LimitingDownloader +from .caching_downloader import CachingDownloader + + +class UrlLibDownloader(CertProvider, DecodingDownloader, LimitingDownloader, CachingDownloader): + """ + A downloader that uses the Python urllib module + + :param settings: + A dict of the various Package Control settings. The Sublime Text + Settings API is not used because this code is run in a thread. + """ + + def __init__(self, settings): + self.opener = None + self.settings = settings + + def close(self): + """ + Closes any persistent/open connections + """ + + if not self.opener: + return + handler = self.get_handler() + if handler: + handler.close() + self.opener = None + + def download(self, url, error_message, timeout, tries, prefer_cached=False): + """ + Downloads a URL and returns the contents + + Uses the proxy settings from the Package Control.sublime-settings file, + however there seem to be a decent number of proxies that this code + does not work with. Patches welcome! + + :param url: + The URL to download + + :param error_message: + A string to include in the console error that is printed + when an error occurs + + :param timeout: + The int number of seconds to set the timeout to + + :param tries: + The int number of times to try and download the URL in the case of + a timeout or HTTP 503 error + + :param prefer_cached: + If a cached version should be returned instead of trying a new request + + :raises: + NoCaCertException: when no CA certs can be found for the url + RateLimitException: when a rate limit is hit + DownloaderException: when any other download error occurs + + :return: + The string contents of the URL + """ + + if prefer_cached: + cached = self.retrieve_cached(url) + if cached: + return cached + + self.setup_opener(url, timeout) + + debug = self.settings.get('debug') + error_string = None + while tries > 0: + tries -= 1 + try: + request_headers = { + "User-Agent": self.settings.get('user_agent'), + # Don't be alarmed if the response from the server does not + # select one of these since the server runs a relatively new + # version of OpenSSL which supports compression on the SSL + # layer, and Apache will use that instead of HTTP-level + # encoding. + "Accept-Encoding": "gzip,deflate" + } + request_headers = self.add_conditional_headers(url, request_headers) + request = Request(url, headers=request_headers) + http_file = self.opener.open(request, timeout=timeout) + self.handle_rate_limit(http_file.headers, url) + + result = http_file.read() + # Make sure the response is closed so we can re-use the connection + http_file.close() + + encoding = http_file.headers.get('content-encoding') + result = self.decode_response(encoding, result) + + return self.cache_result('get', url, http_file.getcode(), + http_file.headers, result) + + except (HTTPException) as e: + # Since we use keep-alives, it is possible the other end closed + # the connection, and we may just need to re-open + if isinstance(e, BadStatusLine): + handler = self.get_handler() + if handler and handler.use_count > 1: + self.close() + self.setup_opener(url, timeout) + tries += 1 + continue + + error_string = u'%s HTTP exception %s (%s) downloading %s.' % ( + error_message, e.__class__.__name__, unicode_from_os(e), url) + + except (HTTPError) as e: + # Make sure the response is closed so we can re-use the connection + e.read() + e.close() + + # Make sure we obey Github's rate limiting headers + self.handle_rate_limit(e.headers, url) + + # Handle cached responses + if unicode_from_os(e.code) == '304': + return self.cache_result('get', url, int(e.code), e.headers, b'') + + # Bitbucket and Github return 503 a decent amount + if unicode_from_os(e.code) == '503' and tries != 0: + error_string = u'Downloading %s was rate limited' % url + if tries: + error_string += ', trying again' + if debug: + console_write(error_string, True) + continue + + error_string = u'%s HTTP error %s downloading %s.' % ( + error_message, unicode_from_os(e.code), url) + + except (URLError) as e: + + # Bitbucket and Github timeout a decent amount + if unicode_from_os(e.reason) == 'The read operation timed out' \ + or unicode_from_os(e.reason) == 'timed out': + error_string = u'Downloading %s timed out' % url + if tries: + error_string += ', trying again' + if debug: + console_write(error_string, True) + continue + + error_string = u'%s URL error %s downloading %s.' % ( + error_message, unicode_from_os(e.reason), url) + + except (ConnectionError): + # Handle broken pipes/reset connections by creating a new opener, and + # thus getting new handlers and a new connection + error_string = u'Connection went away while trying to download %s, trying again' % url + if debug: + console_write(error_string, True) + + self.opener = None + self.setup_opener(url, timeout) + tries += 1 + + continue + + break + + raise DownloaderException(error_string) + + def get_handler(self): + """ + Get the HTTPHandler object for the current connection + """ + + if not self.opener: + return None + + for handler in self.opener.handlers: + if isinstance(handler, ValidatingHTTPSHandler) or isinstance(handler, DebuggableHTTPHandler): + return handler + + def setup_opener(self, url, timeout): + """ + Sets up a urllib OpenerDirector to be used for requests. There is a + fair amount of custom urllib code in Package Control, and part of it + is to handle proxies and keep-alives. Creating an opener the way + below is because the handlers have been customized to send the + "Connection: Keep-Alive" header and hold onto connections so they + can be re-used. + + :param url: + The URL to download + + :param timeout: + The int number of seconds to set the timeout to + """ + + if not self.opener: + http_proxy = self.settings.get('http_proxy') + https_proxy = self.settings.get('https_proxy') + if http_proxy or https_proxy: + proxies = {} + if http_proxy: + proxies['http'] = http_proxy + if https_proxy: + proxies['https'] = https_proxy + proxy_handler = ProxyHandler(proxies) + else: + proxy_handler = ProxyHandler() + + password_manager = HTTPPasswordMgrWithDefaultRealm() + proxy_username = self.settings.get('proxy_username') + proxy_password = self.settings.get('proxy_password') + if proxy_username and proxy_password: + if http_proxy: + password_manager.add_password(None, http_proxy, proxy_username, + proxy_password) + if https_proxy: + password_manager.add_password(None, https_proxy, proxy_username, + proxy_password) + + handlers = [proxy_handler] + + basic_auth_handler = ProxyBasicAuthHandler(password_manager) + digest_auth_handler = ProxyDigestAuthHandler(password_manager) + handlers.extend([digest_auth_handler, basic_auth_handler]) + + debug = self.settings.get('debug') + + if debug: + console_write(u"Urllib Debug Proxy", True) + console_write(u" http_proxy: %s" % http_proxy) + console_write(u" https_proxy: %s" % https_proxy) + console_write(u" proxy_username: %s" % proxy_username) + console_write(u" proxy_password: %s" % proxy_password) + + secure_url_match = re.match('^https://([^/]+)', url) + if secure_url_match != None: + secure_domain = secure_url_match.group(1) + bundle_path = self.check_certs(secure_domain, timeout) + bundle_path = bundle_path.encode(sys.getfilesystemencoding()) + handlers.append(ValidatingHTTPSHandler(ca_certs=bundle_path, + debug=debug, passwd=password_manager, + user_agent=self.settings.get('user_agent'))) + else: + handlers.append(DebuggableHTTPHandler(debug=debug, + passwd=password_manager)) + self.opener = build_opener(*handlers) + + def supports_ssl(self): + """ + Indicates if the object can handle HTTPS requests + + :return: + If the object supports HTTPS requests + """ + return 'ssl' in sys.modules and hasattr(urllib_compat, 'HTTPSHandler') diff --git a/sublime/Packages/Package Control/package_control/downloaders/wget_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/wget_downloader.py new file mode 100644 index 0000000..fb83d1b --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/wget_downloader.py @@ -0,0 +1,347 @@ +import tempfile +import re +import os + +from ..console_write import console_write +from ..unicode import unicode_from_os +from ..open_compat import open_compat, read_compat +from .cli_downloader import CliDownloader +from .non_http_error import NonHttpError +from .non_clean_exit_error import NonCleanExitError +from .rate_limit_exception import RateLimitException +from .downloader_exception import DownloaderException +from .cert_provider import CertProvider +from .decoding_downloader import DecodingDownloader +from .limiting_downloader import LimitingDownloader +from .caching_downloader import CachingDownloader + + +class WgetDownloader(CliDownloader, CertProvider, DecodingDownloader, LimitingDownloader, CachingDownloader): + """ + A downloader that uses the command line program wget + + :param settings: + A dict of the various Package Control settings. The Sublime Text + Settings API is not used because this code is run in a thread. + + :raises: + BinaryNotFoundError: when wget can not be found + """ + + def __init__(self, settings): + self.settings = settings + self.debug = settings.get('debug') + self.wget = self.find_binary('wget') + + def close(self): + """ + No-op for compatibility with UrllibDownloader and WinINetDownloader + """ + + pass + + def download(self, url, error_message, timeout, tries, prefer_cached=False): + """ + Downloads a URL and returns the contents + + :param url: + The URL to download + + :param error_message: + A string to include in the console error that is printed + when an error occurs + + :param timeout: + The int number of seconds to set the timeout to + + :param tries: + The int number of times to try and download the URL in the case of + a timeout or HTTP 503 error + + :param prefer_cached: + If a cached version should be returned instead of trying a new request + + :raises: + NoCaCertException: when no CA certs can be found for the url + RateLimitException: when a rate limit is hit + DownloaderException: when any other download error occurs + + :return: + The string contents of the URL + """ + + if prefer_cached: + cached = self.retrieve_cached(url) + if cached: + return cached + + self.tmp_file = tempfile.NamedTemporaryFile().name + command = [self.wget, '--connect-timeout=' + str(int(timeout)), '-o', + self.tmp_file, '-O', '-', '-U', self.settings.get('user_agent')] + + request_headers = { + # Don't be alarmed if the response from the server does not select + # one of these since the server runs a relatively new version of + # OpenSSL which supports compression on the SSL layer, and Apache + # will use that instead of HTTP-level encoding. + 'Accept-Encoding': 'gzip,deflate' + } + request_headers = self.add_conditional_headers(url, request_headers) + + for name, value in request_headers.items(): + command.extend(['--header', "%s: %s" % (name, value)]) + + secure_url_match = re.match('^https://([^/]+)', url) + if secure_url_match != None: + secure_domain = secure_url_match.group(1) + bundle_path = self.check_certs(secure_domain, timeout) + command.append(u'--ca-certificate=' + bundle_path) + + if self.debug: + command.append('-d') + else: + command.append('-S') + + http_proxy = self.settings.get('http_proxy') + https_proxy = self.settings.get('https_proxy') + proxy_username = self.settings.get('proxy_username') + proxy_password = self.settings.get('proxy_password') + + if proxy_username: + command.append(u"--proxy-user=%s" % proxy_username) + if proxy_password: + command.append(u"--proxy-password=%s" % proxy_password) + + if self.debug: + console_write(u"Wget Debug Proxy", True) + console_write(u" http_proxy: %s" % http_proxy) + console_write(u" https_proxy: %s" % https_proxy) + console_write(u" proxy_username: %s" % proxy_username) + console_write(u" proxy_password: %s" % proxy_password) + + command.append(url) + + if http_proxy: + os.putenv('http_proxy', http_proxy) + if https_proxy: + os.putenv('https_proxy', https_proxy) + + error_string = None + while tries > 0: + tries -= 1 + try: + result = self.execute(command) + + general, headers = self.parse_output() + encoding = headers.get('content-encoding') + if encoding: + result = self.decode_response(encoding, result) + + result = self.cache_result('get', url, general['status'], + headers, result) + + return result + + except (NonCleanExitError) as e: + + try: + general, headers = self.parse_output() + self.handle_rate_limit(headers, url) + + if general['status'] == 304: + return self.cache_result('get', url, general['status'], + headers, None) + + if general['status'] == 503 and tries != 0: + # GitHub and BitBucket seem to rate limit via 503 + error_string = u'Downloading %s was rate limited' % url + if tries: + error_string += ', trying again' + if self.debug: + console_write(error_string, True) + continue + + download_error = 'HTTP error %s' % general['status'] + + except (NonHttpError) as e: + + download_error = unicode_from_os(e) + + # GitHub and BitBucket seem to time out a lot + if download_error.find('timed out') != -1: + error_string = u'Downloading %s timed out' % url + if tries: + error_string += ', trying again' + if self.debug: + console_write(error_string, True) + continue + + error_string = u'%s %s downloading %s.' % (error_message, download_error, url) + + break + + raise DownloaderException(error_string) + + def supports_ssl(self): + """ + Indicates if the object can handle HTTPS requests + + :return: + If the object supports HTTPS requests + """ + + return True + + def parse_output(self): + """ + Parses the wget output file, prints debug information and returns headers + + :return: + A tuple of (general, headers) where general is a dict with the keys: + `version` - HTTP version number (string) + `status` - HTTP status code (integer) + `message` - HTTP status message (string) + And headers is a dict with the keys being lower-case version of the + HTTP header names. + """ + + with open_compat(self.tmp_file, 'r') as f: + output = read_compat(f).splitlines() + self.clean_tmp_file() + + error = None + header_lines = [] + if self.debug: + section = 'General' + last_section = None + for line in output: + if section == 'General': + if self.skippable_line(line): + continue + + # Skip blank lines + if line.strip() == '': + continue + + # Error lines + if line[0:5] == 'wget:': + error = line[5:].strip() + if line[0:7] == 'failed:': + error = line[7:].strip() + + if line == '---request begin---': + section = 'Write' + continue + elif line == '---request end---': + section = 'General' + continue + elif line == '---response begin---': + section = 'Read' + continue + elif line == '---response end---': + section = 'General' + continue + + if section != last_section: + console_write(u"Wget HTTP Debug %s" % section, True) + + if section == 'Read': + header_lines.append(line) + + console_write(u' ' + line) + last_section = section + + else: + for line in output: + if self.skippable_line(line): + continue + + # Check the resolving and connecting to lines for errors + if re.match('(Resolving |Connecting to )', line): + failed_match = re.search(' failed: (.*)$', line) + if failed_match: + error = failed_match.group(1).strip() + + # Error lines + if line[0:5] == 'wget:': + error = line[5:].strip() + if line[0:7] == 'failed:': + error = line[7:].strip() + + if line[0:2] == ' ': + header_lines.append(line.lstrip()) + + if error: + raise NonHttpError(error) + + return self.parse_headers(header_lines) + + def skippable_line(self, line): + """ + Determines if a debug line is skippable - usually because of extraneous + or duplicate information. + + :param line: + The debug line to check + + :return: + True if the line is skippable, otherwise None + """ + + # Skip date lines + if re.match('--\d{4}-\d{2}-\d{2}', line): + return True + if re.match('\d{4}-\d{2}-\d{2}', line): + return True + # Skip HTTP status code lines since we already have that info + if re.match('\d{3} ', line): + return True + # Skip Saving to and progress lines + if re.match('(Saving to:|\s*\d+K)', line): + return True + # Skip notice about ignoring body on HTTP error + if re.match('Skipping \d+ byte', line): + return True + + def parse_headers(self, output=None): + """ + Parses HTTP headers into two dict objects + + :param output: + An array of header lines, if None, loads from temp output file + + :return: + A tuple of (general, headers) where general is a dict with the keys: + `version` - HTTP version number (string) + `status` - HTTP status code (integer) + `message` - HTTP status message (string) + And headers is a dict with the keys being lower-case version of the + HTTP header names. + """ + + if not output: + with open_compat(self.tmp_file, 'r') as f: + output = read_compat(f).splitlines() + self.clean_tmp_file() + + general = { + 'version': '0.9', + 'status': 200, + 'message': 'OK' + } + headers = {} + for line in output: + # When using the -S option, headers have two spaces before them, + # additionally, valid headers won't have spaces, so this is always + # a safe operation to perform + line = line.lstrip() + if line.find('HTTP/') == 0: + match = re.match('HTTP/(\d\.\d)\s+(\d+)(?:\s+(.*))?$', line) + general['version'] = match.group(1) + general['status'] = int(match.group(2)) + general['message'] = match.group(3) or '' + else: + name, value = line.split(':', 1) + headers[name.lower()] = value.strip() + + return (general, headers) diff --git a/sublime/Packages/Package Control/package_control/downloaders/wininet_downloader.py b/sublime/Packages/Package Control/package_control/downloaders/wininet_downloader.py new file mode 100644 index 0000000..7134db9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/downloaders/wininet_downloader.py @@ -0,0 +1,652 @@ +from ctypes import windll, wintypes +import ctypes +import time +import re +import datetime +import struct +import locale + +wininet = windll.wininet + +try: + # Python 3 + from urllib.parse import urlparse +except (ImportError): + # Python 2 + from urlparse import urlparse + +from ..console_write import console_write +from ..unicode import unicode_from_os +from .non_http_error import NonHttpError +from .http_error import HttpError +from .rate_limit_exception import RateLimitException +from .downloader_exception import DownloaderException +from .decoding_downloader import DecodingDownloader +from .limiting_downloader import LimitingDownloader +from .caching_downloader import CachingDownloader + + +class WinINetDownloader(DecodingDownloader, LimitingDownloader, CachingDownloader): + """ + A downloader that uses the Windows WinINet DLL to perform downloads. This + has the benefit of utilizing system-level proxy configuration and CA certs. + + :param settings: + A dict of the various Package Control settings. The Sublime Text + Settings API is not used because this code is run in a thread. + """ + + # General constants + ERROR_INSUFFICIENT_BUFFER = 122 + + # InternetOpen constants + INTERNET_OPEN_TYPE_PRECONFIG = 0 + + # InternetConnect constants + INTERNET_SERVICE_HTTP = 3 + INTERNET_FLAG_EXISTING_CONNECT = 0x20000000 + INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS = 0x00004000 + + # InternetSetOption constants + INTERNET_OPTION_CONNECT_TIMEOUT = 2 + INTERNET_OPTION_SEND_TIMEOUT = 5 + INTERNET_OPTION_RECEIVE_TIMEOUT = 6 + + # InternetQueryOption constants + INTERNET_OPTION_SECURITY_CERTIFICATE_STRUCT = 32 + INTERNET_OPTION_PROXY = 38 + INTERNET_OPTION_PROXY_USERNAME = 43 + INTERNET_OPTION_PROXY_PASSWORD = 44 + INTERNET_OPTION_CONNECTED_STATE = 50 + + # HttpOpenRequest constants + INTERNET_FLAG_KEEP_CONNECTION = 0x00400000 + INTERNET_FLAG_RELOAD = 0x80000000 + INTERNET_FLAG_NO_CACHE_WRITE = 0x04000000 + INTERNET_FLAG_PRAGMA_NOCACHE = 0x00000100 + INTERNET_FLAG_SECURE = 0x00800000 + + # HttpQueryInfo constants + HTTP_QUERY_RAW_HEADERS_CRLF = 22 + + # InternetConnectedState constants + INTERNET_STATE_CONNECTED = 1 + INTERNET_STATE_DISCONNECTED = 2 + INTERNET_STATE_DISCONNECTED_BY_USER = 0x10 + INTERNET_STATE_IDLE = 0x100 + INTERNET_STATE_BUSY = 0x200 + + + def __init__(self, settings): + self.settings = settings + self.debug = settings.get('debug') + self.network_connection = None + self.tcp_connection = None + self.use_count = 0 + self.hostname = None + self.port = None + self.scheme = None + self.was_offline = None + + def close(self): + """ + Closes any persistent/open connections + """ + + closed = False + changed_state_back = False + + if self.tcp_connection: + wininet.InternetCloseHandle(self.tcp_connection) + self.tcp_connection = None + closed = True + + if self.network_connection: + wininet.InternetCloseHandle(self.network_connection) + self.network_connection = None + closed = True + + if self.was_offline: + dw_connected_state = wintypes.DWORD(self.INTERNET_STATE_DISCONNECTED_BY_USER) + dw_flags = wintypes.DWORD(0) + connected_info = InternetConnectedInfo(dw_connected_state, dw_flags) + wininet.InternetSetOptionA(None, + self.INTERNET_OPTION_CONNECTED_STATE, ctypes.byref(connected_info), ctypes.sizeof(connected_info)) + changed_state_back = True + + if self.debug: + s = '' if self.use_count == 1 else 's' + console_write(u"WinINet %s Debug General" % self.scheme.upper(), True) + console_write(u" Closing connection to %s on port %s after %s request%s" % ( + self.hostname, self.port, self.use_count, s)) + if changed_state_back: + console_write(u" Changed Internet Explorer back to Work Offline") + + self.hostname = None + self.port = None + self.scheme = None + self.use_count = 0 + self.was_offline = None + + def download(self, url, error_message, timeout, tries, prefer_cached=False): + """ + Downloads a URL and returns the contents + + :param url: + The URL to download + + :param error_message: + A string to include in the console error that is printed + when an error occurs + + :param timeout: + The int number of seconds to set the timeout to + + :param tries: + The int number of times to try and download the URL in the case of + a timeout or HTTP 503 error + + :param prefer_cached: + If a cached version should be returned instead of trying a new request + + :raises: + RateLimitException: when a rate limit is hit + DownloaderException: when any other download error occurs + + :return: + The string contents of the URL + """ + + if prefer_cached: + cached = self.retrieve_cached(url) + if cached: + return cached + + url_info = urlparse(url) + + if not url_info.port: + port = 443 if url_info.scheme == 'https' else 80 + hostname = url_info.netloc + else: + port = url_info.port + hostname = url_info.hostname + + path = url_info.path + if url_info.params: + path += ';' + url_info.params + if url_info.query: + path += '?' + url_info.query + + request_headers = { + 'Accept-Encoding': 'gzip,deflate' + } + request_headers = self.add_conditional_headers(url, request_headers) + + created_connection = False + # If we switched Internet Explorer out of "Work Offline" mode + changed_to_online = False + + # If the user is requesting a connection to another server, close the connection + if (self.hostname and self.hostname != hostname) or (self.port and self.port != port): + self.close() + + # Reset the error info to a known clean state + ctypes.windll.kernel32.SetLastError(0) + + # Save the internet setup in the class for re-use + if not self.tcp_connection: + created_connection = True + + # Connect to the internet if necessary + state = self.read_option(None, self.INTERNET_OPTION_CONNECTED_STATE) + state = ord(state) + if state & self.INTERNET_STATE_DISCONNECTED or state & self.INTERNET_STATE_DISCONNECTED_BY_USER: + # Track the previous state so we can go back once complete + self.was_offline = True + + dw_connected_state = wintypes.DWORD(self.INTERNET_STATE_CONNECTED) + dw_flags = wintypes.DWORD(0) + connected_info = InternetConnectedInfo(dw_connected_state, dw_flags) + wininet.InternetSetOptionA(None, + self.INTERNET_OPTION_CONNECTED_STATE, ctypes.byref(connected_info), ctypes.sizeof(connected_info)) + changed_to_online = True + + self.network_connection = wininet.InternetOpenW(self.settings.get('user_agent'), + self.INTERNET_OPEN_TYPE_PRECONFIG, None, None, 0) + + if not self.network_connection: + error_string = u'%s %s during network phase of downloading %s.' % (error_message, self.extract_error(), url) + raise DownloaderException(error_string) + + win_timeout = wintypes.DWORD(int(timeout) * 1000) + # Apparently INTERNET_OPTION_CONNECT_TIMEOUT just doesn't work, leaving it in hoping they may fix in the future + wininet.InternetSetOptionA(self.network_connection, + self.INTERNET_OPTION_CONNECT_TIMEOUT, win_timeout, ctypes.sizeof(win_timeout)) + wininet.InternetSetOptionA(self.network_connection, + self.INTERNET_OPTION_SEND_TIMEOUT, win_timeout, ctypes.sizeof(win_timeout)) + wininet.InternetSetOptionA(self.network_connection, + self.INTERNET_OPTION_RECEIVE_TIMEOUT, win_timeout, ctypes.sizeof(win_timeout)) + + # Don't allow HTTPS sites to redirect to HTTP sites + tcp_flags = self.INTERNET_FLAG_IGNORE_REDIRECT_TO_HTTPS + # Try to re-use an existing connection to the server + tcp_flags |= self.INTERNET_FLAG_EXISTING_CONNECT + self.tcp_connection = wininet.InternetConnectW(self.network_connection, + hostname, port, None, None, self.INTERNET_SERVICE_HTTP, tcp_flags, 0) + + if not self.tcp_connection: + error_string = u'%s %s during connection phase of downloading %s.' % (error_message, self.extract_error(), url) + raise DownloaderException(error_string) + + # Normally the proxy info would come from IE, but this allows storing it in + # the Package Control settings file. + proxy_username = self.settings.get('proxy_username') + proxy_password = self.settings.get('proxy_password') + if proxy_username and proxy_password: + username = ctypes.c_wchar_p(proxy_username) + password = ctypes.c_wchar_p(proxy_password) + wininet.InternetSetOptionW(self.tcp_connection, + self.INTERNET_OPTION_PROXY_USERNAME, ctypes.cast(username, ctypes.c_void_p), len(proxy_username)) + wininet.InternetSetOptionW(self.tcp_connection, + self.INTERNET_OPTION_PROXY_PASSWORD, ctypes.cast(password, ctypes.c_void_p), len(proxy_password)) + + self.hostname = hostname + self.port = port + self.scheme = url_info.scheme + + else: + if self.debug: + console_write(u"WinINet %s Debug General" % self.scheme.upper(), True) + console_write(u" Re-using connection to %s on port %s for request #%s" % ( + self.hostname, self.port, self.use_count)) + + error_string = None + while tries > 0: + tries -= 1 + try: + http_connection = None + + # Keep-alive for better performance + http_flags = self.INTERNET_FLAG_KEEP_CONNECTION + # Prevent caching/retrieving from cache + http_flags |= self.INTERNET_FLAG_RELOAD + http_flags |= self.INTERNET_FLAG_NO_CACHE_WRITE + http_flags |= self.INTERNET_FLAG_PRAGMA_NOCACHE + # Use SSL + if self.scheme == 'https': + http_flags |= self.INTERNET_FLAG_SECURE + + http_connection = wininet.HttpOpenRequestW(self.tcp_connection, u'GET', path, u'HTTP/1.1', None, None, http_flags, 0) + if not http_connection: + error_string = u'%s %s during HTTP connection phase of downloading %s.' % (error_message, self.extract_error(), url) + raise DownloaderException(error_string) + + request_header_lines = [] + for header, value in request_headers.items(): + request_header_lines.append(u"%s: %s" % (header, value)) + request_header_lines = u"\r\n".join(request_header_lines) + + success = wininet.HttpSendRequestW(http_connection, request_header_lines, len(request_header_lines), None, 0) + + if not success: + error_string = u'%s %s during HTTP write phase of downloading %s.' % (error_message, self.extract_error(), url) + raise DownloaderException(error_string) + + # If we try to query before here, the proxy info will not be available to the first request + if self.debug: + proxy_struct = self.read_option(self.network_connection, self.INTERNET_OPTION_PROXY) + proxy = '' + if proxy_struct.lpszProxy: + proxy = proxy_struct.lpszProxy.decode('cp1252') + proxy_bypass = '' + if proxy_struct.lpszProxyBypass: + proxy_bypass = proxy_struct.lpszProxyBypass.decode('cp1252') + + proxy_username = self.read_option(self.tcp_connection, self.INTERNET_OPTION_PROXY_USERNAME) + proxy_password = self.read_option(self.tcp_connection, self.INTERNET_OPTION_PROXY_PASSWORD) + + console_write(u"WinINet Debug Proxy", True) + console_write(u" proxy: %s" % proxy) + console_write(u" proxy bypass: %s" % proxy_bypass) + console_write(u" proxy username: %s" % proxy_username) + console_write(u" proxy password: %s" % proxy_password) + + self.use_count += 1 + + if self.debug and created_connection: + if self.scheme == 'https': + cert_struct = self.read_option(http_connection, self.INTERNET_OPTION_SECURITY_CERTIFICATE_STRUCT) + + if cert_struct.lpszIssuerInfo: + issuer_info = cert_struct.lpszIssuerInfo.decode('cp1252') + issuer_parts = issuer_info.split("\r\n") + else: + issuer_parts = ['No issuer info'] + + if cert_struct.lpszSubjectInfo: + subject_info = cert_struct.lpszSubjectInfo.decode('cp1252') + subject_parts = subject_info.split("\r\n") + else: + subject_parts = ["No subject info"] + + common_name = subject_parts[-1] + + if cert_struct.ftStart.dwLowDateTime != 0 and cert_struct.ftStart.dwHighDateTime != 0: + issue_date = self.convert_filetime_to_datetime(cert_struct.ftStart) + issue_date = issue_date.strftime('%a, %d %b %Y %H:%M:%S GMT') + else: + issue_date = u"No issue date" + + if cert_struct.ftExpiry.dwLowDateTime != 0 and cert_struct.ftExpiry.dwHighDateTime != 0: + expiration_date = self.convert_filetime_to_datetime(cert_struct.ftExpiry) + expiration_date = expiration_date.strftime('%a, %d %b %Y %H:%M:%S GMT') + else: + expiration_date = u"No expiration date" + + console_write(u"WinINet HTTPS Debug General", True) + if changed_to_online: + console_write(u" Internet Explorer was set to Work Offline, temporarily going online") + console_write(u" Server SSL Certificate:") + console_write(u" subject: %s" % ", ".join(subject_parts)) + console_write(u" issuer: %s" % ", ".join(issuer_parts)) + console_write(u" common name: %s" % common_name) + console_write(u" issue date: %s" % issue_date) + console_write(u" expire date: %s" % expiration_date) + + elif changed_to_online: + console_write(u"WinINet HTTP Debug General", True) + console_write(u" Internet Explorer was set to Work Offline, temporarily going online") + + if self.debug: + console_write(u"WinINet %s Debug Write" % self.scheme.upper(), True) + # Add in some known headers that WinINet sends since we can't get the real list + console_write(u" GET %s HTTP/1.1" % path) + for header, value in request_headers.items(): + console_write(u" %s: %s" % (header, value)) + console_write(u" User-Agent: %s" % self.settings.get('user_agent')) + console_write(u" Host: %s" % hostname) + console_write(u" Connection: Keep-Alive") + console_write(u" Cache-Control: no-cache") + + header_buffer_size = 8192 + + try_again = True + while try_again: + try_again = False + + to_read_was_read = wintypes.DWORD(header_buffer_size) + headers_buffer = ctypes.create_string_buffer(header_buffer_size) + + success = wininet.HttpQueryInfoA(http_connection, self.HTTP_QUERY_RAW_HEADERS_CRLF, ctypes.byref(headers_buffer), ctypes.byref(to_read_was_read), None) + if not success: + if ctypes.GetLastError() != self.ERROR_INSUFFICIENT_BUFFER: + error_string = u'%s %s during header read phase of downloading %s.' % (error_message, self.extract_error(), url) + raise DownloaderException(error_string) + # The error was a buffer that was too small, so try again + header_buffer_size = to_read_was_read.value + try_again = True + continue + + headers = b'' + if to_read_was_read.value > 0: + headers += headers_buffer.raw[:to_read_was_read.value] + headers = headers.decode('iso-8859-1').rstrip("\r\n").split("\r\n") + + if self.debug: + console_write(u"WinINet %s Debug Read" % self.scheme.upper(), True) + for header in headers: + console_write(u" %s" % header) + + buffer_length = 65536 + output_buffer = ctypes.create_string_buffer(buffer_length) + bytes_read = wintypes.DWORD() + + result = b'' + try_again = True + while try_again: + try_again = False + wininet.InternetReadFile(http_connection, output_buffer, buffer_length, ctypes.byref(bytes_read)) + if bytes_read.value > 0: + result += output_buffer.raw[:bytes_read.value] + try_again = True + + general, headers = self.parse_headers(headers) + self.handle_rate_limit(headers, url) + + if general['status'] == 503 and tries != 0: + # GitHub and BitBucket seem to rate limit via 503 + error_string = u'Downloading %s was rate limited' % url + if tries: + error_string += ', trying again' + if self.debug: + console_write(error_string, True) + continue + + encoding = headers.get('content-encoding') + if encoding: + result = self.decode_response(encoding, result) + + result = self.cache_result('get', url, general['status'], + headers, result) + + if general['status'] not in [200, 304]: + raise HttpError("HTTP error %s" % general['status'], general['status']) + + return result + + except (NonHttpError, HttpError) as e: + + # GitHub and BitBucket seem to time out a lot + if str(e).find('timed out') != -1: + error_string = u'Downloading %s timed out' % url + if tries: + error_string += ', trying again' + if self.debug: + console_write(error_string, True) + continue + + error_string = u'%s %s downloading %s.' % (error_message, e, url) + + finally: + if http_connection: + wininet.InternetCloseHandle(http_connection) + + break + + raise DownloaderException(error_string) + + def convert_filetime_to_datetime(self, filetime): + """ + Windows returns times as 64-bit unsigned longs that are the number + of hundreds of nanoseconds since Jan 1 1601. This converts it to + a datetime object. + + :param filetime: + A FileTime struct object + + :return: + A (UTC) datetime object + """ + + hundreds_nano_seconds = struct.unpack('>Q', struct.pack('>LL', filetime.dwHighDateTime, filetime.dwLowDateTime))[0] + seconds_since_1601 = hundreds_nano_seconds / 10000000 + epoch_seconds = seconds_since_1601 - 11644473600 # Seconds from Jan 1 1601 to Jan 1 1970 + return datetime.datetime.fromtimestamp(epoch_seconds) + + def extract_error(self): + """ + Retrieves and formats an error from WinINet + + :return: + A string with a nice description of the error + """ + + error_num = ctypes.GetLastError() + raw_error_string = ctypes.FormatError(error_num) + + error_string = unicode_from_os(raw_error_string) + + # Try to fill in some known errors + if error_string == u"": + error_lookup = { + 12007: u'host not found', + 12029: u'connection refused', + 12057: u'error checking for server certificate revocation', + 12169: u'invalid secure certificate', + 12157: u'secure channel error, server not providing SSL', + 12002: u'operation timed out' + } + if error_num in error_lookup: + error_string = error_lookup[error_num] + + if error_string == u"": + return u"(errno %s)" % error_num + + error_string = error_string[0].upper() + error_string[1:] + return u"%s (errno %s)" % (error_string, error_num) + + def supports_ssl(self): + """ + Indicates if the object can handle HTTPS requests + + :return: + If the object supports HTTPS requests + """ + + return True + + def read_option(self, handle, option): + """ + Reads information about the internet connection, which may be a string or struct + + :param handle: + The handle to query for the info + + :param option: + The (int) option to get + + :return: + A string, or one of the InternetCertificateInfo or InternetProxyInfo structs + """ + + option_buffer_size = 8192 + try_again = True + + while try_again: + try_again = False + + to_read_was_read = wintypes.DWORD(option_buffer_size) + option_buffer = ctypes.create_string_buffer(option_buffer_size) + ref = ctypes.byref(option_buffer) + + success = wininet.InternetQueryOptionA(handle, option, ref, ctypes.byref(to_read_was_read)) + if not success: + if ctypes.GetLastError() != self.ERROR_INSUFFICIENT_BUFFER: + raise NonHttpError(self.extract_error()) + # The error was a buffer that was too small, so try again + option_buffer_size = to_read_was_read.value + try_again = True + continue + + if option == self.INTERNET_OPTION_SECURITY_CERTIFICATE_STRUCT: + length = min(len(option_buffer), ctypes.sizeof(InternetCertificateInfo)) + cert_info = InternetCertificateInfo() + ctypes.memmove(ctypes.addressof(cert_info), option_buffer, length) + return cert_info + elif option == self.INTERNET_OPTION_PROXY: + length = min(len(option_buffer), ctypes.sizeof(InternetProxyInfo)) + proxy_info = InternetProxyInfo() + ctypes.memmove(ctypes.addressof(proxy_info), option_buffer, length) + return proxy_info + else: + option = b'' + if to_read_was_read.value > 0: + option += option_buffer.raw[:to_read_was_read.value] + return option.decode('cp1252').rstrip("\x00") + + def parse_headers(self, output): + """ + Parses HTTP headers into two dict objects + + :param output: + An array of header lines + + :return: + A tuple of (general, headers) where general is a dict with the keys: + `version` - HTTP version number (string) + `status` - HTTP status code (integer) + `message` - HTTP status message (string) + And headers is a dict with the keys being lower-case version of the + HTTP header names. + """ + + general = { + 'version': '0.9', + 'status': 200, + 'message': 'OK' + } + headers = {} + for line in output: + line = line.lstrip() + if line.find('HTTP/') == 0: + match = re.match('HTTP/(\d\.\d)\s+(\d+)\s+(.*)$', line) + general['version'] = match.group(1) + general['status'] = int(match.group(2)) + general['message'] = match.group(3) + else: + name, value = line.split(':', 1) + headers[name.lower()] = value.strip() + + return (general, headers) + + +class FileTime(ctypes.Structure): + """ + A Windows struct used by InternetCertificateInfo for certificate + date information + """ + + _fields_ = [ + ("dwLowDateTime", wintypes.DWORD), + ("dwHighDateTime", wintypes.DWORD) + ] + + +class InternetCertificateInfo(ctypes.Structure): + """ + A Windows struct used to store information about an SSL certificate + """ + + _fields_ = [ + ("ftExpiry", FileTime), + ("ftStart", FileTime), + ("lpszSubjectInfo", ctypes.c_char_p), + ("lpszIssuerInfo", ctypes.c_char_p), + ("lpszProtocolName", ctypes.c_char_p), + ("lpszSignatureAlgName", ctypes.c_char_p), + ("lpszEncryptionAlgName", ctypes.c_char_p), + ("dwKeySize", wintypes.DWORD) + ] + + +class InternetProxyInfo(ctypes.Structure): + """ + A Windows struct usd to store information about the configured proxy server + """ + + _fields_ = [ + ("dwAccessType", wintypes.DWORD), + ("lpszProxy", ctypes.c_char_p), + ("lpszProxyBypass", ctypes.c_char_p) + ] + + +class InternetConnectedInfo(ctypes.Structure): + """ + A Windows struct usd to store information about the global internet connection state + """ + + _fields_ = [ + ("dwConnectedState", wintypes.DWORD), + ("dwFlags", wintypes.DWORD) + ] diff --git a/sublime/Packages/Package Control/package_control/file_not_found_error.py b/sublime/Packages/Package Control/package_control/file_not_found_error.py new file mode 100644 index 0000000..3fd4da5 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/file_not_found_error.py @@ -0,0 +1,4 @@ +class FileNotFoundError(Exception): + """If a file is not found""" + + pass diff --git a/sublime/Packages/Package Control/package_control/http/__init__.py b/sublime/Packages/Package Control/package_control/http/__init__.py new file mode 100644 index 0000000..e3358df --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/__init__.py @@ -0,0 +1,65 @@ +import sys + +try: + # Python 2 + import urllib2 + import httplib + + # Monkey patch AbstractBasicAuthHandler to prevent infinite recursion + def non_recursive_http_error_auth_reqed(self, authreq, host, req, headers): + authreq = headers.get(authreq, None) + + if not hasattr(self, 'retried'): + self.retried = 0 + + if self.retried > 5: + raise urllib2.HTTPError(req.get_full_url(), 401, "basic auth failed", + headers, None) + else: + self.retried += 1 + + if authreq: + mo = urllib2.AbstractBasicAuthHandler.rx.search(authreq) + if mo: + scheme, quote, realm = mo.groups() + if scheme.lower() == 'basic': + return self.retry_http_basic_auth(host, req, realm) + + urllib2.AbstractBasicAuthHandler.http_error_auth_reqed = non_recursive_http_error_auth_reqed + + # Money patch urllib2.Request and httplib.HTTPConnection so that + # HTTPS proxies work in Python 2.6.1-2 + if sys.version_info < (2, 6, 3): + + urllib2.Request._tunnel_host = None + + def py268_set_proxy(self, host, type): + if self.type == 'https' and not self._tunnel_host: + self._tunnel_host = self.host + else: + self.type = type + # The _Request prefix is to handle python private name mangling + self._Request__r_host = self._Request__original + self.host = host + urllib2.Request.set_proxy = py268_set_proxy + + if sys.version_info < (2, 6, 5): + + def py268_set_tunnel(self, host, port=None, headers=None): + """ Sets up the host and the port for the HTTP CONNECT Tunnelling. + + The headers argument should be a mapping of extra HTTP headers + to send with the CONNECT request. + """ + self._tunnel_host = host + self._tunnel_port = port + if headers: + self._tunnel_headers = headers + else: + self._tunnel_headers.clear() + httplib.HTTPConnection._set_tunnel = py268_set_tunnel + + +except (ImportError): + # Python 3 does not need to be patched + pass diff --git a/sublime/Packages/Package Control/package_control/http/debuggable_http_connection.py b/sublime/Packages/Package Control/package_control/http/debuggable_http_connection.py new file mode 100644 index 0000000..e0044a9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/debuggable_http_connection.py @@ -0,0 +1,72 @@ +import os +import re +import socket + +try: + # Python 3 + from http.client import HTTPConnection + from urllib.error import URLError +except (ImportError): + # Python 2 + from httplib import HTTPConnection + from urllib2 import URLError + +from ..console_write import console_write +from .debuggable_http_response import DebuggableHTTPResponse + + +class DebuggableHTTPConnection(HTTPConnection): + """ + A custom HTTPConnection that formats debugging info for Sublime Text + """ + + response_class = DebuggableHTTPResponse + _debug_protocol = 'HTTP' + + def __init__(self, host, port=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + **kwargs): + self.passwd = kwargs.get('passwd') + + # Python 2.6.1 on OS X 10.6 does not include these + self._tunnel_host = None + self._tunnel_port = None + self._tunnel_headers = {} + if 'debug' in kwargs and kwargs['debug']: + self.debuglevel = 5 + elif 'debuglevel' in kwargs: + self.debuglevel = kwargs['debuglevel'] + + HTTPConnection.__init__(self, host, port=port, timeout=timeout) + + def connect(self): + if self.debuglevel == -1: + console_write(u'Urllib %s Debug General' % self._debug_protocol, True) + console_write(u" Connecting to %s on port %s" % (self.host, self.port)) + HTTPConnection.connect(self) + + def send(self, string): + # We have to use a positive debuglevel to get it passed to the + # HTTPResponse object, however we don't want to use it because by + # default debugging prints to the stdout and we can't capture it, so + # we temporarily set it to -1 for the standard httplib code + reset_debug = False + if self.debuglevel == 5: + reset_debug = 5 + self.debuglevel = -1 + HTTPConnection.send(self, string) + if reset_debug or self.debuglevel == -1: + if len(string.strip()) > 0: + console_write(u'Urllib %s Debug Write' % self._debug_protocol, True) + for line in string.strip().splitlines(): + console_write(u' ' + line.decode('iso-8859-1')) + if reset_debug: + self.debuglevel = reset_debug + + def request(self, method, url, body=None, headers={}): + original_headers = headers.copy() + + # By default urllib2 and urllib.request override the Connection header, + # however, it is preferred to be able to re-use it + original_headers['Connection'] = 'Keep-Alive' + + HTTPConnection.request(self, method, url, body, original_headers) diff --git a/sublime/Packages/Package Control/package_control/http/debuggable_http_handler.py b/sublime/Packages/Package Control/package_control/http/debuggable_http_handler.py new file mode 100644 index 0000000..ae4b8d1 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/debuggable_http_handler.py @@ -0,0 +1,35 @@ +import sys + +try: + # Python 3 + from urllib.request import HTTPHandler +except (ImportError): + # Python 2 + from urllib2 import HTTPHandler + +from .debuggable_http_connection import DebuggableHTTPConnection +from .persistent_handler import PersistentHandler + + +class DebuggableHTTPHandler(PersistentHandler, HTTPHandler): + """ + A custom HTTPHandler that formats debugging info for Sublime Text + """ + + def __init__(self, debuglevel=0, debug=False, **kwargs): + # This is a special value that will not trigger the standard debug + # functionality, but custom code where we can format the output + if debug: + self._debuglevel = 5 + else: + self._debuglevel = debuglevel + self.passwd = kwargs.get('passwd') + + def http_open(self, req): + def http_class_wrapper(host, **kwargs): + kwargs['passwd'] = self.passwd + if 'debuglevel' not in kwargs: + kwargs['debuglevel'] = self._debuglevel + return DebuggableHTTPConnection(host, **kwargs) + + return self.do_open(http_class_wrapper, req) diff --git a/sublime/Packages/Package Control/package_control/http/debuggable_http_response.py b/sublime/Packages/Package Control/package_control/http/debuggable_http_response.py new file mode 100644 index 0000000..2dd3af6 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/debuggable_http_response.py @@ -0,0 +1,66 @@ +try: + # Python 3 + from http.client import HTTPResponse, IncompleteRead +except (ImportError): + # Python 2 + from httplib import HTTPResponse, IncompleteRead + +from ..console_write import console_write + + +class DebuggableHTTPResponse(HTTPResponse): + """ + A custom HTTPResponse that formats debugging info for Sublime Text + """ + + _debug_protocol = 'HTTP' + + def __init__(self, sock, debuglevel=0, method=None, **kwargs): + # We have to use a positive debuglevel to get it passed to here, + # however we don't want to use it because by default debugging prints + # to the stdout and we can't capture it, so we use a special -1 value + if debuglevel == 5: + debuglevel = -1 + HTTPResponse.__init__(self, sock, debuglevel=debuglevel, method=method) + + def begin(self): + return_value = HTTPResponse.begin(self) + if self.debuglevel == -1: + console_write(u'Urllib %s Debug Read' % self._debug_protocol, True) + + # Python 2 + if hasattr(self.msg, 'headers'): + headers = self.msg.headers + # Python 3 + else: + headers = [] + for header in self.msg: + headers.append("%s: %s" % (header, self.msg[header])) + + versions = { + 9: 'HTTP/0.9', + 10: 'HTTP/1.0', + 11: 'HTTP/1.1' + } + status_line = versions[self.version] + ' ' + str(self.status) + ' ' + self.reason + headers.insert(0, status_line) + for line in headers: + console_write(u" %s" % line.rstrip()) + return return_value + + def is_keep_alive(self): + # Python 2 + if hasattr(self.msg, 'headers'): + connection = self.msg.getheader('connection') + # Python 3 + else: + connection = self.msg['connection'] + if connection and connection.lower() == 'keep-alive': + return True + return False + + def read(self, *args): + try: + return HTTPResponse.read(self, *args) + except (IncompleteRead) as e: + return e.partial diff --git a/sublime/Packages/Package Control/package_control/http/debuggable_https_response.py b/sublime/Packages/Package Control/package_control/http/debuggable_https_response.py new file mode 100644 index 0000000..edc9fb0 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/debuggable_https_response.py @@ -0,0 +1,9 @@ +from .debuggable_http_response import DebuggableHTTPResponse + + +class DebuggableHTTPSResponse(DebuggableHTTPResponse): + """ + A version of DebuggableHTTPResponse that sets the debug protocol to HTTPS + """ + + _debug_protocol = 'HTTPS' diff --git a/sublime/Packages/Package Control/package_control/http/invalid_certificate_exception.py b/sublime/Packages/Package Control/package_control/http/invalid_certificate_exception.py new file mode 100644 index 0000000..2715707 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/invalid_certificate_exception.py @@ -0,0 +1,25 @@ +try: + # Python 3 + from http.client import HTTPException + from urllib.error import URLError +except (ImportError): + # Python 2 + from httplib import HTTPException + from urllib2 import URLError + + +class InvalidCertificateException(HTTPException, URLError): + """ + An exception for when an SSL certification is not valid for the URL + it was presented for. + """ + + def __init__(self, host, cert, reason): + HTTPException.__init__(self) + self.host = host + self.cert = cert + self.reason = reason + + def __str__(self): + return ('Host %s returned an invalid certificate (%s) %s\n' % + (self.host, self.reason, self.cert)) diff --git a/sublime/Packages/Package Control/package_control/http/persistent_handler.py b/sublime/Packages/Package Control/package_control/http/persistent_handler.py new file mode 100644 index 0000000..4bfd3d7 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/persistent_handler.py @@ -0,0 +1,116 @@ +import sys +import socket + +try: + # Python 3 + from urllib.error import URLError +except ImportError: + # Python 2 + from urllib2 import URLError + from urllib import addinfourl + +from ..console_write import console_write + + +class PersistentHandler: + connection = None + use_count = 0 + + def close(self): + if self.connection: + if self._debuglevel == 5: + s = '' if self.use_count == 1 else 's' + console_write(u"Urllib %s Debug General" % self.connection._debug_protocol, True) + console_write(u" Closing connection to %s on port %s after %s request%s" % ( + self.connection.host, self.connection.port, self.use_count, s)) + self.connection.close() + self.connection = None + self.use_count = 0 + + def do_open(self, http_class, req): + # Large portions from Python 3.3 Lib/urllib/request.py and + # Python 2.6 Lib/urllib2.py + + if sys.version_info >= (3,): + host = req.host + else: + host = req.get_host() + + if not host: + raise URLError('no host given') + + if self.connection and self.connection.host != host: + self.close() + + # Re-use the connection if possible + self.use_count += 1 + if not self.connection: + h = http_class(host, timeout=req.timeout) + else: + h = self.connection + if self._debuglevel == 5: + console_write(u"Urllib %s Debug General" % h._debug_protocol, True) + console_write(u" Re-using connection to %s on port %s for request #%s" % ( + h.host, h.port, self.use_count)) + + if sys.version_info >= (3,): + headers = dict(req.unredirected_hdrs) + headers.update(dict((k, v) for k, v in req.headers.items() + if k not in headers)) + headers = dict((name.title(), val) for name, val in headers.items()) + + else: + h.set_debuglevel(self._debuglevel) + + headers = dict(req.headers) + headers.update(req.unredirected_hdrs) + headers = dict( + (name.title(), val) for name, val in headers.items()) + + if req._tunnel_host and not self.connection: + tunnel_headers = {} + proxy_auth_hdr = "Proxy-Authorization" + if proxy_auth_hdr in headers: + tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] + del headers[proxy_auth_hdr] + + if sys.version_info >= (3,): + h.set_tunnel(req._tunnel_host, headers=tunnel_headers) + else: + h._set_tunnel(req._tunnel_host, headers=tunnel_headers) + + try: + if sys.version_info >= (3,): + h.request(req.get_method(), req.selector, req.data, headers) + else: + h.request(req.get_method(), req.get_selector(), req.data, headers) + except socket.error as err: # timeout error + h.close() + raise URLError(err) + else: + r = h.getresponse() + + # Keep the connection around for re-use + if r.is_keep_alive(): + self.connection = h + else: + if self._debuglevel == 5: + s = '' if self.use_count == 1 else 's' + console_write(u"Urllib %s Debug General" % h._debug_protocol, True) + console_write(u" Closing connection to %s on port %s after %s request%s" % ( + h.host, h.port, self.use_count, s)) + self.use_count = 0 + self.connection = None + + if sys.version_info >= (3,): + r.url = req.get_full_url() + r.msg = r.reason + return r + + r.recv = r.read + fp = socket._fileobject(r, close=True) + + resp = addinfourl(fp, r.msg, req.get_full_url()) + resp.code = r.status + resp.msg = r.reason + return resp diff --git a/sublime/Packages/Package Control/package_control/http/validating_https_connection.py b/sublime/Packages/Package Control/package_control/http/validating_https_connection.py new file mode 100644 index 0000000..a01afdb --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/validating_https_connection.py @@ -0,0 +1,345 @@ +import re +import socket +import base64 +import hashlib +import os +import sys + +try: + # Python 3 + from http.client import HTTPS_PORT + from urllib.request import parse_keqv_list, parse_http_list +except (ImportError): + # Python 2 + from httplib import HTTPS_PORT + from urllib2 import parse_keqv_list, parse_http_list + +from ..console_write import console_write +from .debuggable_https_response import DebuggableHTTPSResponse +from .debuggable_http_connection import DebuggableHTTPConnection +from .invalid_certificate_exception import InvalidCertificateException + + +# The following code is wrapped in a try because the Linux versions of Sublime +# Text do not include the ssl module due to the fact that different distros +# have different versions +try: + import ssl + + class ValidatingHTTPSConnection(DebuggableHTTPConnection): + """ + A custom HTTPConnection class that validates SSL certificates, and + allows proxy authentication for HTTPS connections. + """ + + default_port = HTTPS_PORT + + response_class = DebuggableHTTPSResponse + _debug_protocol = 'HTTPS' + + def __init__(self, host, port=None, key_file=None, cert_file=None, + ca_certs=None, **kwargs): + passed_args = {} + if 'timeout' in kwargs: + passed_args['timeout'] = kwargs['timeout'] + if 'debug' in kwargs: + passed_args['debug'] = kwargs['debug'] + DebuggableHTTPConnection.__init__(self, host, port, **passed_args) + + self.passwd = kwargs.get('passwd') + self.key_file = key_file + self.cert_file = cert_file + self.ca_certs = ca_certs + if 'user_agent' in kwargs: + self.user_agent = kwargs['user_agent'] + if self.ca_certs: + self.cert_reqs = ssl.CERT_REQUIRED + else: + self.cert_reqs = ssl.CERT_NONE + + def get_valid_hosts_for_cert(self, cert): + """ + Returns a list of valid hostnames for an SSL certificate + + :param cert: A dict from SSLSocket.getpeercert() + + :return: An array of hostnames + """ + + if 'subjectAltName' in cert: + return [x[1] for x in cert['subjectAltName'] + if x[0].lower() == 'dns'] + else: + return [x[0][1] for x in cert['subject'] + if x[0][0].lower() == 'commonname'] + + def validate_cert_host(self, cert, hostname): + """ + Checks if the cert is valid for the hostname + + :param cert: A dict from SSLSocket.getpeercert() + + :param hostname: A string hostname to check + + :return: A boolean if the cert is valid for the hostname + """ + + hosts = self.get_valid_hosts_for_cert(cert) + for host in hosts: + host_re = host.replace('.', '\.').replace('*', '[^.]*') + if re.search('^%s$' % (host_re,), hostname, re.I): + return True + return False + + def _tunnel(self): + """ + This custom _tunnel method allows us to read and print the debug + log for the whole response before throwing an error, and adds + support for proxy authentication + """ + + self._proxy_host = self.host + self._proxy_port = self.port + self._set_hostport(self._tunnel_host, self._tunnel_port) + + self._tunnel_headers['Host'] = u"%s:%s" % (self.host, self.port) + self._tunnel_headers['User-Agent'] = self.user_agent + self._tunnel_headers['Proxy-Connection'] = 'Keep-Alive' + + request = "CONNECT %s:%d HTTP/1.1\r\n" % (self.host, self.port) + for header, value in self._tunnel_headers.items(): + request += "%s: %s\r\n" % (header, value) + request += "\r\n" + + if sys.version_info >= (3,): + request = bytes(request, 'iso-8859-1') + + self.send(request) + + response = self.response_class(self.sock, method=self._method) + (version, code, message) = response._read_status() + + status_line = u"%s %s %s" % (version, code, message.rstrip()) + headers = [status_line] + + if self.debuglevel in [-1, 5]: + console_write(u'Urllib %s Debug Read' % self._debug_protocol, True) + console_write(u" %s" % status_line) + + content_length = 0 + close_connection = False + while True: + line = response.fp.readline() + + if sys.version_info >= (3,): + line = str(line, encoding='iso-8859-1') + + if line == '\r\n': + break + + headers.append(line.rstrip()) + + parts = line.rstrip().split(': ', 1) + name = parts[0].lower() + value = parts[1].lower().strip() + if name == 'content-length': + content_length = int(value) + + if name in ['connection', 'proxy-connection'] and value == 'close': + close_connection = True + + if self.debuglevel in [-1, 5]: + console_write(u" %s" % line.rstrip()) + + # Handle proxy auth for SSL connections since regular urllib punts on this + if code == 407 and self.passwd and 'Proxy-Authorization' not in self._tunnel_headers: + if content_length: + response._safe_read(content_length) + + supported_auth_methods = {} + for line in headers: + parts = line.split(': ', 1) + if parts[0].lower() != 'proxy-authenticate': + continue + details = parts[1].split(' ', 1) + supported_auth_methods[details[0].lower()] = details[1] if len(details) > 1 else '' + + username, password = self.passwd.find_user_password(None, "%s:%s" % ( + self._proxy_host, self._proxy_port)) + + if 'digest' in supported_auth_methods: + response_value = self.build_digest_response( + supported_auth_methods['digest'], username, password) + if response_value: + self._tunnel_headers['Proxy-Authorization'] = u"Digest %s" % response_value + + elif 'basic' in supported_auth_methods: + response_value = u"%s:%s" % (username, password) + response_value = base64.b64encode(response_value).strip() + self._tunnel_headers['Proxy-Authorization'] = u"Basic %s" % response_value + + if 'Proxy-Authorization' in self._tunnel_headers: + self.host = self._proxy_host + self.port = self._proxy_port + + # If the proxy wanted the connection closed, we need to make a new connection + if close_connection: + self.sock.close() + self.sock = socket.create_connection((self.host, self.port), self.timeout) + + return self._tunnel() + + if code != 200: + self.close() + raise socket.error("Tunnel connection failed: %d %s" % (code, + message.strip())) + + def build_digest_response(self, fields, username, password): + """ + Takes a Proxy-Authenticate: Digest header and creates a response + header + + :param fields: + The string portion of the Proxy-Authenticate header after + "Digest " + + :param username: + The username to use for the response + + :param password: + The password to use for the response + + :return: + None if invalid Proxy-Authenticate header, otherwise the + string of fields for the Proxy-Authorization: Digest header + """ + + fields = parse_keqv_list(parse_http_list(fields)) + + realm = fields.get('realm') + nonce = fields.get('nonce') + qop = fields.get('qop') + algorithm = fields.get('algorithm') + if algorithm: + algorithm = algorithm.lower() + opaque = fields.get('opaque') + + if algorithm in ['md5', None]: + def md5hash(string): + return hashlib.md5(string).hexdigest() + hash = md5hash + + elif algorithm == 'sha': + def sha1hash(string): + return hashlib.sha1(string).hexdigest() + hash = sha1hash + + else: + return None + + host_port = u"%s:%s" % (self.host, self.port) + + a1 = "%s:%s:%s" % (username, realm, password) + a2 = "CONNECT:%s" % host_port + ha1 = hash(a1) + ha2 = hash(a2) + + if qop == None: + response = hash(u"%s:%s:%s" % (ha1, nonce, ha2)) + elif qop == 'auth': + nc = '00000001' + cnonce = hash(os.urandom(8))[:8] + response = hash(u"%s:%s:%s:%s:%s:%s" % (ha1, nonce, nc, cnonce, qop, ha2)) + else: + return None + + response_fields = { + 'username': username, + 'realm': realm, + 'nonce': nonce, + 'response': response, + 'uri': host_port + } + if algorithm: + response_fields['algorithm'] = algorithm + if qop == 'auth': + response_fields['nc'] = nc + response_fields['cnonce'] = cnonce + response_fields['qop'] = qop + if opaque: + response_fields['opaque'] = opaque + + return ', '.join([u"%s=\"%s\"" % (field, response_fields[field]) for field in response_fields]) + + def connect(self): + """ + Adds debugging and SSL certification validation + """ + + if self.debuglevel == -1: + console_write(u"Urllib HTTPS Debug General", True) + console_write(u" Connecting to %s on port %s" % (self.host, self.port)) + + self.sock = socket.create_connection((self.host, self.port), self.timeout) + if self._tunnel_host: + self._tunnel() + + if self.debuglevel == -1: + console_write(u"Urllib HTTPS Debug General", True) + console_write(u" Connecting to %s on port %s" % (self.host, self.port)) + console_write(u" CA certs file at %s" % (self.ca_certs.decode(sys.getfilesystemencoding()))) + + self.sock = ssl.wrap_socket(self.sock, keyfile=self.key_file, + certfile=self.cert_file, cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs) + + if self.debuglevel == -1: + console_write(u" Successfully upgraded connection to %s:%s with SSL" % ( + self.host, self.port)) + + # This debugs and validates the SSL certificate + if self.cert_reqs & ssl.CERT_REQUIRED: + cert = self.sock.getpeercert() + + if self.debuglevel == -1: + subjectMap = { + 'organizationName': 'O', + 'commonName': 'CN', + 'organizationalUnitName': 'OU', + 'countryName': 'C', + 'serialNumber': 'serialNumber', + 'commonName': 'CN', + 'localityName': 'L', + 'stateOrProvinceName': 'S' + } + subject_list = list(cert['subject']) + subject_list.reverse() + subject_parts = [] + for pair in subject_list: + if pair[0][0] in subjectMap: + field_name = subjectMap[pair[0][0]] + else: + field_name = pair[0][0] + subject_parts.append(field_name + '=' + pair[0][1]) + + console_write(u" Server SSL certificate:") + console_write(u" subject: " + ','.join(subject_parts)) + if 'subjectAltName' in cert: + console_write(u" common name: " + cert['subjectAltName'][0][1]) + if 'notAfter' in cert: + console_write(u" expire date: " + cert['notAfter']) + + hostname = self.host.split(':', 0)[0] + + if not self.validate_cert_host(cert, hostname): + if self.debuglevel == -1: + console_write(u" Certificate INVALID") + + raise InvalidCertificateException(hostname, cert, + 'hostname mismatch') + + if self.debuglevel == -1: + console_write(u" Certificate validated for %s" % hostname) + +except (ImportError): + pass diff --git a/sublime/Packages/Package Control/package_control/http/validating_https_handler.py b/sublime/Packages/Package Control/package_control/http/validating_https_handler.py new file mode 100644 index 0000000..5b02c7a --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http/validating_https_handler.py @@ -0,0 +1,59 @@ +try: + # Python 3 + from urllib.error import URLError + import urllib.request as urllib_compat +except (ImportError): + # Python 2 + from urllib2 import URLError + import urllib2 as urllib_compat + + +# The following code is wrapped in a try because the Linux versions of Sublime +# Text do not include the ssl module due to the fact that different distros +# have different versions +try: + import ssl + + from .validating_https_connection import ValidatingHTTPSConnection + from .invalid_certificate_exception import InvalidCertificateException + from .persistent_handler import PersistentHandler + + if hasattr(urllib_compat, 'HTTPSHandler'): + class ValidatingHTTPSHandler(PersistentHandler, urllib_compat.HTTPSHandler): + """ + A urllib handler that validates SSL certificates for HTTPS requests + """ + + def __init__(self, **kwargs): + # This is a special value that will not trigger the standard debug + # functionality, but custom code where we can format the output + self._debuglevel = 0 + if 'debug' in kwargs and kwargs['debug']: + self._debuglevel = 5 + elif 'debuglevel' in kwargs: + self._debuglevel = kwargs['debuglevel'] + self._connection_args = kwargs + + def https_open(self, req): + def http_class_wrapper(host, **kwargs): + full_kwargs = dict(self._connection_args) + full_kwargs.update(kwargs) + return ValidatingHTTPSConnection(host, **full_kwargs) + + try: + return self.do_open(http_class_wrapper, req) + except URLError as e: + if type(e.reason) == ssl.SSLError and e.reason.args[0] == 1: + raise InvalidCertificateException(req.host, '', + e.reason.args[1]) + raise + + https_request = urllib_compat.AbstractHTTPHandler.do_request_ + else: + raise ImportError() + +except (ImportError) as e: + + class ValidatingHTTPSHandler(): + def __init__(self, **kwargs): + raise e diff --git a/sublime/Packages/Package Control/package_control/http_cache.py b/sublime/Packages/Package Control/package_control/http_cache.py new file mode 100644 index 0000000..2f6f3a2 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/http_cache.py @@ -0,0 +1,75 @@ +import os +import time + +import sublime + +from .open_compat import open_compat, read_compat + + +class HttpCache(object): + """ + A data store for caching HTTP response data. + """ + + def __init__(self, ttl): + self.base_path = os.path.join(sublime.packages_path(), 'User', 'Package Control.cache') + if not os.path.exists(self.base_path): + os.mkdir(self.base_path) + self.clear(int(ttl)) + + def clear(self, ttl): + """ + Removes all cache entries older than the TTL + + :param ttl: + The number of seconds a cache entry should be valid for + """ + + ttl = int(ttl) + + for filename in os.listdir(self.base_path): + path = os.path.join(self.base_path, filename) + # There should not be any folders in the cache dir, but we + # ignore to prevent an exception + if os.path.isdir(path): + continue + mtime = os.stat(path).st_mtime + if mtime < time.time() - ttl: + os.unlink(path) + + def get(self, key): + """ + Returns a cached value + + :param key: + The key to fetch the cache for + + :return: + The (binary) cached value, or False + """ + + cache_file = os.path.join(self.base_path, key) + if not os.path.exists(cache_file): + return False + + with open_compat(cache_file, 'rb') as f: + return read_compat(f) + + def has(self, key): + cache_file = os.path.join(self.base_path, key) + return os.path.exists(cache_file) + + def set(self, key, content): + """ + Saves a value in the cache + + :param key: + The key to save the cache with + + :param content: + The (binary) content to cache + """ + + cache_file = os.path.join(self.base_path, key) + with open_compat(cache_file, 'wb') as f: + f.write(content) diff --git a/sublime/Packages/Package Control/package_control/open_compat.py b/sublime/Packages/Package Control/package_control/open_compat.py new file mode 100644 index 0000000..b22f066 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/open_compat.py @@ -0,0 +1,27 @@ +import os +import sys + +from .file_not_found_error import FileNotFoundError + + +def open_compat(path, mode='r'): + if mode in ['r', 'rb'] and not os.path.exists(path): + raise FileNotFoundError(u"The file \"%s\" could not be found" % path) + + if sys.version_info >= (3,): + encoding = 'utf-8' + errors = 'replace' + if mode in ['rb', 'wb', 'ab']: + encoding = None + errors = None + return open(path, mode, encoding=encoding, errors=errors) + + else: + return open(path, mode) + + +def read_compat(file_obj): + if sys.version_info >= (3,): + return file_obj.read() + else: + return unicode(file_obj.read(), 'utf-8', errors='replace') diff --git a/sublime/Packages/Package Control/package_control/package_cleanup.py b/sublime/Packages/Package Control/package_control/package_cleanup.py new file mode 100644 index 0000000..352f4d4 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/package_cleanup.py @@ -0,0 +1,107 @@ +import threading +import os +import shutil + +import sublime + +from .show_error import show_error +from .console_write import console_write +from .unicode import unicode_from_os +from .clear_directory import clear_directory +from .automatic_upgrader import AutomaticUpgrader +from .package_manager import PackageManager +from .package_renamer import PackageRenamer +from .open_compat import open_compat +from .package_io import package_file_exists + + +class PackageCleanup(threading.Thread, PackageRenamer): + """ + Cleans up folders for packages that were removed, but that still have files + in use. + """ + + def __init__(self): + self.manager = PackageManager() + self.load_settings() + threading.Thread.__init__(self) + + def run(self): + found_pkgs = [] + installed_pkgs = list(self.installed_packages) + for package_name in os.listdir(sublime.packages_path()): + package_dir = os.path.join(sublime.packages_path(), package_name) + + # Cleanup packages that could not be removed due to in-use files + cleanup_file = os.path.join(package_dir, 'package-control.cleanup') + if os.path.exists(cleanup_file): + try: + shutil.rmtree(package_dir) + console_write(u'Removed old directory for package %s' % package_name, True) + + except (OSError) as e: + if not os.path.exists(cleanup_file): + open_compat(cleanup_file, 'w').close() + + error_string = (u'Unable to remove old directory for package ' + + u'%s - deferring until next start: %s') % ( + package_name, unicode_from_os(e)) + console_write(error_string, True) + + # Finish reinstalling packages that could not be upgraded due to + # in-use files + reinstall = os.path.join(package_dir, 'package-control.reinstall') + if os.path.exists(reinstall): + metadata_path = os.path.join(package_dir, 'package-metadata.json') + if not clear_directory(package_dir, [metadata_path]): + if not os.path.exists(reinstall): + open_compat(reinstall, 'w').close() + # Assigning this here prevents the callback from referencing the value + # of the "package_name" variable when it is executed + restart_message = (u'An error occurred while trying to ' + + u'finish the upgrade of %s. You will most likely need to ' + + u'restart your computer to complete the upgrade.') % package_name + + def show_still_locked(): + show_error(restart_message) + sublime.set_timeout(show_still_locked, 10) + else: + self.manager.install_package(package_name) + + # This adds previously installed packages from old versions of PC + if package_file_exists(package_name, 'package-metadata.json') and \ + package_name not in self.installed_packages: + installed_pkgs.append(package_name) + params = { + 'package': package_name, + 'operation': 'install', + 'version': \ + self.manager.get_metadata(package_name).get('version') + } + self.manager.record_usage(params) + + found_pkgs.append(package_name) + + if int(sublime.version()) >= 3000: + package_files = os.listdir(sublime.installed_packages_path()) + found_pkgs += [file.replace('.sublime-package', '') for file in package_files] + + sublime.set_timeout(lambda: self.finish(installed_pkgs, found_pkgs), 10) + + def finish(self, installed_pkgs, found_pkgs): + """ + A callback that can be run the main UI thread to perform saving of the + Package Control.sublime-settings file. Also fires off the + :class:`AutomaticUpgrader`. + + :param installed_pkgs: + A list of the string package names of all "installed" packages, + even ones that do not appear to be in the filesystem. + + :param found_pkgs: + A list of the string package names of all packages that are + currently installed on the filesystem. + """ + + self.save_packages(installed_pkgs) + AutomaticUpgrader(found_pkgs).start() diff --git a/sublime/Packages/Package Control/package_control/package_creator.py b/sublime/Packages/Package Control/package_control/package_creator.py new file mode 100644 index 0000000..47a3087 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/package_creator.py @@ -0,0 +1,39 @@ +import os + +from .show_error import show_error +from .package_manager import PackageManager + + +class PackageCreator(): + """ + Abstract class for commands that create .sublime-package files + """ + + def show_panel(self): + """ + Shows a list of packages that can be turned into a .sublime-package file + """ + + self.manager = PackageManager() + self.packages = self.manager.list_packages(unpacked_only=True) + if not self.packages: + show_error('There are no packages available to be packaged') + return + self.window.show_quick_panel(self.packages, self.on_done) + + def get_package_destination(self): + """ + Retrieves the destination for .sublime-package files + + :return: + A string - the path to the folder to save .sublime-package files in + """ + + destination = self.manager.settings.get('package_destination') + + # We check destination via an if statement instead of using + # the dict.get() method since the key may be set, but to a blank value + if not destination: + destination = os.path.join(os.path.expanduser('~'), 'Desktop') + + return destination diff --git a/sublime/Packages/Package Control/package_control/package_installer.py b/sublime/Packages/Package Control/package_control/package_installer.py new file mode 100644 index 0000000..9c8809c --- /dev/null +++ b/sublime/Packages/Package Control/package_control/package_installer.py @@ -0,0 +1,247 @@ +import os +import re +import threading + +import sublime + +from .preferences_filename import preferences_filename +from .thread_progress import ThreadProgress +from .package_manager import PackageManager +from .upgraders.git_upgrader import GitUpgrader +from .upgraders.hg_upgrader import HgUpgrader +from .versions import version_comparable + + +class PackageInstaller(): + """ + Provides helper functionality related to installing packages + """ + + def __init__(self): + self.manager = PackageManager() + + def make_package_list(self, ignore_actions=[], override_action=None, + ignore_packages=[]): + """ + Creates a list of packages and what operation would be performed for + each. Allows filtering by the applicable action or package name. + Returns the information in a format suitable for displaying in the + quick panel. + + :param ignore_actions: + A list of actions to ignore packages by. Valid actions include: + `install`, `upgrade`, `downgrade`, `reinstall`, `overwrite`, + `pull` and `none`. `pull` andd `none` are for Git and Hg + repositories. `pull` is present when incoming changes are detected, + where as `none` is selected if no commits are available. `overwrite` + is for packages that do not include version information via the + `package-metadata.json` file. + + :param override_action: + A string action name to override the displayed action for all listed + packages. + + :param ignore_packages: + A list of packages names that should not be returned in the list + + :return: + A list of lists, each containing three strings: + 0 - package name + 1 - package description + 2 - action; [extra info;] package url + """ + + packages = self.manager.list_available_packages() + installed_packages = self.manager.list_packages() + + package_list = [] + for package in sorted(iter(packages.keys()), key=lambda s: s.lower()): + if ignore_packages and package in ignore_packages: + continue + package_entry = [package] + info = packages[package] + download = info['download'] + + if package in installed_packages: + installed = True + metadata = self.manager.get_metadata(package) + if metadata.get('version'): + installed_version = metadata['version'] + else: + installed_version = None + else: + installed = False + + installed_version_name = 'v' + installed_version if \ + installed and installed_version else 'unknown version' + new_version = 'v' + download['version'] + + vcs = None + package_dir = self.manager.get_package_dir(package) + settings = self.manager.settings + + if override_action: + action = override_action + extra = '' + + else: + if os.path.exists(os.path.join(package_dir, '.git')): + if settings.get('ignore_vcs_packages'): + continue + vcs = 'git' + incoming = GitUpgrader(settings.get('git_binary'), + settings.get('git_update_command'), package_dir, + settings.get('cache_length'), settings.get('debug') + ).incoming() + elif os.path.exists(os.path.join(package_dir, '.hg')): + if settings.get('ignore_vcs_packages'): + continue + vcs = 'hg' + incoming = HgUpgrader(settings.get('hg_binary'), + settings.get('hg_update_command'), package_dir, + settings.get('cache_length'), settings.get('debug') + ).incoming() + + if installed: + if vcs: + if incoming: + action = 'pull' + extra = ' with ' + vcs + else: + action = 'none' + extra = '' + elif not installed_version: + action = 'overwrite' + extra = ' %s with %s' % (installed_version_name, + new_version) + else: + installed_version = version_comparable(installed_version) + download_version = version_comparable(download['version']) + if download_version > installed_version: + action = 'upgrade' + extra = ' to %s from %s' % (new_version, + installed_version_name) + elif download_version < installed_version: + action = 'downgrade' + extra = ' to %s from %s' % (new_version, + installed_version_name) + else: + action = 'reinstall' + extra = ' %s' % new_version + else: + action = 'install' + extra = ' %s' % new_version + extra += ';' + + if action in ignore_actions: + continue + + description = info.get('description') + if not description: + description = 'No description provided' + package_entry.append(description) + package_entry.append(action + extra + ' ' + + re.sub('^https?://', '', info['homepage'])) + package_list.append(package_entry) + return package_list + + def disable_packages(self, packages): + """ + Disables one or more packages before installing or upgrading to prevent + errors where Sublime Text tries to read files that no longer exist, or + read a half-written file. + + :param packages: The string package name, or an array of strings + """ + + if not isinstance(packages, list): + packages = [packages] + + # Don't disable Package Control so it does not get stuck disabled + if 'Package Control' in packages: + packages.remove('Package Control') + + disabled = [] + + settings = sublime.load_settings(preferences_filename()) + ignored = settings.get('ignored_packages') + if not ignored: + ignored = [] + for package in packages: + if not package in ignored: + ignored.append(package) + disabled.append(package) + settings.set('ignored_packages', ignored) + sublime.save_settings(preferences_filename()) + return disabled + + def reenable_package(self, package): + """ + Re-enables a package after it has been installed or upgraded + + :param package: The string package name + """ + + settings = sublime.load_settings(preferences_filename()) + ignored = settings.get('ignored_packages') + if not ignored: + return + if package in ignored: + settings.set('ignored_packages', + list(set(ignored) - set([package]))) + sublime.save_settings(preferences_filename()) + + def on_done(self, picked): + """ + Quick panel user selection handler - disables a package, installs or + upgrades it, then re-enables the package + + :param picked: + An integer of the 0-based package name index from the presented + list. -1 means the user cancelled. + """ + + if picked == -1: + return + name = self.package_list[picked][0] + + if name in self.disable_packages(name): + on_complete = lambda: self.reenable_package(name) + else: + on_complete = None + + thread = PackageInstallerThread(self.manager, name, on_complete) + thread.start() + ThreadProgress(thread, 'Installing package %s' % name, + 'Package %s successfully %s' % (name, self.completion_type)) + + +class PackageInstallerThread(threading.Thread): + """ + A thread to run package install/upgrade operations in so that the main + Sublime Text thread does not get blocked and freeze the UI + """ + + def __init__(self, manager, package, on_complete): + """ + :param manager: + An instance of :class:`PackageManager` + + :param package: + The string package name to install/upgrade + + :param on_complete: + A callback to run after installing/upgrading the package + """ + + self.package = package + self.manager = manager + self.on_complete = on_complete + threading.Thread.__init__(self) + + def run(self): + try: + self.result = self.manager.install_package(self.package) + finally: + if self.on_complete: + sublime.set_timeout(self.on_complete, 1) diff --git a/sublime/Packages/Package Control/package_control/package_io.py b/sublime/Packages/Package Control/package_control/package_io.py new file mode 100644 index 0000000..14ab134 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/package_io.py @@ -0,0 +1,126 @@ +import os +import zipfile + +import sublime + +from .console_write import console_write +from .open_compat import open_compat, read_compat +from .unicode import unicode_from_os +from .file_not_found_error import FileNotFoundError + + +def read_package_file(package, relative_path, binary=False, debug=False): + package_dir = _get_package_dir(package) + file_path = os.path.join(package_dir, relative_path) + + if os.path.exists(package_dir): + result = _read_regular_file(package, relative_path, binary, debug) + if result != False: + return result + + if int(sublime.version()) >= 3000: + result = _read_zip_file(package, relative_path, binary, debug) + if result != False: + return result + + if debug: + console_write(u"Unable to find file %s in the package %s" % (relative_path, package), True) + return False + + +def package_file_exists(package, relative_path): + package_dir = _get_package_dir(package) + file_path = os.path.join(package_dir, relative_path) + + if os.path.exists(package_dir): + result = _regular_file_exists(package, relative_path) + if result: + return result + + if int(sublime.version()) >= 3000: + return _zip_file_exists(package, relative_path) + + return False + + +def _get_package_dir(package): + """:return: The full filesystem path to the package directory""" + + return os.path.join(sublime.packages_path(), package) + + +def _read_regular_file(package, relative_path, binary=False, debug=False): + package_dir = _get_package_dir(package) + file_path = os.path.join(package_dir, relative_path) + try: + with open_compat(file_path, ('rb' if binary else 'r')) as f: + return read_compat(f) + + except (FileNotFoundError) as e: + if debug: + console_write(u"Unable to find file %s in the package folder for %s" % (relative_path, package), True) + return False + + +def _read_zip_file(package, relative_path, binary=False, debug=False): + zip_path = os.path.join(sublime.installed_packages_path(), + package + '.sublime-package') + + if not os.path.exists(zip_path): + if debug: + console_write(u"Unable to find a sublime-package file for %s" % package, True) + return False + + try: + package_zip = zipfile.ZipFile(zip_path, 'r') + + except (zipfile.BadZipfile): + console_write(u'An error occurred while trying to unzip the sublime-package file for %s.' % package, True) + return False + + try: + contents = package_zip.read(relative_path) + if not binary: + contents = contents.decode('utf-8') + return contents + + except (KeyError) as e: + if debug: + console_write(u"Unable to find file %s in the sublime-package file for %s" % (relative_path, package), True) + + except (IOError) as e: + message = unicode_from_os(e) + console_write(u'Unable to read file from sublime-package file for %s due to an invalid filename' % package, True) + + except (UnicodeDecodeError): + console_write(u'Unable to read file from sublime-package file for %s due to an invalid filename or character encoding issue' % package, True) + + return False + + +def _regular_file_exists(package, relative_path): + package_dir = _get_package_dir(package) + file_path = os.path.join(package_dir, relative_path) + return os.path.exists(file_path) + + +def _zip_file_exists(package, relative_path): + zip_path = os.path.join(sublime.installed_packages_path(), + package + '.sublime-package') + + if not os.path.exists(zip_path): + return False + + try: + package_zip = zipfile.ZipFile(zip_path, 'r') + + except (zipfile.BadZipfile): + console_write(u'An error occurred while trying to unzip the sublime-package file for %s.' % package_name, True) + return False + + try: + package_zip.getinfo(relative_path) + return True + + except (KeyError) as e: + return False diff --git a/sublime/Packages/Package Control/package_control/package_manager.py b/sublime/Packages/Package Control/package_control/package_manager.py new file mode 100644 index 0000000..c013254 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/package_manager.py @@ -0,0 +1,1026 @@ +import sys +import os +import re +import socket +import json +import time +import zipfile +import shutil +from fnmatch import fnmatch +import datetime +import tempfile +import locale + +try: + # Python 3 + from urllib.parse import urlencode, urlparse + import compileall + str_cls = str +except (ImportError): + # Python 2 + from urllib import urlencode + from urlparse import urlparse + str_cls = unicode + +import sublime + +from .show_error import show_error +from .console_write import console_write +from .open_compat import open_compat, read_compat +from .unicode import unicode_from_os +from .clear_directory import clear_directory +from .cache import (clear_cache, set_cache, get_cache, merge_cache_under_settings, + merge_cache_over_settings, set_cache_under_settings, set_cache_over_settings) +from .versions import version_comparable, version_sort +from .downloaders.background_downloader import BackgroundDownloader +from .downloaders.downloader_exception import DownloaderException +from .providers.provider_exception import ProviderException +from .clients.client_exception import ClientException +from .download_manager import downloader +from .providers.channel_provider import ChannelProvider +from .upgraders.git_upgrader import GitUpgrader +from .upgraders.hg_upgrader import HgUpgrader +from .package_io import read_package_file +from .providers import CHANNEL_PROVIDERS, REPOSITORY_PROVIDERS +from . import __version__ + + +class PackageManager(): + """ + Allows downloading, creating, installing, upgrading, and deleting packages + + Delegates metadata retrieval to the CHANNEL_PROVIDERS classes. + Uses VcsUpgrader-based classes for handling git and hg repositories in the + Packages folder. Downloader classes are utilized to fetch contents of URLs. + + Also handles displaying package messaging, and sending usage information to + the usage server. + """ + + def __init__(self): + # Here we manually copy the settings since sublime doesn't like + # code accessing settings from threads + self.settings = {} + settings = sublime.load_settings('Package Control.sublime-settings') + for setting in ['timeout', 'repositories', 'channels', + 'package_name_map', 'dirs_to_ignore', 'files_to_ignore', + 'package_destination', 'cache_length', 'auto_upgrade', + 'files_to_ignore_binary', 'files_to_keep', 'dirs_to_keep', + 'git_binary', 'git_update_command', 'hg_binary', + 'hg_update_command', 'http_proxy', 'https_proxy', + 'auto_upgrade_ignore', 'auto_upgrade_frequency', + 'submit_usage', 'submit_url', 'renamed_packages', + 'files_to_include', 'files_to_include_binary', 'certs', + 'ignore_vcs_packages', 'proxy_username', 'proxy_password', + 'debug', 'user_agent', 'http_cache', 'http_cache_length', + 'install_prereleases', 'openssl_binary']: + if settings.get(setting) == None: + continue + self.settings[setting] = settings.get(setting) + + # https_proxy will inherit from http_proxy unless it is set to a + # string value or false + no_https_proxy = self.settings.get('https_proxy') in ["", None] + if no_https_proxy and self.settings.get('http_proxy'): + self.settings['https_proxy'] = self.settings.get('http_proxy') + if self.settings.get('https_proxy') == False: + self.settings['https_proxy'] = '' + + self.settings['platform'] = sublime.platform() + self.settings['version'] = sublime.version() + + # Use the cache to see if settings have changed since the last + # time the package manager was created, and clearing any cached + # values if they have. + previous_settings = get_cache('filtered_settings', {}) + + # Reduce the settings down to exclude channel info since that will + # make the settings always different + filtered_settings = self.settings.copy() + for key in ['repositories', 'channels', 'package_name_map', 'cache']: + if key in filtered_settings: + del filtered_settings[key] + + if filtered_settings != previous_settings and previous_settings != {}: + console_write(u'Settings change detected, clearing cache', True) + clear_cache() + set_cache('filtered_settings', filtered_settings) + + def get_metadata(self, package): + """ + Returns the package metadata for an installed package + + :return: + A dict with the keys: + version + url + description + or an empty dict on error + """ + + try: + debug = self.settings.get('debug') + metadata_json = read_package_file(package, 'package-metadata.json', debug=debug) + if metadata_json: + return json.loads(metadata_json) + + except (IOError, ValueError) as e: + pass + + return {} + + def list_repositories(self): + """ + Returns a master list of all repositories pulled from all sources + + These repositories come from the channels specified in the + "channels" setting, plus any repositories listed in the + "repositories" setting. + + :return: + A list of all available repositories + """ + + cache_ttl = self.settings.get('cache_length') + + repositories = self.settings.get('repositories') + channels = self.settings.get('channels') + for channel in channels: + channel = channel.strip() + + # Caches various info from channels for performance + cache_key = channel + '.repositories' + channel_repositories = get_cache(cache_key) + + merge_cache_under_settings(self, 'package_name_map', channel) + merge_cache_under_settings(self, 'renamed_packages', channel) + merge_cache_under_settings(self, 'unavailable_packages', channel, list_=True) + + # If any of the info was not retrieved from the cache, we need to + # grab the channel to get it + if channel_repositories == None or \ + self.settings.get('package_name_map') == None or \ + self.settings.get('renamed_packages') == None: + + for provider_class in CHANNEL_PROVIDERS: + if provider_class.match_url(channel): + provider = provider_class(channel, self.settings) + break + + try: + channel_repositories = provider.get_repositories() + set_cache(cache_key, channel_repositories, cache_ttl) + + for repo in channel_repositories: + repo_packages = provider.get_packages(repo) + packages_cache_key = repo + '.packages' + set_cache(packages_cache_key, repo_packages, cache_ttl) + + # Have the local name map override the one from the channel + name_map = provider.get_name_map() + set_cache_under_settings(self, 'package_name_map', channel, name_map, cache_ttl) + + renamed_packages = provider.get_renamed_packages() + set_cache_under_settings(self, 'renamed_packages', channel, renamed_packages, cache_ttl) + + unavailable_packages = provider.get_unavailable_packages() + set_cache_under_settings(self, 'unavailable_packages', channel, unavailable_packages, cache_ttl, list_=True) + + provider_certs = provider.get_certs() + certs = self.settings.get('certs', {}).copy() + certs.update(provider_certs) + # Save the master list of certs, used by downloaders/cert_provider.py + set_cache('*.certs', certs, cache_ttl) + + except (DownloaderException, ClientException, ProviderException) as e: + console_write(e, True) + continue + + repositories.extend(channel_repositories) + return [repo.strip() for repo in repositories] + + def list_available_packages(self): + """ + Returns a master list of every available package from all sources + + :return: + A dict in the format: + { + 'Package Name': { + # Package details - see example-packages.json for format + }, + ... + } + """ + + if self.settings.get('debug'): + console_write(u"Fetching list of available packages", True) + console_write(u" Platform: %s-%s" % (sublime.platform(),sublime.arch())) + console_write(u" Sublime Text Version: %s" % sublime.version()) + console_write(u" Package Control Version: %s" % __version__) + + cache_ttl = self.settings.get('cache_length') + repositories = self.list_repositories() + packages = {} + bg_downloaders = {} + active = [] + repos_to_download = [] + name_map = self.settings.get('package_name_map', {}) + + # Repositories are run in reverse order so that the ones first + # on the list will overwrite those last on the list + for repo in repositories[::-1]: + cache_key = repo + '.packages' + repository_packages = get_cache(cache_key) + + if repository_packages != None: + packages.update(repository_packages) + + else: + domain = urlparse(repo).hostname + if domain not in bg_downloaders: + bg_downloaders[domain] = BackgroundDownloader( + self.settings, REPOSITORY_PROVIDERS) + bg_downloaders[domain].add_url(repo) + repos_to_download.append(repo) + + for bg_downloader in list(bg_downloaders.values()): + bg_downloader.start() + active.append(bg_downloader) + + # Wait for all of the downloaders to finish + while active: + bg_downloader = active.pop() + bg_downloader.join() + + # Grabs the results and stuff it all in the cache + for repo in repos_to_download: + domain = urlparse(repo).hostname + bg_downloader = bg_downloaders[domain] + provider = bg_downloader.get_provider(repo) + + # Allow name mapping of packages for schema version < 2.0 + repository_packages = {} + for name, info in provider.get_packages(): + name = name_map.get(name, name) + info['name'] = name + repository_packages[name] = info + + # Display errors we encountered while fetching package info + for url, exception in provider.get_failed_sources(): + console_write(exception, True) + for name, exception in provider.get_broken_packages(): + console_write(exception, True) + + cache_key = repo + '.packages' + set_cache(cache_key, repository_packages, cache_ttl) + packages.update(repository_packages) + + renamed_packages = provider.get_renamed_packages() + set_cache_under_settings(self, 'renamed_packages', repo, renamed_packages, cache_ttl) + + unavailable_packages = provider.get_unavailable_packages() + set_cache_under_settings(self, 'unavailable_packages', repo, unavailable_packages, cache_ttl, list_=True) + + return packages + + def list_packages(self, unpacked_only=False): + """ + :param unpacked_only: + Only list packages that are not inside of .sublime-package files + + :return: A list of all installed, non-default, package names + """ + + package_names = os.listdir(sublime.packages_path()) + package_names = [path for path in package_names if + os.path.isdir(os.path.join(sublime.packages_path(), path))] + + if int(sublime.version()) > 3000 and unpacked_only == False: + package_files = os.listdir(sublime.installed_packages_path()) + package_names += [f.replace('.sublime-package', '') for f in package_files if re.search('\.sublime-package$', f) != None] + + # Ignore things to be deleted + ignored = ['User'] + for package in package_names: + cleanup_file = os.path.join(sublime.packages_path(), package, + 'package-control.cleanup') + if os.path.exists(cleanup_file): + ignored.append(package) + + packages = list(set(package_names) - set(ignored) - + set(self.list_default_packages())) + packages = sorted(packages, key=lambda s: s.lower()) + + return packages + + def list_all_packages(self): + """ :return: A list of all installed package names, including default packages""" + + packages = self.list_default_packages() + self.list_packages() + packages = sorted(packages, key=lambda s: s.lower()) + return packages + + def list_default_packages(self): + """ :return: A list of all default package names""" + + if int(sublime.version()) > 3000: + bundled_packages_path = os.path.join(os.path.dirname(sublime.executable_path()), + 'Packages') + files = os.listdir(bundled_packages_path) + + else: + files = os.listdir(os.path.join(os.path.dirname( + sublime.packages_path()), 'Pristine Packages')) + files = list(set(files) - set(os.listdir( + sublime.installed_packages_path()))) + packages = [file.replace('.sublime-package', '') for file in files] + packages = sorted(packages, key=lambda s: s.lower()) + return packages + + def get_package_dir(self, package): + """:return: The full filesystem path to the package directory""" + + return os.path.join(sublime.packages_path(), package) + + def get_mapped_name(self, package): + """:return: The name of the package after passing through mapping rules""" + + return self.settings.get('package_name_map', {}).get(package, package) + + def create_package(self, package_name, package_destination, + binary_package=False): + """ + Creates a .sublime-package file from the running Packages directory + + :param package_name: + The package to create a .sublime-package file for + + :param package_destination: + The full filesystem path of the directory to save the new + .sublime-package file in. + + :param binary_package: + If the created package should follow the binary package include/ + exclude patterns from the settings. These normally include a setup + to exclude .py files and include .pyc files, but that can be + changed via settings. + + :return: bool if the package file was successfully created + """ + + package_dir = self.get_package_dir(package_name) + + if not os.path.exists(package_dir): + show_error(u'The folder for the package name specified, %s, does not exist in %s' % ( + package_name, sublime.packages_path())) + return False + + package_filename = package_name + '.sublime-package' + package_path = os.path.join(package_destination, + package_filename) + + if not os.path.exists(sublime.installed_packages_path()): + os.mkdir(sublime.installed_packages_path()) + + if os.path.exists(package_path): + os.remove(package_path) + + try: + package_file = zipfile.ZipFile(package_path, "w", + compression=zipfile.ZIP_DEFLATED) + except (OSError, IOError) as e: + show_error(u'An error occurred creating the package file %s in %s.\n\n%s' % ( + package_filename, package_destination, unicode_from_os(e))) + return False + + if int(sublime.version()) >= 3000: + compileall.compile_dir(package_dir, quiet=True, legacy=True, optimize=2) + + dirs_to_ignore = self.settings.get('dirs_to_ignore', []) + if not binary_package: + files_to_ignore = self.settings.get('files_to_ignore', []) + files_to_include = self.settings.get('files_to_include', []) + else: + files_to_ignore = self.settings.get('files_to_ignore_binary', []) + files_to_include = self.settings.get('files_to_include_binary', []) + + slash = '\\' if os.name == 'nt' else '/' + trailing_package_dir = package_dir + slash if package_dir[-1] != slash else package_dir + package_dir_regex = re.compile('^' + re.escape(trailing_package_dir)) + for root, dirs, files in os.walk(package_dir): + [dirs.remove(dir_) for dir_ in dirs if dir_ in dirs_to_ignore] + paths = dirs + paths.extend(files) + for path in paths: + full_path = os.path.join(root, path) + relative_path = re.sub(package_dir_regex, '', full_path) + + ignore_matches = [fnmatch(relative_path, p) for p in files_to_ignore] + include_matches = [fnmatch(relative_path, p) for p in files_to_include] + if any(ignore_matches) and not any(include_matches): + continue + + if os.path.isdir(full_path): + continue + package_file.write(full_path, relative_path) + + package_file.close() + + return True + + def install_package(self, package_name): + """ + Downloads and installs (or upgrades) a package + + Uses the self.list_available_packages() method to determine where to + retrieve the package file from. + + The install process consists of: + + 1. Finding the package + 2. Downloading the .sublime-package/.zip file + 3. Extracting the package file + 4. Showing install/upgrade messaging + 5. Submitting usage info + 6. Recording that the package is installed + + :param package_name: + The package to download and install + + :return: bool if the package was successfully installed + """ + + packages = self.list_available_packages() + + is_available = package_name in list(packages.keys()) + is_unavailable = package_name in self.settings.get('unavailable_packages', []) + + if is_unavailable and not is_available: + console_write(u'The package "%s" is not available on this platform.' % package_name, True) + return False + + if not is_available: + show_error(u'The package specified, %s, is not available' % package_name) + return False + + url = packages[package_name]['download']['url'] + package_filename = package_name + '.sublime-package' + + tmp_dir = tempfile.mkdtemp() + + try: + # This is refers to the zipfile later on, so we define it here so we can + # close the zip file if set during the finally clause + package_zip = None + + tmp_package_path = os.path.join(tmp_dir, package_filename) + + unpacked_package_dir = self.get_package_dir(package_name) + package_path = os.path.join(sublime.installed_packages_path(), + package_filename) + pristine_package_path = os.path.join(os.path.dirname( + sublime.packages_path()), 'Pristine Packages', package_filename) + + if os.path.exists(os.path.join(unpacked_package_dir, '.git')): + if self.settings.get('ignore_vcs_packages'): + show_error(u'Skipping git package %s since the setting ignore_vcs_packages is set to true' % package_name) + return False + return GitUpgrader(self.settings['git_binary'], + self.settings['git_update_command'], unpacked_package_dir, + self.settings['cache_length'], self.settings['debug']).run() + elif os.path.exists(os.path.join(unpacked_package_dir, '.hg')): + if self.settings.get('ignore_vcs_packages'): + show_error(u'Skipping hg package %s since the setting ignore_vcs_packages is set to true' % package_name) + return False + return HgUpgrader(self.settings['hg_binary'], + self.settings['hg_update_command'], unpacked_package_dir, + self.settings['cache_length'], self.settings['debug']).run() + + old_version = self.get_metadata(package_name).get('version') + is_upgrade = old_version != None + + # Download the sublime-package or zip file + try: + with downloader(url, self.settings) as manager: + package_bytes = manager.fetch(url, 'Error downloading package.') + except (DownloaderException) as e: + console_write(e, True) + show_error(u'Unable to download %s. Please view the console for more details.' % package_name) + return False + + with open_compat(tmp_package_path, "wb") as package_file: + package_file.write(package_bytes) + + # Try to open it as a zip file + try: + package_zip = zipfile.ZipFile(tmp_package_path, 'r') + except (zipfile.BadZipfile): + show_error(u'An error occurred while trying to unzip the package file for %s. Please try installing the package again.' % package_name) + return False + + # Scan through the root level of the zip file to gather some info + root_level_paths = [] + last_path = None + for path in package_zip.namelist(): + try: + if not isinstance(path, str_cls): + path = path.decode('utf-8', 'strict') + except (UnicodeDecodeError): + console_write(u'One or more of the zip file entries in %s is not encoded using UTF-8, aborting' % package_name, True) + return False + + last_path = path + + if path.find('/') in [len(path) - 1, -1]: + root_level_paths.append(path) + # Make sure there are no paths that look like security vulnerabilities + if path[0] == '/' or path.find('../') != -1 or path.find('..\\') != -1: + show_error(u'The package specified, %s, contains files outside of the package dir and cannot be safely installed.' % package_name) + return False + + if last_path and len(root_level_paths) == 0: + root_level_paths.append(last_path[0:last_path.find('/') + 1]) + + # If there is only a single directory at the top leve, the file + # is most likely a zip from BitBucket or GitHub and we need + # to skip the top-level dir when extracting + skip_root_dir = len(root_level_paths) == 1 and \ + root_level_paths[0].endswith('/') + + no_package_file_zip_path = '.no-sublime-package' + if skip_root_dir: + no_package_file_zip_path = root_level_paths[0] + no_package_file_zip_path + + # If we should extract unpacked or as a .sublime-package file + unpack = True + + # By default, ST3 prefers .sublime-package files since this allows + # overriding files in the Packages/{package_name}/ folder + if int(sublime.version()) >= 3000: + unpack = False + + # If the package maintainer doesn't want a .sublime-package + try: + package_zip.getinfo(no_package_file_zip_path) + unpack = True + except (KeyError): + pass + + # If we already have a package-metadata.json file in + # Packages/{package_name}/, the only way to successfully upgrade + # will be to unpack + unpacked_metadata_file = os.path.join(unpacked_package_dir, + 'package-metadata.json') + if os.path.exists(unpacked_metadata_file): + unpack = True + + # If we determined it should be unpacked, we extract directly + # into the Packages/{package_name}/ folder + if unpack: + self.backup_package_dir(package_name) + package_dir = unpacked_package_dir + + # Otherwise we go into a temp dir since we will be creating a + # new .sublime-package file later + else: + tmp_working_dir = os.path.join(tmp_dir, 'working') + os.mkdir(tmp_working_dir) + package_dir = tmp_working_dir + + package_metadata_file = os.path.join(package_dir, + 'package-metadata.json') + + if not os.path.exists(package_dir): + os.mkdir(package_dir) + + os.chdir(package_dir) + + # Here we don't use .extractall() since it was having issues on OS X + overwrite_failed = False + extracted_paths = [] + for path in package_zip.namelist(): + dest = path + + try: + if not isinstance(dest, str_cls): + dest = dest.decode('utf-8', 'strict') + except (UnicodeDecodeError): + console_write(u'One or more of the zip file entries in %s is not encoded using UTF-8, aborting' % package_name, True) + return False + + if os.name == 'nt': + regex = ':|\*|\?|"|<|>|\|' + if re.search(regex, dest) != None: + console_write(u'Skipping file from package named %s due to an invalid filename' % package_name, True) + continue + + # If there was only a single directory in the package, we remove + # that folder name from the paths as we extract entries + if skip_root_dir: + dest = dest[len(root_level_paths[0]):] + + if os.name == 'nt': + dest = dest.replace('/', '\\') + else: + dest = dest.replace('\\', '/') + + dest = os.path.join(package_dir, dest) + + def add_extracted_dirs(dir_): + while dir_ not in extracted_paths: + extracted_paths.append(dir_) + dir_ = os.path.dirname(dir_) + if dir_ == package_dir: + break + + if path.endswith('/'): + if not os.path.exists(dest): + os.makedirs(dest) + add_extracted_dirs(dest) + else: + dest_dir = os.path.dirname(dest) + if not os.path.exists(dest_dir): + os.makedirs(dest_dir) + add_extracted_dirs(dest_dir) + extracted_paths.append(dest) + try: + open_compat(dest, 'wb').write(package_zip.read(path)) + except (IOError) as e: + message = unicode_from_os(e) + if re.search('[Ee]rrno 13', message): + overwrite_failed = True + break + console_write(u'Skipping file from package named %s due to an invalid filename' % package_name, True) + + except (UnicodeDecodeError): + console_write(u'Skipping file from package named %s due to an invalid filename' % package_name, True) + + package_zip.close() + package_zip = None + + # If upgrading failed, queue the package to upgrade upon next start + if overwrite_failed: + reinstall_file = os.path.join(package_dir, 'package-control.reinstall') + open_compat(reinstall_file, 'w').close() + + # Don't delete the metadata file, that way we have it + # when the reinstall happens, and the appropriate + # usage info can be sent back to the server + clear_directory(package_dir, [reinstall_file, package_metadata_file]) + + show_error(u'An error occurred while trying to upgrade %s. Please restart Sublime Text to finish the upgrade.' % package_name) + return False + + # Here we clean out any files that were not just overwritten. It is ok + # if there is an error removing a file. The next time there is an + # upgrade, it should be cleaned out successfully then. + clear_directory(package_dir, extracted_paths) + + self.print_messages(package_name, package_dir, is_upgrade, old_version) + + with open_compat(package_metadata_file, 'w') as f: + metadata = { + "version": packages[package_name]['download']['version'], + "url": packages[package_name]['homepage'], + "description": packages[package_name]['description'] + } + json.dump(metadata, f) + + # Submit install and upgrade info + if is_upgrade: + params = { + 'package': package_name, + 'operation': 'upgrade', + 'version': packages[package_name]['download']['version'], + 'old_version': old_version + } + else: + params = { + 'package': package_name, + 'operation': 'install', + 'version': packages[package_name]['download']['version'] + } + self.record_usage(params) + + # Record the install in the settings file so that you can move + # settings across computers and have the same packages installed + def save_package(): + settings = sublime.load_settings('Package Control.sublime-settings') + installed_packages = settings.get('installed_packages', []) + if not installed_packages: + installed_packages = [] + installed_packages.append(package_name) + installed_packages = list(set(installed_packages)) + installed_packages = sorted(installed_packages, + key=lambda s: s.lower()) + settings.set('installed_packages', installed_packages) + sublime.save_settings('Package Control.sublime-settings') + sublime.set_timeout(save_package, 1) + + # If we didn't extract directly into the Packages/{package_name}/ + # folder, we need to create a .sublime-package file and install it + if not unpack: + try: + # Remove the downloaded file since we are going to overwrite it + os.remove(tmp_package_path) + package_zip = zipfile.ZipFile(tmp_package_path, "w", + compression=zipfile.ZIP_DEFLATED) + except (OSError, IOError) as e: + show_error(u'An error occurred creating the package file %s in %s.\n\n%s' % ( + package_filename, tmp_dir, unicode_from_os(e))) + return False + + package_dir_regex = re.compile('^' + re.escape(package_dir)) + for root, dirs, files in os.walk(package_dir): + paths = dirs + paths.extend(files) + for path in paths: + full_path = os.path.join(root, path) + relative_path = re.sub(package_dir_regex, '', full_path) + if os.path.isdir(full_path): + continue + package_zip.write(full_path, relative_path) + + package_zip.close() + package_zip = None + + if os.path.exists(package_path): + os.remove(package_path) + shutil.move(tmp_package_path, package_path) + + # We have to remove the pristine package too or else Sublime Text 2 + # will silently delete the package + if os.path.exists(pristine_package_path): + os.remove(pristine_package_path) + + os.chdir(sublime.packages_path()) + return True + + finally: + # We need to make sure the zipfile is closed to + # help prevent permissions errors on Windows + if package_zip: + package_zip.close() + + # Try to remove the tmp dir after a second to make sure + # a virus scanner is holding a reference to the zipfile + # after we close it. + def remove_tmp_dir(): + try: + shutil.rmtree(tmp_dir) + except (PermissionError): + # If we can't remove the tmp dir, don't let an uncaught exception + # fall through and break the install process + pass + sublime.set_timeout(remove_tmp_dir, 1000) + + def backup_package_dir(self, package_name): + """ + Does a full backup of the Packages/{package}/ dir to Backup/ + + :param package_name: + The name of the package to back up + + :return: + If the backup succeeded + """ + + package_dir = os.path.join(sublime.packages_path(), package_name) + if not os.path.exists(package_dir): + return True + + try: + backup_dir = os.path.join(os.path.dirname( + sublime.packages_path()), 'Backup', + datetime.datetime.now().strftime('%Y%m%d%H%M%S')) + if not os.path.exists(backup_dir): + os.makedirs(backup_dir) + package_backup_dir = os.path.join(backup_dir, package_name) + if os.path.exists(package_backup_dir): + console_write(u"FOLDER %s ALREADY EXISTS!" % package_backup_dir) + shutil.copytree(package_dir, package_backup_dir) + return True + + except (OSError, IOError) as e: + show_error(u'An error occurred while trying to backup the package directory for %s.\n\n%s' % ( + package_name, unicode_from_os(e))) + if os.path.exists(package_backup_dir): + shutil.rmtree(package_backup_dir) + return False + + def print_messages(self, package, package_dir, is_upgrade, old_version): + """ + Prints out package install and upgrade messages + + The functionality provided by this allows package maintainers to + show messages to the user when a package is installed, or when + certain version upgrade occur. + + :param package: + The name of the package the message is for + + :param package_dir: + The full filesystem path to the package directory + + :param is_upgrade: + If the install was actually an upgrade + + :param old_version: + The string version of the package before the upgrade occurred + """ + + messages_file = os.path.join(package_dir, 'messages.json') + if not os.path.exists(messages_file): + return + + messages_fp = open_compat(messages_file, 'r') + try: + message_info = json.loads(read_compat(messages_fp)) + except (ValueError): + console_write(u'Error parsing messages.json for %s' % package, True) + return + messages_fp.close() + + output = '' + if not is_upgrade and message_info.get('install'): + install_messages = os.path.join(package_dir, + message_info.get('install')) + message = '\n\n%s:\n%s\n\n ' % (package, + ('-' * len(package))) + with open_compat(install_messages, 'r') as f: + message += read_compat(f).replace('\n', '\n ') + output += message + '\n' + + elif is_upgrade and old_version: + upgrade_messages = list(set(message_info.keys()) - + set(['install'])) + upgrade_messages = version_sort(upgrade_messages, reverse=True) + old_version_cmp = version_comparable(old_version) + + for version in upgrade_messages: + if version_comparable(version) <= old_version_cmp: + break + if not output: + message = '\n\n%s:\n%s\n' % (package, + ('-' * len(package))) + output += message + upgrade_message_path = os.path.join(package_dir, + message_info.get(version)) + message = '\n ' + with open_compat(upgrade_message_path, 'r') as f: + message += read_compat(f).replace('\n', '\n ') + output += message + '\n' + + if not output: + return + + def print_to_panel(): + window = sublime.active_window() + + views = window.views() + view = None + for _view in views: + if _view.name() == 'Package Control Messages': + view = _view + break + + if not view: + view = window.new_file() + view.set_name('Package Control Messages') + view.set_scratch(True) + + def write(string): + view.run_command('package_message', {'string': string}) + + if not view.size(): + view.settings().set("word_wrap", True) + write('Package Control Messages\n' + + '========================') + + write(output) + sublime.set_timeout(print_to_panel, 1) + + def remove_package(self, package_name): + """ + Deletes a package + + The deletion process consists of: + + 1. Deleting the directory (or marking it for deletion if deletion fails) + 2. Submitting usage info + 3. Removing the package from the list of installed packages + + :param package_name: + The package to delete + + :return: bool if the package was successfully deleted + """ + + installed_packages = self.list_packages() + + if package_name not in installed_packages: + show_error(u'The package specified, %s, is not installed' % package_name) + return False + + os.chdir(sublime.packages_path()) + + # Give Sublime Text some time to ignore the package + time.sleep(1) + + package_filename = package_name + '.sublime-package' + installed_package_path = os.path.join(sublime.installed_packages_path(), + package_filename) + pristine_package_path = os.path.join(os.path.dirname( + sublime.packages_path()), 'Pristine Packages', package_filename) + package_dir = self.get_package_dir(package_name) + + version = self.get_metadata(package_name).get('version') + + try: + if os.path.exists(installed_package_path): + os.remove(installed_package_path) + except (OSError, IOError) as e: + show_error(u'An error occurred while trying to remove the installed package file for %s.\n\n%s' % ( + package_name, unicode_from_os(e))) + return False + + try: + if os.path.exists(pristine_package_path): + os.remove(pristine_package_path) + except (OSError, IOError) as e: + show_error(u'An error occurred while trying to remove the pristine package file for %s.\n\n%s' % ( + package_name, unicode_from_os(e))) + return False + + # We don't delete the actual package dir immediately due to a bug + # in sublime_plugin.py + can_delete_dir = True + if not clear_directory(package_dir): + # If there is an error deleting now, we will mark it for + # cleanup the next time Sublime Text starts + open_compat(os.path.join(package_dir, 'package-control.cleanup'), + 'w').close() + can_delete_dir = False + + params = { + 'package': package_name, + 'operation': 'remove', + 'version': version + } + self.record_usage(params) + + # Remove the package from the installed packages list + def clear_package(): + settings = sublime.load_settings('Package Control.sublime-settings') + installed_packages = settings.get('installed_packages', []) + if not installed_packages: + installed_packages = [] + installed_packages.remove(package_name) + settings.set('installed_packages', installed_packages) + sublime.save_settings('Package Control.sublime-settings') + sublime.set_timeout(clear_package, 1) + + if can_delete_dir and os.path.exists(package_dir): + os.rmdir(package_dir) + + return True + + def record_usage(self, params): + """ + Submits install, upgrade and delete actions to a usage server + + The usage information is currently displayed on the Package Control + community package list at http://wbond.net/sublime_packages/community + + :param params: + A dict of the information to submit + """ + + if not self.settings.get('submit_usage'): + return + params['package_control_version'] = \ + self.get_metadata('Package Control').get('version') + params['sublime_platform'] = self.settings.get('platform') + params['sublime_version'] = self.settings.get('version') + + # For Python 2, we need to explicitly encoding the params + for param in params: + if isinstance(params[param], str_cls): + params[param] = params[param].encode('utf-8') + + url = self.settings.get('submit_url') + '?' + urlencode(params) + + try: + with downloader(url, self.settings) as manager: + result = manager.fetch(url, 'Error submitting usage information.') + except (DownloaderException) as e: + console_write(e, True) + return + + try: + result = json.loads(result.decode('utf-8')) + if result['result'] != 'success': + raise ValueError() + except (ValueError): + console_write(u'Error submitting usage information for %s' % params['package'], True) diff --git a/sublime/Packages/Package Control/package_control/package_renamer.py b/sublime/Packages/Package Control/package_control/package_renamer.py new file mode 100644 index 0000000..73e83fd --- /dev/null +++ b/sublime/Packages/Package Control/package_control/package_renamer.py @@ -0,0 +1,117 @@ +import os + +import sublime + +from .console_write import console_write +from .package_io import package_file_exists + + +class PackageRenamer(): + """ + Class to handle renaming packages via the renamed_packages setting + gathered from channels and repositories. + """ + + def load_settings(self): + """ + Loads the list of installed packages from the + Package Control.sublime-settings file. + """ + + self.settings_file = 'Package Control.sublime-settings' + self.settings = sublime.load_settings(self.settings_file) + self.installed_packages = self.settings.get('installed_packages', []) + if not isinstance(self.installed_packages, list): + self.installed_packages = [] + + def rename_packages(self, installer): + """ + Renames any installed packages that the user has installed. + + :param installer: + An instance of :class:`PackageInstaller` + """ + + # Fetch the packages since that will pull in the renamed packages list + installer.manager.list_available_packages() + renamed_packages = installer.manager.settings.get('renamed_packages', {}) + if not renamed_packages: + renamed_packages = {} + + # These are packages that have been tracked as installed + installed_pkgs = self.installed_packages + # There are the packages actually present on the filesystem + present_packages = installer.manager.list_packages() + + # Rename directories for packages that have changed names + for package_name in renamed_packages: + package_dir = os.path.join(sublime.packages_path(), package_name) + if not package_file_exists(package_name, 'package-metadata.json'): + continue + + new_package_name = renamed_packages[package_name] + new_package_dir = os.path.join(sublime.packages_path(), + new_package_name) + + changing_case = package_name.lower() == new_package_name.lower() + case_insensitive_fs = sublime.platform() in ['windows', 'osx'] + + # Since Windows and OSX use case-insensitive filesystems, we have to + # scan through the list of installed packages if the rename of the + # package is just changing the case of it. If we don't find the old + # name for it, we continue the loop since os.path.exists() will return + # true due to the case-insensitive nature of the filesystems. + if case_insensitive_fs and changing_case: + has_old = False + for present_package_name in present_packages: + if present_package_name == package_name: + has_old = True + break + if not has_old: + continue + + if not os.path.exists(new_package_dir) or (case_insensitive_fs and changing_case): + + # Windows will not allow you to rename to the same name with + # a different case, so we work around that with a temporary name + if os.name == 'nt' and changing_case: + temp_package_name = '__' + new_package_name + temp_package_dir = os.path.join(sublime.packages_path(), + temp_package_name) + os.rename(package_dir, temp_package_dir) + package_dir = temp_package_dir + + os.rename(package_dir, new_package_dir) + installed_pkgs.append(new_package_name) + + console_write(u'Renamed %s to %s' % (package_name, new_package_name), True) + + else: + installer.manager.remove_package(package_name) + message_string = u'Removed %s since package with new name (%s) already exists' % ( + package_name, new_package_name) + console_write(message_string, True) + + try: + installed_pkgs.remove(package_name) + except (ValueError): + pass + + sublime.set_timeout(lambda: self.save_packages(installed_pkgs), 10) + + def save_packages(self, installed_packages): + """ + Saves the list of installed packages (after having been appropriately + renamed) + + :param installed_packages: + The new list of installed packages + """ + + installed_packages = list(set(installed_packages)) + installed_packages = sorted(installed_packages, + key=lambda s: s.lower()) + + if installed_packages != self.installed_packages: + self.settings.set('installed_packages', installed_packages) + sublime.save_settings(self.settings_file) diff --git a/sublime/Packages/Package Control/package_control/preferences_filename.py b/sublime/Packages/Package Control/package_control/preferences_filename.py new file mode 100644 index 0000000..7091dd9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/preferences_filename.py @@ -0,0 +1,11 @@ +import sublime + + +def preferences_filename(): + """ + :return: The appropriate settings filename based on the version of Sublime Text + """ + + if int(sublime.version()) >= 2174: + return 'Preferences.sublime-settings' + return 'Global.sublime-settings' diff --git a/sublime/Packages/Package Control/package_control/providers/__init__.py b/sublime/Packages/Package Control/package_control/providers/__init__.py new file mode 100644 index 0000000..cfea3bd --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/__init__.py @@ -0,0 +1,12 @@ +from .bitbucket_repository_provider import BitBucketRepositoryProvider +from .github_repository_provider import GitHubRepositoryProvider +from .github_user_provider import GitHubUserProvider +from .repository_provider import RepositoryProvider + +from .channel_provider import ChannelProvider + + +REPOSITORY_PROVIDERS = [BitBucketRepositoryProvider, GitHubRepositoryProvider, + GitHubUserProvider, RepositoryProvider] + +CHANNEL_PROVIDERS = [ChannelProvider] diff --git a/sublime/Packages/Package Control/package_control/providers/bitbucket_repository_provider.py b/sublime/Packages/Package Control/package_control/providers/bitbucket_repository_provider.py new file mode 100644 index 0000000..b5d603f --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/bitbucket_repository_provider.py @@ -0,0 +1,163 @@ +import re + +from ..clients.bitbucket_client import BitBucketClient +from ..downloaders.downloader_exception import DownloaderException +from ..clients.client_exception import ClientException +from .provider_exception import ProviderException + + +class BitBucketRepositoryProvider(): + """ + Allows using a public BitBucket repository as the source for a single package. + For legacy purposes, this can also be treated as the source for a Package + Control "repository". + + :param repo: + The public web URL to the BitBucket repository. Should be in the format + `https://bitbucket.org/user/package`. + + :param settings: + A dict containing at least the following fields: + `cache_length`, + `debug`, + `timeout`, + `user_agent` + Optional fields: + `http_proxy`, + `https_proxy`, + `proxy_username`, + `proxy_password`, + `query_string_params` + `install_prereleases` + """ + + def __init__(self, repo, settings): + self.cache = {} + self.repo = repo + self.settings = settings + self.failed_sources = {} + + @classmethod + def match_url(cls, repo): + """Indicates if this provider can handle the provided repo""" + + return re.search('^https?://bitbucket.org/([^/]+/[^/]+)/?$', repo) != None + + def prefetch(self): + """ + Go out and perform HTTP operations, caching the result + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + """ + + [name for name, info in self.get_packages()] + + def get_failed_sources(self): + """ + List of any URLs that could not be accessed while accessing this repository + + :return: + A generator of ("https://bitbucket.org/user/repo", Exception()) tuples + """ + + return self.failed_sources.items() + + def get_broken_packages(self): + """ + For API-compatibility with RepositoryProvider + """ + + return {}.items() + + def get_packages(self, invalid_sources=None): + """ + Uses the BitBucket API to construct necessary info for a package + + :param invalid_sources: + A list of URLs that should be ignored + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + + :return: + A generator of + ( + 'Package Name', + { + 'name': name, + 'description': description, + 'author': author, + 'homepage': homepage, + 'last_modified': last modified date, + 'download': { + 'url': url, + 'date': date, + 'version': version + }, + 'previous_names': [], + 'labels': [], + 'sources': [the repo URL], + 'readme': url, + 'issues': url, + 'donate': url, + 'buy': None + } + ) + tuples + """ + + if 'get_packages' in self.cache: + for key, value in self.cache['get_packages'].items(): + yield (key, value) + return + + client = BitBucketClient(self.settings) + + if invalid_sources != None and self.repo in invalid_sources: + raise StopIteration() + + try: + repo_info = client.repo_info(self.repo) + download = client.download_info(self.repo) + + name = repo_info['name'] + details = { + 'name': name, + 'description': repo_info['description'], + 'homepage': repo_info['homepage'], + 'author': repo_info['author'], + 'last_modified': download.get('date'), + 'download': download, + 'previous_names': [], + 'labels': [], + 'sources': [self.repo], + 'readme': repo_info['readme'], + 'issues': repo_info['issues'], + 'donate': repo_info['donate'], + 'buy': None + } + self.cache['get_packages'] = {name: details} + yield (name, details) + + except (DownloaderException, ClientException, ProviderException) as e: + self.failed_sources[self.repo] = e + self.cache['get_packages'] = {} + raise StopIteration() + + def get_renamed_packages(self): + """For API-compatibility with RepositoryProvider""" + + return {} + + def get_unavailable_packages(self): + """ + Method for compatibility with RepositoryProvider class. These providers + are based on API calls, and thus do not support different platform + downloads, making it impossible for there to be unavailable packages. + + :return: An empty list + """ + return [] diff --git a/sublime/Packages/Package Control/package_control/providers/channel_provider.py b/sublime/Packages/Package Control/package_control/providers/channel_provider.py new file mode 100644 index 0000000..5543bdc --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/channel_provider.py @@ -0,0 +1,312 @@ +import json +import os +import re + +try: + # Python 3 + from urllib.parse import urlparse +except (ImportError): + # Python 2 + from urlparse import urlparse + +from ..console_write import console_write +from .release_selector import ReleaseSelector +from .provider_exception import ProviderException +from ..downloaders.downloader_exception import DownloaderException +from ..clients.client_exception import ClientException +from ..download_manager import downloader + + +class ChannelProvider(ReleaseSelector): + """ + Retrieves a channel and provides an API into the information + + The current channel/repository infrastructure caches repository info into + the channel to improve the Package Control client performance. This also + has the side effect of lessening the load on the GitHub and BitBucket APIs + and getting around not-infrequent HTTP 503 errors from those APIs. + + :param channel: + The URL of the channel + + :param settings: + A dict containing at least the following fields: + `cache_length`, + `debug`, + `timeout`, + `user_agent` + Optional fields: + `http_proxy`, + `https_proxy`, + `proxy_username`, + `proxy_password`, + `query_string_params` + `install_prereleases` + """ + + def __init__(self, channel, settings): + self.channel_info = None + self.schema_version = 0.0 + self.channel = channel + self.settings = settings + self.unavailable_packages = [] + + @classmethod + def match_url(cls, channel): + """Indicates if this provider can handle the provided channel""" + + return True + + def prefetch(self): + """ + Go out and perform HTTP operations, caching the result + + :raises: + ProviderException: when an error occurs trying to open a file + DownloaderException: when an error occurs trying to open a URL + """ + + self.fetch() + + def fetch(self): + """ + Retrieves and loads the JSON for other methods to use + + :raises: + ProviderException: when an error occurs with the channel contents + DownloaderException: when an error occurs trying to open a URL + """ + + if self.channel_info != None: + return + + if re.match('https?://', self.channel, re.I): + with downloader(self.channel, self.settings) as manager: + channel_json = manager.fetch(self.channel, + 'Error downloading channel.') + + # All other channels are expected to be filesystem paths + else: + if not os.path.exists(self.channel): + raise ProviderException(u'Error, file %s does not exist' % self.channel) + + if self.settings.get('debug'): + console_write(u'Loading %s as a channel' % self.channel, True) + + # We open as binary so we get bytes like the DownloadManager + with open(self.channel, 'rb') as f: + channel_json = f.read() + + try: + channel_info = json.loads(channel_json.decode('utf-8')) + except (ValueError): + raise ProviderException(u'Error parsing JSON from channel %s.' % self.channel) + + schema_error = u'Channel %s does not appear to be a valid channel file because ' % self.channel + + if 'schema_version' not in channel_info: + raise ProviderException(u'%s the "schema_version" JSON key is missing.' % schema_error) + + try: + self.schema_version = float(channel_info.get('schema_version')) + except (ValueError): + raise ProviderException(u'%s the "schema_version" is not a valid number.' % schema_error) + + if self.schema_version not in [1.0, 1.1, 1.2, 2.0]: + raise ProviderException(u'%s the "schema_version" is not recognized. Must be one of: 1.0, 1.1, 1.2 or 2.0.' % schema_error) + + self.channel_info = channel_info + + def get_name_map(self): + """ + :raises: + ProviderException: when an error occurs with the channel contents + DownloaderException: when an error occurs trying to open a URL + + :return: + A dict of the mapping for URL slug -> package name + """ + + self.fetch() + + if self.schema_version >= 2.0: + return {} + + return self.channel_info.get('package_name_map', {}) + + def get_renamed_packages(self): + """ + :raises: + ProviderException: when an error occurs with the channel contents + DownloaderException: when an error occurs trying to open a URL + + :return: + A dict of the packages that have been renamed + """ + + self.fetch() + + if self.schema_version >= 2.0: + return {} + + return self.channel_info.get('renamed_packages', {}) + + def get_repositories(self): + """ + :raises: + ProviderException: when an error occurs with the channel contents + DownloaderException: when an error occurs trying to open a URL + + :return: + A list of the repository URLs + """ + + self.fetch() + + if 'repositories' not in self.channel_info: + raise ProviderException(u'Channel %s does not appear to be a valid channel file because the "repositories" JSON key is missing.' % self.channel) + + # Determine a relative root so repositories can be defined + # relative to the location of the channel file. + if re.match('https?://', self.channel, re.I): + url_pieces = urlparse(self.channel) + domain = url_pieces.scheme + '://' + url_pieces.netloc + path = '/' if url_pieces.path == '' else url_pieces.path + if path[-1] != '/': + path = os.path.dirname(path) + relative_base = domain + path + else: + relative_base = os.path.dirname(self.channel) + '/' + + output = [] + repositories = self.channel_info.get('repositories', []) + for repository in repositories: + if re.match('^\./|\.\./', repository): + repository = os.path.normpath(relative_base + repository) + output.append(repository) + + return output + + def get_certs(self): + """ + Provides a secure way for distribution of SSL CA certificates + + Unfortunately Python does not include a bundle of CA certs with urllib + to perform SSL certificate validation. To circumvent this issue, + Package Control acts as a distributor of the CA certs for all HTTPS + URLs of package downloads. + + The default channel scrapes and caches info about all packages + periodically, and in the process it checks the CA certs for all of + the HTTPS URLs listed in the repositories. The contents of the CA cert + files are then hashed, and the CA cert is stored in a filename with + that hash. This is a fingerprint to ensure that Package Control has + the appropriate CA cert for a domain name. + + Next, the default channel file serves up a JSON object of the domain + names and the hashes of their current CA cert files. If Package Control + does not have the appropriate hash for a domain, it may retrieve it + from the channel server. To ensure that Package Control is talking to + a trusted authority to get the CA certs from, the CA cert for + sublime.wbond.net is bundled with Package Control. Then when downloading + the channel file, Package Control can ensure that the channel file's + SSL certificate is valid, thus ensuring the resulting CA certs are + legitimate. + + As a matter of optimization, the distribution of Package Control also + includes the current CA certs for all known HTTPS domains that are + included in the channel, as of the time when Package Control was + last released. + + :raises: + ProviderException: when an error occurs with the channel contents + DownloaderException: when an error occurs trying to open a URL + + :return: + A dict of {'Domain Name': ['cert_file_hash', 'cert_file_download_url']} + """ + + self.fetch() + + return self.channel_info.get('certs', {}) + + def get_packages(self, repo): + """ + Provides access to the repository info that is cached in a channel + + :param repo: + The URL of the repository to get the cached info of + + :raises: + ProviderException: when an error occurs with the channel contents + DownloaderException: when an error occurs trying to open a URL + + :return: + A dict in the format: + { + 'Package Name': { + 'name': name, + 'description': description, + 'author': author, + 'homepage': homepage, + 'last_modified': last modified date, + 'download': { + 'url': url, + 'date': date, + 'version': version + }, + 'previous_names': [old_name, ...], + 'labels': [label, ...], + 'readme': url, + 'issues': url, + 'donate': url, + 'buy': url + }, + ... + } + """ + + self.fetch() + + # The 2.0 channel schema renamed the key cached package info was + # stored under in order to be more clear to new users. + packages_key = 'packages_cache' if self.schema_version >= 2.0 else 'packages' + + if self.channel_info.get(packages_key, False) == False: + return {} + + if self.channel_info[packages_key].get(repo, False) == False: + return {} + + output = {} + for package in self.channel_info[packages_key][repo]: + copy = package.copy() + + # In schema version 2.0, we store a list of dicts containing info + # about all available releases. These include "version" and + # "platforms" keys that are used to pick the download for the + # current machine. + if self.schema_version >= 2.0: + copy = self.select_release(copy) + else: + copy = self.select_platform(copy) + + if not copy: + self.unavailable_packages.append(package['name']) + continue + + output[copy['name']] = copy + + return output + + def get_unavailable_packages(self): + """ + Provides a list of packages that are unavailable for the current + platform/architecture that Sublime Text is running on. + + This list will be empty unless get_packages() is called first. + + :return: A list of package names + """ + + return self.unavailable_packages diff --git a/sublime/Packages/Package Control/package_control/providers/github_repository_provider.py b/sublime/Packages/Package Control/package_control/providers/github_repository_provider.py new file mode 100644 index 0000000..158c850 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/github_repository_provider.py @@ -0,0 +1,169 @@ +import re + +from ..clients.github_client import GitHubClient +from ..downloaders.downloader_exception import DownloaderException +from ..clients.client_exception import ClientException +from .provider_exception import ProviderException + + +class GitHubRepositoryProvider(): + """ + Allows using a public GitHub repository as the source for a single package. + For legacy purposes, this can also be treated as the source for a Package + Control "repository". + + :param repo: + The public web URL to the GitHub repository. Should be in the format + `https://github.com/user/package` for the master branch, or + `https://github.com/user/package/tree/{branch_name}` for any other + branch. + + :param settings: + A dict containing at least the following fields: + `cache_length`, + `debug`, + `timeout`, + `user_agent` + Optional fields: + `http_proxy`, + `https_proxy`, + `proxy_username`, + `proxy_password`, + `query_string_params` + `install_prereleases` + """ + + def __init__(self, repo, settings): + self.cache = {} + # Clean off the trailing .git to be more forgiving + self.repo = re.sub('\.git$', '', repo) + self.settings = settings + self.failed_sources = {} + + @classmethod + def match_url(cls, repo): + """Indicates if this provider can handle the provided repo""" + + master = re.search('^https?://github.com/[^/]+/[^/]+/?$', repo) + branch = re.search('^https?://github.com/[^/]+/[^/]+/tree/[^/]+/?$', + repo) + return master != None or branch != None + + def prefetch(self): + """ + Go out and perform HTTP operations, caching the result + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + """ + + [name for name, info in self.get_packages()] + + def get_failed_sources(self): + """ + List of any URLs that could not be accessed while accessing this repository + + :return: + A generator of ("https://github.com/user/repo", Exception()) tuples + """ + + return self.failed_sources.items() + + def get_broken_packages(self): + """ + For API-compatibility with RepositoryProvider + """ + + return {}.items() + + def get_packages(self, invalid_sources=None): + """ + Uses the GitHub API to construct necessary info for a package + + :param invalid_sources: + A list of URLs that should be ignored + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + + :return: + A generator of + ( + 'Package Name', + { + 'name': name, + 'description': description, + 'author': author, + 'homepage': homepage, + 'last_modified': last modified date, + 'download': { + 'url': url, + 'date': date, + 'version': version + }, + 'previous_names': [], + 'labels': [], + 'sources': [the repo URL], + 'readme': url, + 'issues': url, + 'donate': url, + 'buy': None + } + ) + tuples + """ + + if 'get_packages' in self.cache: + for key, value in self.cache['get_packages'].items(): + yield (key, value) + return + + client = GitHubClient(self.settings) + + if invalid_sources != None and self.repo in invalid_sources: + raise StopIteration() + + try: + repo_info = client.repo_info(self.repo) + download = client.download_info(self.repo) + + name = repo_info['name'] + details = { + 'name': name, + 'description': repo_info['description'], + 'homepage': repo_info['homepage'], + 'author': repo_info['author'], + 'last_modified': download.get('date'), + 'download': download, + 'previous_names': [], + 'labels': [], + 'sources': [self.repo], + 'readme': repo_info['readme'], + 'issues': repo_info['issues'], + 'donate': repo_info['donate'], + 'buy': None + } + self.cache['get_packages'] = {name: details} + yield (name, details) + + except (DownloaderException, ClientException, ProviderException) as e: + self.failed_sources[self.repo] = e + self.cache['get_packages'] = {} + raise StopIteration() + + def get_renamed_packages(self): + """For API-compatibility with RepositoryProvider""" + + return {} + + def get_unavailable_packages(self): + """ + Method for compatibility with RepositoryProvider class. These providers + are based on API calls, and thus do not support different platform + downloads, making it impossible for there to be unavailable packages. + + :return: An empty list + """ + return [] diff --git a/sublime/Packages/Package Control/package_control/providers/github_user_provider.py b/sublime/Packages/Package Control/package_control/providers/github_user_provider.py new file mode 100644 index 0000000..6af60be --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/github_user_provider.py @@ -0,0 +1,172 @@ +import re + +from ..clients.github_client import GitHubClient +from ..downloaders.downloader_exception import DownloaderException +from ..clients.client_exception import ClientException +from .provider_exception import ProviderException + + +class GitHubUserProvider(): + """ + Allows using a GitHub user/organization as the source for multiple packages, + or in Package Control terminology, a "repository". + + :param repo: + The public web URL to the GitHub user/org. Should be in the format + `https://github.com/user`. + + :param settings: + A dict containing at least the following fields: + `cache_length`, + `debug`, + `timeout`, + `user_agent`, + Optional fields: + `http_proxy`, + `https_proxy`, + `proxy_username`, + `proxy_password`, + `query_string_params` + `install_prereleases` + """ + + def __init__(self, repo, settings): + self.cache = {} + self.repo = repo + self.settings = settings + self.failed_sources = {} + + @classmethod + def match_url(cls, repo): + """Indicates if this provider can handle the provided repo""" + + return re.search('^https?://github.com/[^/]+/?$', repo) != None + + def prefetch(self): + """ + Go out and perform HTTP operations, caching the result + """ + + [name for name, info in self.get_packages()] + + def get_failed_sources(self): + """ + List of any URLs that could not be accessed while accessing this repository + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + + :return: + A generator of ("https://github.com/user/repo", Exception()) tuples + """ + + return self.failed_sources.items() + + def get_broken_packages(self): + """ + For API-compatibility with RepositoryProvider + """ + + return {}.items() + + def get_packages(self, invalid_sources=None): + """ + Uses the GitHub API to construct necessary info for all packages + + :param invalid_sources: + A list of URLs that should be ignored + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + + :return: + A generator of + ( + 'Package Name', + { + 'name': name, + 'description': description, + 'author': author, + 'homepage': homepage, + 'last_modified': last modified date, + 'download': { + 'url': url, + 'date': date, + 'version': version + }, + 'previous_names': [], + 'labels': [], + 'sources': [the user URL], + 'readme': url, + 'issues': url, + 'donate': url, + 'buy': None + } + ) + tuples + """ + + if 'get_packages' in self.cache: + for key, value in self.cache['get_packages'].items(): + yield (key, value) + return + + client = GitHubClient(self.settings) + + if invalid_sources != None and self.repo in invalid_sources: + raise StopIteration() + + try: + user_repos = client.user_info(self.repo) + except (DownloaderException, ClientException, ProviderException) as e: + self.failed_sources = [self.repo] + self.cache['get_packages'] = e + raise e + + output = {} + for repo_info in user_repos: + try: + name = repo_info['name'] + repo_url = 'https://github.com/' + repo_info['user_repo'] + + download = client.download_info(repo_url) + + details = { + 'name': name, + 'description': repo_info['description'], + 'homepage': repo_info['homepage'], + 'author': repo_info['author'], + 'last_modified': download.get('date'), + 'download': download, + 'previous_names': [], + 'labels': [], + 'sources': [self.repo], + 'readme': repo_info['readme'], + 'issues': repo_info['issues'], + 'donate': repo_info['donate'], + 'buy': None + } + output[name] = details + yield (name, details) + + except (DownloaderException, ClientException, ProviderException) as e: + self.failed_sources[repo_url] = e + + self.cache['get_packages'] = output + + def get_renamed_packages(self): + """For API-compatibility with RepositoryProvider""" + + return {} + + def get_unavailable_packages(self): + """ + Method for compatibility with RepositoryProvider class. These providers + are based on API calls, and thus do not support different platform + downloads, making it impossible for there to be unavailable packages. + + :return: An empty list + """ + return [] diff --git a/sublime/Packages/Package Control/package_control/providers/provider_exception.py b/sublime/Packages/Package Control/package_control/providers/provider_exception.py new file mode 100644 index 0000000..e98295f --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/provider_exception.py @@ -0,0 +1,5 @@ +class ProviderException(Exception): + """If a provider could not return information""" + + def __str__(self): + return self.args[0] diff --git a/sublime/Packages/Package Control/package_control/providers/release_selector.py b/sublime/Packages/Package Control/package_control/providers/release_selector.py new file mode 100644 index 0000000..5305468 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/release_selector.py @@ -0,0 +1,125 @@ +import re +import sublime + +from ..versions import version_sort, version_exclude_prerelease + + +class ReleaseSelector(): + """ + A base class for finding the best version of a package for the current machine + """ + + def select_release(self, package_info): + """ + Returns a modified package info dict for package from package schema version 2.0 + + :param package_info: + A package info dict with a "releases" key + + :return: + The package info dict with the "releases" key deleted, and a + "download" key added that contains a dict with "version", "url" and + "date" keys. + None if no compatible relases are available. + """ + + releases = version_sort(package_info['releases']) + if not self.settings.get('install_prereleases'): + releases = version_exclude_prerelease(releases) + + for release in releases: + platforms = release.get('platforms', '*') + if not isinstance(platforms, list): + platforms = [platforms] + + best_platform = self.get_best_platform(platforms) + if not best_platform: + continue + + if not self.is_compatible_version(release.get('sublime_text', '<3000')): + continue + + package_info['download'] = release + package_info['last_modified'] = release.get('date') + del package_info['releases'] + + return package_info + + return None + + def select_platform(self, package_info): + """ + Returns a modified package info dict for package from package schema version <= 1.2 + + :param package_info: + A package info dict with a "platforms" key + + :return: + The package info dict with the "platforms" key deleted, and a + "download" key added that contains a dict with "version" and "url" + keys. + None if no compatible platforms. + """ + platforms = list(package_info['platforms'].keys()) + best_platform = self.get_best_platform(platforms) + if not best_platform: + return None + + package_info['download'] = package_info['platforms'][best_platform][0] + package_info['download']['date'] = package_info.get('last_modified') + del package_info['platforms'] + + return package_info + + def get_best_platform(self, platforms): + """ + Returns the most specific platform that matches the current machine + + :param platforms: + An array of platform names for a package. E.g. ['*', 'windows', 'linux-x64'] + + :return: A string reprenting the most specific matching platform + """ + + ids = [sublime.platform() + '-' + sublime.arch(), sublime.platform(), + '*'] + + for id in ids: + if id in platforms: + return id + + return None + + def is_compatible_version(self, version_range): + min_version = float("-inf") + max_version = float("inf") + + if version_range == '*': + return True + + gt_match = re.match('>(\d+)$', version_range) + ge_match = re.match('>=(\d+)$', version_range) + lt_match = re.match('<(\d+)$', version_range) + le_match = re.match('<=(\d+)$', version_range) + range_match = re.match('(\d+) - (\d+)$', version_range) + + if gt_match: + min_version = int(gt_match.group(1)) + 1 + elif ge_match: + min_version = int(ge_match.group(1)) + elif lt_match: + max_version = int(lt_match.group(1)) - 1 + elif le_match: + max_version = int(le_match.group(1)) + elif range_match: + min_version = int(range_match.group(1)) + max_version = int(range_match.group(2)) + else: + return None + + if min_version > int(sublime.version()): + return False + if max_version < int(sublime.version()): + return False + + return True diff --git a/sublime/Packages/Package Control/package_control/providers/repository_provider.py b/sublime/Packages/Package Control/package_control/providers/repository_provider.py new file mode 100644 index 0000000..01a5ad9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/providers/repository_provider.py @@ -0,0 +1,441 @@ +import json +import re +import os +from itertools import chain + +try: + # Python 3 + from urllib.parse import urlparse +except (ImportError): + # Python 2 + from urlparse import urlparse + +from ..console_write import console_write +from .release_selector import ReleaseSelector +from .provider_exception import ProviderException +from ..downloaders.downloader_exception import DownloaderException +from ..clients.client_exception import ClientException +from ..clients.github_client import GitHubClient +from ..clients.bitbucket_client import BitBucketClient +from ..download_manager import downloader + + +class RepositoryProvider(ReleaseSelector): + """ + Generic repository downloader that fetches package info + + With the current channel/repository architecture where the channel file + caches info from all includes repositories, these package providers just + serve the purpose of downloading packages not in the default channel. + + The structure of the JSON a repository should contain is located in + example-packages.json. + + :param repo: + The URL of the package repository + + :param settings: + A dict containing at least the following fields: + `cache_length`, + `debug`, + `timeout`, + `user_agent` + Optional fields: + `http_proxy`, + `https_proxy`, + `proxy_username`, + `proxy_password`, + `query_string_params` + `install_prereleases` + """ + + def __init__(self, repo, settings): + self.cache = {} + self.repo_info = None + self.schema_version = 0.0 + self.repo = repo + self.settings = settings + self.unavailable_packages = [] + self.failed_sources = {} + self.broken_packages = {} + + @classmethod + def match_url(cls, repo): + """Indicates if this provider can handle the provided repo""" + + return True + + def prefetch(self): + """ + Go out and perform HTTP operations, caching the result + + :raises: + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + """ + + [name for name, info in self.get_packages()] + + def get_failed_sources(self): + """ + List of any URLs that could not be accessed while accessing this repository + + :return: + A generator of ("https://example.com", Exception()) tuples + """ + + return self.failed_sources.items() + + def get_broken_packages(self): + """ + List of package names for packages that are missing information + + :return: + A generator of ("Package Name", Exception()) tuples + """ + + return self.broken_packages.items() + + def fetch(self): + """ + Retrieves and loads the JSON for other methods to use + + :raises: + ProviderException: when an error occurs trying to open a file + DownloaderException: when an error occurs trying to open a URL + """ + + if self.repo_info != None: + return + + self.repo_info = self.fetch_location(self.repo) + + if 'includes' not in self.repo_info: + return + + # Allow repositories to include other repositories + if re.match('https?://', self.repo, re.I): + url_pieces = urlparse(self.repo) + domain = url_pieces.scheme + '://' + url_pieces.netloc + path = '/' if url_pieces.path == '' else url_pieces.path + if path[-1] != '/': + path = os.path.dirname(path) + relative_base = domain + path + else: + relative_base = os.path.dirname(self.repo) + '/' + + includes = self.repo_info.get('includes', []) + del self.repo_info['includes'] + for include in includes: + if re.match('^\./|\.\./', include): + include = os.path.normpath(relative_base + include) + include_info = self.fetch_location(include) + included_packages = include_info.get('packages', []) + self.repo_info['packages'].extend(included_packages) + + def fetch_location(self, location): + """ + Fetches the contents of a URL of file path + + :param location: + The URL or file path + + :raises: + ProviderException: when an error occurs trying to open a file + DownloaderException: when an error occurs trying to open a URL + + :return: + A dict of the parsed JSON + """ + + if re.match('https?://', self.repo, re.I): + with downloader(location, self.settings) as manager: + json_string = manager.fetch(location, 'Error downloading repository.') + + # Anything that is not a URL is expected to be a filesystem path + else: + if not os.path.exists(location): + raise ProviderException(u'Error, file %s does not exist' % location) + + if self.settings.get('debug'): + console_write(u'Loading %s as a repository' % location, True) + + # We open as binary so we get bytes like the DownloadManager + with open(location, 'rb') as f: + json_string = f.read() + + try: + return json.loads(json_string.decode('utf-8')) + except (ValueError): + raise ProviderException(u'Error parsing JSON from repository %s.' % location) + + def get_packages(self, invalid_sources=None): + """ + Provides access to the packages in this repository + + :param invalid_sources: + A list of URLs that are permissible to fetch data from + + :raises: + ProviderException: when an error occurs trying to open a file + DownloaderException: when there is an issue download package info + ClientException: when there is an issue parsing package info + + :return: + A generator of + ( + 'Package Name', + { + 'name': name, + 'description': description, + 'author': author, + 'homepage': homepage, + 'last_modified': last modified date, + 'download': { + 'url': url, + 'date': date, + 'version': version + }, + 'previous_names': [old_name, ...], + 'labels': [label, ...], + 'sources': [url, ...], + 'readme': url, + 'issues': url, + 'donate': url, + 'buy': url + } + ) + tuples + """ + + if 'get_packages' in self.cache: + for key, value in self.cache['get_packages'].items(): + yield (key, value) + return + + if invalid_sources != None and self.repo in invalid_sources: + raise StopIteration() + + self.fetch() + + def fail(message): + exception = ProviderException(message) + self.failed_sources[self.repo] = exception + self.cache['get_packages'] = {} + return + schema_error = u'Repository %s does not appear to be a valid repository file because ' % self.repo + + if 'schema_version' not in self.repo_info: + error_string = u'%s the "schema_version" JSON key is missing.' % schema_error + fail(error_string) + return + + try: + self.schema_version = float(self.repo_info.get('schema_version')) + except (ValueError): + error_string = u'%s the "schema_version" is not a valid number.' % schema_error + fail(error_string) + return + + if self.schema_version not in [1.0, 1.1, 1.2, 2.0]: + error_string = u'%s the "schema_version" is not recognized. Must be one of: 1.0, 1.1, 1.2 or 2.0.' % schema_error + fail(error_string) + return + + if 'packages' not in self.repo_info: + error_string = u'%s the "packages" JSON key is missing.' % schema_error + fail(error_string) + return + + github_client = GitHubClient(self.settings) + bitbucket_client = BitBucketClient(self.settings) + + # Backfill the "previous_names" keys for old schemas + previous_names = {} + if self.schema_version < 2.0: + renamed = self.get_renamed_packages() + for old_name in renamed: + new_name = renamed[old_name] + if new_name not in previous_names: + previous_names[new_name] = [] + previous_names[new_name].append(old_name) + + output = {} + for package in self.repo_info['packages']: + info = { + 'sources': [self.repo] + } + + for field in ['name', 'description', 'author', 'last_modified', 'previous_names', + 'labels', 'homepage', 'readme', 'issues', 'donate', 'buy']: + if package.get(field): + info[field] = package.get(field) + + # Schema version 2.0 allows for grabbing details about a pacakge, or its + # download from "details" urls. See the GitHubClient and BitBucketClient + # classes for valid URLs. + if self.schema_version >= 2.0: + details = package.get('details') + releases = package.get('releases') + + # Try to grab package-level details from GitHub or BitBucket + if details: + if invalid_sources != None and details in invalid_sources: + continue + + info['sources'].append(details) + + try: + github_repo_info = github_client.repo_info(details) + bitbucket_repo_info = bitbucket_client.repo_info(details) + + # When grabbing details, prefer explicit field values over the values + # from the GitHub or BitBucket API + if github_repo_info: + info = dict(chain(github_repo_info.items(), info.items())) + elif bitbucket_repo_info: + info = dict(chain(bitbucket_repo_info.items(), info.items())) + else: + raise ProviderException(u'Invalid "details" value "%s" for one of the packages in the repository %s.' % (details, self.repo)) + + except (DownloaderException, ClientException, ProviderException) as e: + if 'name' in info: + self.broken_packages[info['name']] = e + self.failed_sources[details] = e + continue + + # If no releases info was specified, also grab the download info from GH or BB + if not releases and details: + releases = [{'details': details}] + + # This allows developers to specify a GH or BB location to get releases from, + # especially tags URLs (https://github.com/user/repo/tags or + # https://bitbucket.org/user/repo#tags) + info['releases'] = [] + for release in releases: + download_details = None + download_info = {} + + # Make sure that explicit fields are copied over + for field in ['platforms', 'sublime_text', 'version', 'url', 'date']: + if field in release: + download_info[field] = release[field] + + if 'details' in release: + download_details = release['details'] + + try: + github_download = github_client.download_info(download_details) + bitbucket_download = bitbucket_client.download_info(download_details) + + # Overlay the explicit field values over values fetched from the APIs + if github_download: + download_info = dict(chain(github_download.items(), download_info.items())) + # No matching tags + elif github_download == False: + download_info = {} + elif bitbucket_download: + download_info = dict(chain(bitbucket_download.items(), download_info.items())) + # No matching tags + elif bitbucket_download == False: + download_info = {} + else: + raise ProviderException(u'Invalid "details" value "%s" under the "releases" key for the package "%s" in the repository %s.' % (download_details, info['name'], self.repo)) + + except (DownloaderException, ClientException, ProviderException) as e: + if 'name' in info: + self.broken_packages[info['name']] = e + self.failed_sources[download_details] = e + continue + + if download_info: + info['releases'].append(download_info) + + info = self.select_release(info) + + # Schema version 1.0, 1.1 and 1.2 just require that all values be + # explicitly specified in the package JSON + else: + info['platforms'] = package.get('platforms') + info = self.select_platform(info) + + if not info: + self.unavailable_packages.append(package['name']) + continue + + if 'download' not in info and 'releases' not in info: + self.broken_packages[info['name']] = ProviderException(u'No "releases" key for the package "%s" in the repository %s.' % (info['name'], self.repo)) + continue + + for field in ['previous_names', 'labels']: + if field not in info: + info[field] = [] + + for field in ['readme', 'issues', 'donate', 'buy']: + if field not in info: + info[field] = None + + if 'homepage' not in info: + info['homepage'] = self.repo + + if 'download' in info: + # Rewrites the legacy "zipball" URLs to the new "zip" format + info['download']['url'] = re.sub( + '^(https://nodeload.github.com/[^/]+/[^/]+/)zipball(/.*)$', + '\\1zip\\2', info['download']['url']) + + # Rewrites the legacy "nodeload" URLs to the new "codeload" subdomain + info['download']['url'] = info['download']['url'].replace( + 'nodeload.github.com', 'codeload.github.com') + + # Extract the date from the download + if 'last_modified' not in info: + info['last_modified'] = info['download']['date'] + + elif 'releases' in info and 'last_modified' not in info: + # Extract a date from the newest download + date = '1970-01-01 00:00:00' + for release in info['releases']: + if 'date' in release and release['date'] > date: + date = release['date'] + info['last_modified'] = date + + if info['name'] in previous_names: + info['previous_names'].extend(previous_names[info['name']]) + + output[info['name']] = info + yield (info['name'], info) + + self.cache['get_packages'] = output + + def get_renamed_packages(self): + """:return: A dict of the packages that have been renamed""" + + if self.schema_version < 2.0: + return self.repo_info.get('renamed_packages', {}) + + output = {} + for package in self.repo_info['packages']: + if 'previous_names' not in package: + continue + + previous_names = package['previous_names'] + if not isinstance(previous_names, list): + previous_names = [previous_names] + + for previous_name in previous_names: + output[previous_name] = package['name'] + + return output + + def get_unavailable_packages(self): + """ + Provides a list of packages that are unavailable for the current + platform/architecture that Sublime Text is running on. + + This list will be empty unless get_packages() is called first. + + :return: A list of package names + """ + + return self.unavailable_packages diff --git a/sublime/Packages/Package Control/package_control/reloader.py b/sublime/Packages/Package Control/package_control/reloader.py new file mode 100644 index 0000000..0696022 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/reloader.py @@ -0,0 +1,130 @@ +import sys + +import sublime + + +st_version = 2 +# With the way ST3 works, the sublime module is not "available" at startup +# which results in an empty version number +if sublime.version() == '' or int(sublime.version()) > 3000: + st_version = 3 + from imp import reload + + +# Python allows reloading modules on the fly, which allows us to do live upgrades. +# The only caveat to this is that you have to reload in the dependency order. +# +# Thus is module A depends on B and we don't reload B before A, when A is reloaded +# it will still have a reference to the old B. Thus we hard-code the dependency +# order of the various Package Control modules so they get reloaded properly. +# +# There are solutions for doing this all programatically, but this is much easier +# to understand. + +reload_mods = [] +for mod in sys.modules: + if mod[0:15].lower().replace(' ', '_') == 'package_control' and sys.modules[mod] != None: + reload_mods.append(mod) + +mod_prefix = 'package_control' +if st_version == 3: + mod_prefix = 'Package Control.' + mod_prefix + +mods_load_order = [ + '', + + '.sys_path', + '.cache', + '.http_cache', + '.ca_certs', + '.clear_directory', + '.cmd', + '.console_write', + '.preferences_filename', + '.show_error', + '.unicode', + '.thread_progress', + '.package_io', + '.semver', + '.versions', + + '.http', + '.http.invalid_certificate_exception', + '.http.debuggable_http_response', + '.http.debuggable_https_response', + '.http.debuggable_http_connection', + '.http.persistent_handler', + '.http.debuggable_http_handler', + '.http.validating_https_connection', + '.http.validating_https_handler', + + '.clients', + '.clients.client_exception', + '.clients.bitbucket_client', + '.clients.github_client', + '.clients.readme_client', + '.clients.json_api_client', + + '.providers', + '.providers.provider_exception', + '.providers.bitbucket_repository_provider', + '.providers.channel_provider', + '.providers.github_repository_provider', + '.providers.github_user_provider', + '.providers.repository_provider', + '.providers.release_selector', + + '.download_manager', + + '.downloaders', + '.downloaders.downloader_exception', + '.downloaders.rate_limit_exception', + '.downloaders.binary_not_found_error', + '.downloaders.non_clean_exit_error', + '.downloaders.non_http_error', + '.downloaders.caching_downloader', + '.downloaders.decoding_downloader', + '.downloaders.limiting_downloader', + '.downloaders.cert_provider', + '.downloaders.urllib_downloader', + '.downloaders.cli_downloader', + '.downloaders.curl_downloader', + '.downloaders.wget_downloader', + '.downloaders.wininet_downloader', + '.downloaders.background_downloader', + + '.upgraders', + '.upgraders.vcs_upgrader', + '.upgraders.git_upgrader', + '.upgraders.hg_upgrader', + + '.package_manager', + '.package_creator', + '.package_installer', + '.package_renamer', + + '.commands', + '.commands.add_channel_command', + '.commands.add_repository_command', + '.commands.create_binary_package_command', + '.commands.create_package_command', + '.commands.disable_package_command', + '.commands.discover_packages_command', + '.commands.enable_package_command', + '.commands.existing_packages_command', + '.commands.grab_certs_command', + '.commands.install_package_command', + '.commands.list_packages_command', + '.commands.package_message_command', + '.commands.remove_package_command', + '.commands.upgrade_all_packages_command', + '.commands.upgrade_package_command', + + '.package_cleanup', + '.automatic_upgrader' +] + +for suffix in mods_load_order: + mod = mod_prefix + suffix + if mod in reload_mods: + reload(sys.modules[mod]) diff --git a/sublime/Packages/Package Control/package_control/semver.py b/sublime/Packages/Package Control/package_control/semver.py new file mode 100644 index 0000000..917fa77 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/semver.py @@ -0,0 +1,833 @@ +"""pysemver: Semantic Version comparing for Python. + +Provides comparing of semantic versions by using SemVer objects using rich comperations plus the +possibility to match a selector string against versions. Interesting for version dependencies. +Versions look like: "1.7.12+b.133" +Selectors look like: ">1.7.0 || 1.6.9+b.111 - 1.6.9+b.113" + +Example usages: + >>> SemVer(1, 2, 3, build=13) + SemVer("1.2.3+13") + >>> SemVer.valid("1.2.3.4") + False + >>> SemVer.clean("this is unimportant text 1.2.3-2 and will be stripped") + "1.2.3-2" + >>> SemVer("1.7.12+b.133").satisfies(">1.7.0 || 1.6.9+b.111 - 1.6.9+b.113") + True + >>> SemSel(">1.7.0 || 1.6.9+b.111 - 1.6.9+b.113").matches(SemVer("1.7.12+b.133"), + ... SemVer("1.6.9+b.112"), SemVer("1.6.10")) + [SemVer("1.7.12+b.133"), SemVer("1.6.9+b.112")] + >>> min(_) + SemVer("1.6.9+b.112") + >>> _.patch + 9 + +Exported classes: + * SemVer(collections.namedtuple()) + Parses semantic versions and defines methods for them. Supports rich comparisons. + * SemSel(tuple) + Parses semantic version selector strings and defines methods for them. + * SelParseError(Exception) + An error among others raised when parsing a semantic version selector failed. + +Other classes: + * SemComparator(object) + * SemSelAndChunk(list) + * SemSelOrChunk(list) + +Functions/Variables/Constants: + none + + +Copyright (c) 2013 Zachary King, FichteFoll + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and +associated documentation files (the "Software"), to deal in the Software without restriction, +including without limitation the rights to use, copy, modify, merge, publish, distribute, +sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: The above copyright notice and this +permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT +NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES +OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +""" + +import re +import sys +from collections import namedtuple # Python >=2.6 + + +__all__ = ('SemVer', 'SemSel', 'SelParseError') + + +if sys.version_info[0] == 3: + basestring = str + cmp = lambda a, b: (a > b) - (a < b) + + +# @functools.total_ordering would be nice here but was added in 2.7, __cmp__ is not Py3 +class SemVer(namedtuple("_SemVer", 'major, minor, patch, prerelease, build')): + """Semantic Version, consists of 3 to 5 components defining the version's adicity. + + See http://semver.org/ (2.0.0-rc.1) for the standard mainly used for this implementation, few + changes have been made. + + Information on this particular class and their instances: + - Immutable and hashable. + - Subclasses `collections.namedtuple`. + - Always `True` in boolean context. + - len() returns an int between 3 and 5; 4 when a pre-release is set and 5 when a build is + set. Note: Still returns 5 when build is set but not pre-release. + - Parts of the semantic version can be accessed by integer indexing, key (string) indexing, + slicing and getting an attribute. Returned slices are tuple. Leading '-' and '+' of + optional components are not stripped. Supported keys/attributes: + major, minor, patch, prerelease, build. + + Examples: + s = SemVer("1.2.3-4.5+6") + s[2] == 3 + s[:3] == (1, 2, 3) + s['build'] == '-4.5' + s.major == 1 + + Short information on semantic version structure: + + Semantic versions consist of: + * a major component (numeric) + * a minor component (numeric) + * a patch component (numeric) + * a pre-release component [optional] + * a build component [optional] + + The pre-release component is indicated by a hyphen '-' and followed by alphanumeric[1] sequences + separated by dots '.'. Sequences are compared numerically if applicable (both sequences of two + versions are numeric) or lexicographically. May also include hyphens. The existence of a + pre-release component lowers the actual version; the shorter pre-release component is considered + lower. An 'empty' pre-release component is considered to be the least version for this + major-minor-patch combination (e.g. "1.0.0-"). + + The build component may follow the optional pre-release component and is indicated by a plus '+' + followed by sequences, just as the pre-release component. Comparing works similarly. However the + existence of a build component raises the actual version and may also raise a pre-release. An + 'empty' build component is considered to be the highest version for this + major-minor-patch-prerelease combination (e.g. "1.2.3+"). + + + [1]: Regexp for a sequence: r'[0-9A-Za-z-]+'. + """ + + # Static class variables + _base_regex = r'''(?x) + (?P[0-9]+) + \.(?P[0-9]+) + \.(?P[0-9]+) + (?:\-(?P(?:[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?))? + (?:\+(?P(?:[0-9A-Za-z-]+(?:\.[0-9A-Za-z-]+)*)?))?''' + _search_regex = re.compile(_base_regex) + _match_regex = re.compile('^%s$' % _base_regex) # required because of $ anchor + + # "Constructor" + def __new__(cls, *args, **kwargs): + """There are two different constructor styles that are allowed: + - Option 1 allows specification of a semantic version as a string and the option to "clean" + the string before parsing it. + - Option 2 allows specification of each component separately as one parameter. + + Note that all the parameters specified in the following sections can be passed either as + positional or as named parameters while considering the usual Python rules for this. As + such, `SemVer(1, 2, minor=1)` will result in an exception and not in `SemVer("1.1.2")`. + + Option 1: + Constructor examples: + SemVer("1.0.1") + SemVer("this version 1.0.1-pre.1 here", True) + SemVer(ver="0.0.9-pre-alpha+34", clean=False) + + Parameters: + * ver (str) + The string containing the version. + * clean = `False` (bool; optional) + If this is true in boolean context, `SemVer.clean(ver)` is called before + parsing. + + Option 2: + Constructor examples: + SemVer(1, 0, 1) + SemVer(1, '0', prerelease='pre-alpha', patch=1, build=34) + SemVer(**dict(minor=2, major=1, patch=3)) + + Parameters: + * major (int, str, float ...) + * minor (...) + * patch (...) + Major to patch components must be an integer or convertable to an int (e.g. a + string or another number type). + + * prerelease = `None` (str, int, float ...; optional) + * build = `None` (...; optional) + Pre-release and build components should be a string (or number) type. + Will be passed to `str()` if not already a string but the final string must + match '^[0-9A-Za-z.-]*$' + + Raises: + * TypeError + Invalid parameter type(s) or combination (e.g. option 1 and 2). + * ValueError + Invalid semantic version or option 2 parameters unconvertable. + """ + ver, clean, comps = None, False, None + kw, l = kwargs.copy(), len(args) + len(kwargs) + + def inv(): + raise TypeError("Invalid parameter combination: args=%s; kwargs=%s" % (args, kwargs)) + + # Do validation and parse the parameters + if l == 0 or l > 5: + raise TypeError("SemVer accepts at least 1 and at most 5 arguments (%d given)" % l) + + elif l < 3: + if len(args) == 2: + ver, clean = args + else: + ver = args[0] if args else kw.pop('ver', None) + clean = kw.pop('clean', clean) + if kw: + inv() + + else: + comps = list(args) + [kw.pop(cls._fields[k], None) for k in range(len(args), 5)] + if kw or any(comps[i] is None for i in range(3)): + inv() + + typecheck = (int,) * 3 + (basestring,) * 2 + for i, (v, t) in enumerate(zip(comps, typecheck)): + if v is None: + continue + elif not isinstance(v, t): + try: + if i < 3: + v = typecheck[i](v) + else: # The real `basestring` can not be instatiated (Py2) + v = str(v) + except ValueError as e: + # Modify the exception message. I can't believe this actually works + e.args = ("Parameter #%d must be of type %s or convertable" + % (i, t.__name__),) + raise + else: + comps[i] = v + if t is basestring and not re.match(r"^[0-9A-Za-z.-]*$", v): + raise ValueError("Build and pre-release strings must match '^[0-9A-Za-z.-]*$'") + + # Final adjustments + if not comps: + if ver is None or clean is None: + inv() + ver = clean and cls.clean(ver) or ver + comps = cls._parse(ver) + + # Create the obj + return super(SemVer, cls).__new__(cls, *comps) + + # Magic methods + def __str__(self): + return ('.'.join(map(str, self[:3])) + + ('-' + self.prerelease if self.prerelease is not None else '') + + ('+' + self.build if self.build is not None else '')) + + def __repr__(self): + # Use the shortest representation - what would you prefer? + return 'SemVer("%s")' % str(self) + # return 'SemVer(%s)' % ', '.join('%s=%r' % (k, getattr(self, k)) for k in self._fields) + + def __len__(self): + return 3 + (self.build is not None and 2 or self.prerelease is not None) + + # Magic rich comparing methods + def __gt__(self, other): + return self._compare(other) == 1 if isinstance(other, SemVer) else NotImplemented + + def __eq__(self, other): + return self._compare(other) == 0 if isinstance(other, SemVer) else NotImplemented + + def __lt__(self, other): + return not (self > other or self == other) + + def __ge__(self, other): + return not (self < other) + + def __le__(self, other): + return not (self > other) + + def __ne__(self, other): + return not (self == other) + + # Utility (class-)methods + def satisfies(self, sel): + """Alias for `bool(sel.matches(self))` or `bool(SemSel(sel).matches(self))`. + + See `SemSel.__init__()` and `SemSel.matches(*vers)` for possible exceptions. + + Returns: + * bool: `True` if the version matches the passed selector, `False` otherwise. + """ + if not isinstance(sel, SemSel): + sel = SemSel(sel) # just "re-raise" exceptions + + return bool(sel.matches(self)) + + @classmethod + def valid(cls, ver): + """Check if `ver` is a valid semantic version. Classmethod. + + Parameters: + * ver (str) + The string that should be stripped. + + Raises: + * TypeError + Invalid parameter type. + + Returns: + * bool: `True` if it is valid, `False` otherwise. + """ + if not isinstance(ver, basestring): + raise TypeError("%r is not a string" % ver) + + if cls._match_regex.match(ver): + return True + else: + return False + + @classmethod + def clean(cls, vers): + """Remove everything before and after a valid version string. Classmethod. + + Parameters: + * vers (str) + The string that should be stripped. + + Raises: + * TypeError + Invalid parameter type. + + Returns: + * str: The stripped version string. Only the first version is matched. + * None: No version found in the string. + """ + if not isinstance(vers, basestring): + raise TypeError("%r is not a string" % vers) + m = cls._search_regex.search(vers) + if m: + return vers[m.start():m.end()] + else: + return None + + # Private (class-)methods + @classmethod + def _parse(cls, ver): + """Private. Do not touch. Classmethod. + """ + if not isinstance(ver, basestring): + raise TypeError("%r is not a string" % ver) + + match = cls._match_regex.match(ver) + + if match is None: + raise ValueError("'%s' is not a valid SemVer string" % ver) + + g = list(match.groups()) + for i in range(3): + g[i] = int(g[i]) + + return g # Will be passed as namedtuple(...)(*g) + + def _compare(self, other): + """Private. Do not touch. + self > other: 1 + self = other: 0 + self < other: -1 + """ + # Shorthand lambdas + cp_len = lambda t, i=0: cmp(len(t[i]), len(t[not i])) + + for i, (x1, x2) in enumerate(zip(self, other)): + if i > 2: + if x1 is None and x2 is None: + continue + + # self is greater when other has a prerelease but self doesn't + # self is less when other has a build but self doesn't + if x1 is None or x2 is None: + return int(2 * (i - 3.5)) * (1 - 2 * (x1 is None)) + + # self is less when other's build is empty + if i == 4 and (not x1 or not x2) and x1 != x2: + return 1 - 2 * bool(x1) + + # Split by '.' and use numeric comp or lexicographical order + t2 = [x1.split('.'), x2.split('.')] + for y1, y2 in zip(*t2): + if y1.isdigit() and y2.isdigit(): + y1 = int(y1) + y2 = int(y2) + if y1 > y2: + return 1 + elif y1 < y2: + return -1 + + # The "longer" sub-version is greater + d = cp_len(t2) + if d: + return d + else: + if x1 > x2: + return 1 + elif x1 < x2: + return -1 + + # The versions equal + return 0 + + +class SemComparator(object): + """Holds a SemVer object and a comparing operator and can match these against a given version. + + Constructor: SemComparator('<=', SemVer("1.2.3")) + + Methods: + * matches(ver) + """ + # Private properties + _ops = { + '>=': '__ge__', + '<=': '__le__', + '>': '__gt__', + '<': '__lt__', + '=': '__eq__', + '!=': '__ne__' + } + _ops_satisfy = ('~', '!') + + # Constructor + def __init__(self, op, ver): + """Constructor examples: + SemComparator('<=', SemVer("1.2.3")) + SemComparator('!=', SemVer("2.3.4")) + + Parameters: + * op (str, False, None) + One of [>=, <=, >, <, =, !=, !, ~] or evaluates to `False` which defaults to '~'. + '~' means a "satisfy" operation where pre-releases and builds are ignored. + '!' is a negative "~". + * ver (SemVer) + Holds the version to compare with. + + Raises: + * ValueError + Invalid `op` parameter. + * TypeError + Invalid `ver` parameter. + """ + super(SemComparator, self).__init__() + + if op and op not in self._ops_satisfy and op not in self._ops: + raise ValueError("Invalid value for `op` parameter.") + if not isinstance(ver, SemVer): + raise TypeError("`ver` parameter is not instance of SemVer.") + + # Default to '~' for versions with no build or pre-release + op = op or '~' + # Fallback to '=' and '!=' if len > 3 + if len(ver) != 3: + if op == '~': + op = '=' + if op == '!': + op = '!=' + + self.op = op + self.ver = ver + + # Magic methods + def __str__(self): + return (self.op or "") + str(self.ver) + + # Utility methods + def matches(self, ver): + """Match the internal version (constructor) against `ver`. + + Parameters: + * ver (SemVer) + + Raises: + * TypeError + Could not compare `ver` against the version passed in the constructor with the + passed operator. + + Returns: + * bool + `True` if the version matched the specified operator and internal version, `False` + otherwise. + """ + if self.op in self._ops_satisfy: + # Compare only the first three parts (which are tuples) and directly + return bool((self.ver[:3] == ver[:3]) + (self.op == '!') * -1) + ret = getattr(ver, self._ops[self.op])(self.ver) + if ret == NotImplemented: + raise TypeError("Unable to compare %r with operator '%s'" % (ver, self.op)) + return ret + + +class SemSelAndChunk(list): + """Extends list and defines a few methods used for matching versions. + + New elements should be added by calling `.add_child(op, ver)` which creates a SemComparator + instance and adds that to itself. + + Methods: + * matches(ver) + * add_child(op, ver) + """ + # Magic methods + def __str__(self): + return ' '.join(map(str, self)) + + # Utitlity methods + def matches(self, ver): + """Match all of the added children against `ver`. + + Parameters: + * ver (SemVer) + + Raises: + * TypeError + Invalid `ver` parameter. + + Returns: + * bool: + `True` if *all* of the SemComparator children match `ver`, `False` otherwise. + """ + if not isinstance(ver, SemVer): + raise TypeError("`ver` parameter is not instance of SemVer.") + return all(cp.matches(ver) for cp in self) + + def add_child(self, op, ver): + """Create a SemComparator instance with the given parameters and appends that to self. + + Parameters: + * op (str) + * ver (SemVer) + Both parameters are forwarded to `SemComparator.__init__`, see there for a more detailed + description. + + Raises: + Exceptions raised by `SemComparator.__init__`. + """ + self.append(SemComparator(op, SemVer(ver))) + + +class SemSelOrChunk(list): + """Extends list and defines a few methods used for matching versions. + + New elements should be added by calling `.new_child()` which returns a SemSelAndChunk + instance. + + Methods: + * matches(ver) + * new_child() + """ + # Magic methods + def __str__(self): + return ' || '.join(map(str, self)) + + # Utility methods + def matches(self, ver): + """Match all of the added children against `ver`. + + Parameters: + * ver (SemVer) + + Raises: + * TypeError + Invalid `ver` parameter. + + Returns: + * bool + `True` if *any* of the SemSelAndChunk children matches `ver`. + `False` otherwise. + """ + if not isinstance(ver, SemVer): + raise TypeError("`ver` parameter is not instance of SemVer.") + return any(ch.matches(ver) for ch in self) + + def new_child(self): + """Creates a new SemSelAndChunk instance, appends it to self and returns it. + + Returns: + * SemSelAndChunk: An empty instance. + """ + ch = SemSelAndChunk() + self.append(ch) + return ch + + +class SelParseError(Exception): + """An Exception raised when parsing a semantic selector failed. + """ + pass + + +# Subclass `tuple` because this is a somewhat simple method to make this immutable +class SemSel(tuple): + """A Semantic Version Selector, holds a selector and can match it against semantic versions. + + Information on this particular class and their instances: + - Immutable but not hashable because the content within might have changed. + - Subclasses `tuple` but does not behave like one. + - Always `True` in boolean context. + - len() returns the number of containing *and chunks* (see below). + - Iterable, iterates over containing *and chunks*. + + When talking about "versions" it refers to a semantic version (SemVer). For information on how + versions compare to one another, see SemVer's doc string. + + List for **comparators**: + "1.0.0" matches the version 1.0.0 and all its pre-release and build variants + "!1.0.0" matches any version that is not 1.0.0 or any of its variants + "=1.0.0" matches only the version 1.0.0 + "!=1.0.0" matches any version that is not 1.0.0 + ">=1.0.0" matches versions greater than or equal 1.0.0 + "<1.0.0" matches versions smaller than 1.0.0 + "1.0.0 - 1.0.3" matches versions greater than or equal 1.0.0 thru 1.0.3 + "~1.0" matches versions greater than or equal 1.0.0 thru 1.0.9999 (and more) + "~1", "1.x", "1.*" match versions greater than or equal 1.0.0 thru 1.9999.9999 (and more) + "~1.1.2" matches versions greater than or equal 1.1.2 thru 1.1.9999 (and more) + "~1.1.2+any" matches versions greater than or equal 1.1.2+any thru 1.1.9999 (and more) + "*", "~", "~x" match any version + + Multiple comparators can be combined by using ' ' spaces and every comparator must match to make + the **and chunk** match a version. + Multiple and chunks can be combined to **or chunks** using ' || ' and match if any of the and + chunks split by these matches. + + A complete example would look like: + ~1 || 0.0.3 || <0.0.2 >0.0.1+b.1337 || 2.0.x || 2.1.0 - 2.1.0+b.12 !=2.1.0+b.9 + + Methods: + * matches(*vers) + """ + # Private properties + _fuzzy_regex = re.compile(r'''(?x)^ + (?P[<>]=?|~>?=?)? + (?:(?P\d+) + (?:\.(?P\d+) + (?:\.(?P\d+) + (?P[-+][a-zA-Z0-9-+.]*)? + )? + )? + )?$''') + _xrange_regex = re.compile(r'''(?x)^ + (?P[<>]=?|~>?=?)? + (?:(?P\d+|[xX*]) + (?:\.(?P\d+|[xX*]) + (?:\.(?P\d+|[xX*]))? + )? + ) + (?P.*)$''') + _split_op_regex = re.compile(r'^(?P=|[<>!]=?)?(?P.*)$') + + # "Constructor" + def __new__(cls, sel): + """Constructor examples: + SemSel(">1.0.0") + SemSel("~1.2.9 !=1.2.12") + + Parameters: + * sel (str) + A version selector string. + + Raises: + * TypeError + `sel` parameter is not a string. + * ValueError + A version in the selector could not be matched as a SemVer. + * SemParseError + The version selector's syntax is unparsable; invalid ranges (fuzzy, xrange or + explicit range) or invalid '||' + """ + chunk = cls._parse(sel) + return super(SemSel, cls).__new__(cls, (chunk,)) + + # Magic methods + def __str__(self): + return str(self._chunk) + + def __repr__(self): + return 'SemSel("%s")' % self._chunk + + def __len__(self): + # What would you expect? + return len(self._chunk) + + def __iter__(self): + return iter(self._chunk) + + # Read-only (private) attributes + @property + def _chunk(self): + return self[0] + + # Utility methods + def matches(self, *vers): + """Match the selector against a selection of versions. + + Parameters: + * *vers (str, SemVer) + Versions can be passed as strings and SemVer objects will be created with them. + May also be a mixed list. + + Raises: + * TypeError + A version is not an instance of str (basestring) or SemVer. + * ValueError + A string version could not be parsed as a SemVer. + + Returns: + * list + A list with all the versions that matched, may be empty. Use `max()` to determine + the highest matching version, or `min()` for the lowest. + """ + ret = [] + for v in vers: + if isinstance(v, str): + t = self._chunk.matches(SemVer(v)) + elif isinstance(v, SemVer): + t = self._chunk.matches(v) + else: + raise TypeError("Invalid parameter type '%s': %s" % (v, type(v))) + if t: + ret.append(v) + + return ret + + # Private methods + @classmethod + def _parse(cls, sel): + """Private. Do not touch. + + 1. split by whitespace into tokens + a. start new and_chunk on ' || ' + b. parse " - " ranges + c. replace "xX*" ranges with "~" equivalent + d. parse "~" ranges + e. parse unmatched token as comparator + ~. append to current and_chunk + 2. return SemSelOrChunk + + Raises TypeError, ValueError or SelParseError. + """ + if not isinstance(sel, basestring): + raise TypeError("Selector must be a string") + if not sel: + raise ValueError("String must not be empty") + + # Split selector by spaces and crawl the tokens + tokens = sel.split() + i = -1 + or_chunk = SemSelOrChunk() + and_chunk = or_chunk.new_child() + + while i + 1 < len(tokens): + i += 1 + t = tokens[i] + + # Replace x ranges with ~ selector + m = cls._xrange_regex.match(t) + m = m and m.groups('') + if m and any(not x.isdigit() for x in m[1:4]) and not m[0].startswith('>'): + # (do not match '>1.0' or '>*') + if m[4]: + raise SelParseError("XRanges do not allow pre-release or build components") + + # Only use digit parts and fail if digit found after non-digit + mm, xran = [], False + for x in m[1:4]: + if x.isdigit(): + if xran: + raise SelParseError("Invalid fuzzy range or XRange '%s'" % tokens[i]) + mm.append(x) + else: + xran = True + t = m[0] + '.'.join(mm) # x for x in m[1:4] if x.isdigit()) + # Append "~" if not already present + if not t.startswith('~'): + t = '~' + t + + # switch t: + if t == '||': + if i == 0 or tokens[i - 1] == '||' or i + 1 == len(tokens): + raise SelParseError("OR range must not be empty") + # Start a new and_chunk + and_chunk = or_chunk.new_child() + + elif t == '-': + # ' - ' range + i += 1 + invalid = False + try: + # If these result in exceptions, you know you're doing it wrong + t = tokens[i] + c = and_chunk[-1] + except: + raise SelParseError("Invalid ' - ' range position") + + # If there is an op in front of one of the bound versions + invalid = (c.op not in ('=', '~') + or cls._split_op_regex.match(t).group(1) not in (None, '=')) + if invalid: + raise SelParseError("Invalid ' - ' range '%s - %s'" + % (tokens[i - 2], tokens[i])) + + c.op = ">=" + and_chunk.add_child('<=', t) + + elif t == '': + # Multiple spaces + pass + + elif t.startswith('~'): + m = cls._fuzzy_regex.match(t) + if not m: + raise SelParseError("Invalid fuzzy range or XRange '%s'" % tokens[i]) + + mm, m = m.groups('')[1:4], m.groupdict('') # mm: major to patch + + # Minimum requirement + min_ver = ('.'.join(x or '0' for x in mm) + '-' + if not m['other'] + else cls._split_op_regex(t[1:]).group('ver')) + and_chunk.add_child('>=', min_ver) + + if m['major']: + # Increase version before none (or second to last if '~1.2.3') + e = [0, 0, 0] + for j, d in enumerate(mm): + if not d or j == len(mm) - 1: + e[j - 1] = e[j - 1] + 1 + break + e[j] = int(d) + + and_chunk.add_child('<', '.'.join(str(x) for x in e) + '-') + + # else: just plain '~' or '*', or '~>X' which are already handled + + else: + # A normal comparator + m = cls._split_op_regex.match(t).groupdict() # this regex can't fail + and_chunk.add_child(**m) + + # Finally return the or_chunk + return or_chunk \ No newline at end of file diff --git a/sublime/Packages/Package Control/package_control/show_error.py b/sublime/Packages/Package Control/package_control/show_error.py new file mode 100644 index 0000000..b8169c9 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/show_error.py @@ -0,0 +1,12 @@ +import sublime + + +def show_error(string): + """ + Displays an error message with a standard "Package Control" header + + :param string: + The error to display + """ + + sublime.error_message(u'Package Control\n\n%s' % string) diff --git a/sublime/Packages/Package Control/package_control/sys_path.py b/sublime/Packages/Package Control/package_control/sys_path.py new file mode 100644 index 0000000..10daa3d --- /dev/null +++ b/sublime/Packages/Package Control/package_control/sys_path.py @@ -0,0 +1,27 @@ +import sys +import os + +if os.name == 'nt': + from ctypes import windll, create_unicode_buffer + +import sublime + + +def add_to_path(path): + # Python 2.x on Windows can't properly import from non-ASCII paths, so + # this code added the DOC 8.3 version of the lib folder to the path in + # case the user's username includes non-ASCII characters + if os.name == 'nt': + buf = create_unicode_buffer(512) + if windll.kernel32.GetShortPathNameW(path, buf, len(buf)): + path = buf.value + + if path not in sys.path: + sys.path.append(path) + + +lib_folder = os.path.join(sublime.packages_path(), 'Package Control', 'lib') +add_to_path(os.path.join(lib_folder, 'all')) + +if os.name == 'nt': + add_to_path(os.path.join(lib_folder, 'windows')) diff --git a/sublime/Packages/Package Control/package_control/thread_progress.py b/sublime/Packages/Package Control/package_control/thread_progress.py new file mode 100644 index 0000000..b40c564 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/thread_progress.py @@ -0,0 +1,46 @@ +import sublime + + +class ThreadProgress(): + """ + Animates an indicator, [= ], in the status area while a thread runs + + :param thread: + The thread to track for activity + + :param message: + The message to display next to the activity indicator + + :param success_message: + The message to display once the thread is complete + """ + + def __init__(self, thread, message, success_message): + self.thread = thread + self.message = message + self.success_message = success_message + self.addend = 1 + self.size = 8 + sublime.set_timeout(lambda: self.run(0), 100) + + def run(self, i): + if not self.thread.is_alive(): + if hasattr(self.thread, 'result') and not self.thread.result: + sublime.status_message('') + return + sublime.status_message(self.success_message) + return + + before = i % self.size + after = (self.size - 1) - before + + sublime.status_message('%s [%s=%s]' % \ + (self.message, ' ' * before, ' ' * after)) + + if not after: + self.addend = -1 + if not before: + self.addend = 1 + i += self.addend + + sublime.set_timeout(lambda: self.run(i), 100) diff --git a/sublime/Packages/Package Control/package_control/unicode.py b/sublime/Packages/Package Control/package_control/unicode.py new file mode 100644 index 0000000..f0464a2 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/unicode.py @@ -0,0 +1,49 @@ +import os +import locale +import sys + + +# Sublime Text on OS X does not seem to report the correct encoding +# so we hard-code that to UTF-8 +_encoding = 'utf-8' if sys.platform == 'darwin' else locale.getpreferredencoding() + +_fallback_encodings = ['utf-8', 'cp1252'] + + +def unicode_from_os(e): + """ + This is needed as some exceptions coming from the OS are + already encoded and so just calling unicode(e) will result + in an UnicodeDecodeError as the string isn't in ascii form. + + :param e: + The exception to get the value of + + :return: + The unicode version of the exception message + """ + + if sys.version_info >= (3,): + return str(e) + + try: + if isinstance(e, Exception): + e = e.message + + if isinstance(e, unicode): + return e + + if isinstance(e, int): + e = str(e) + + return unicode(e, _encoding) + + # If the "correct" encoding did not work, try some defaults, and then just + # obliterate characters that we can't seen to decode properly + except UnicodeDecodeError: + for encoding in _fallback_encodings: + try: + return unicode(e, encoding, errors='strict') + except: + pass + return unicode(e, errors='replace') diff --git a/sublime/Packages/Package Control/package_control/upgraders/__init__.py b/sublime/Packages/Package Control/package_control/upgraders/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sublime/Packages/Package Control/package_control/upgraders/git_upgrader.py b/sublime/Packages/Package Control/package_control/upgraders/git_upgrader.py new file mode 100644 index 0000000..878b1fd --- /dev/null +++ b/sublime/Packages/Package Control/package_control/upgraders/git_upgrader.py @@ -0,0 +1,106 @@ +import os + +from ..cache import set_cache, get_cache +from ..show_error import show_error +from .vcs_upgrader import VcsUpgrader + + +class GitUpgrader(VcsUpgrader): + """ + Allows upgrading a local git-repository-based package + """ + + cli_name = 'git' + + def retrieve_binary(self): + """ + Returns the path to the git executable + + :return: The string path to the executable or False on error + """ + + name = 'git' + if os.name == 'nt': + name += '.exe' + binary = self.find_binary(name) + if binary and os.path.isdir(binary): + full_path = os.path.join(binary, name) + if os.path.exists(full_path): + binary = full_path + if not binary: + show_error((u'Unable to find %s. Please set the git_binary setting by accessing the ' + + u'Preferences > Package Settings > Package Control > Settings \u2013 User menu entry. ' + + u'The Settings \u2013 Default entry can be used for reference, but changes to that will be ' + + u'overwritten upon next upgrade.') % name) + return False + + if os.name == 'nt': + tortoise_plink = self.find_binary('TortoisePlink.exe') + if tortoise_plink: + os.environ.setdefault('GIT_SSH', tortoise_plink) + return binary + + def get_working_copy_info(self): + binary = self.retrieve_binary() + if not binary: + return False + + # Get the current branch name + res = self.execute([binary, 'symbolic-ref', '-q', 'HEAD'], self.working_copy) + branch = res.replace('refs/heads/', '') + + # Figure out the remote and the branch name on the remote + remote = self.execute([binary, 'config', '--get', 'branch.%s.remote' % branch], self.working_copy) + res = self.execute([binary, 'config', '--get', 'branch.%s.merge' % branch], self.working_copy) + remote_branch = res.replace('refs/heads/', '') + + return { + 'branch': branch, + 'remote': remote, + 'remote_branch': remote_branch + } + + def run(self): + """ + Updates the repository with remote changes + + :return: False or error, or True on success + """ + + binary = self.retrieve_binary() + if not binary: + return False + + info = self.get_working_copy_info() + + args = [binary] + args.extend(self.update_command) + args.extend([info['remote'], info['remote_branch']]) + self.execute(args, self.working_copy) + return True + + def incoming(self): + """:return: bool if remote revisions are available""" + + cache_key = self.working_copy + '.incoming' + incoming = get_cache(cache_key) + if incoming != None: + return incoming + + binary = self.retrieve_binary() + if not binary: + return False + + info = self.get_working_copy_info() + + res = self.execute([binary, 'fetch', info['remote']], self.working_copy) + if res == False: + return False + + args = [binary, 'log'] + args.append('..%s/%s' % (info['remote'], info['remote_branch'])) + output = self.execute(args, self.working_copy) + incoming = len(output) > 0 + + set_cache(cache_key, incoming, self.cache_length) + return incoming diff --git a/sublime/Packages/Package Control/package_control/upgraders/hg_upgrader.py b/sublime/Packages/Package Control/package_control/upgraders/hg_upgrader.py new file mode 100644 index 0000000..36dfb48 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/upgraders/hg_upgrader.py @@ -0,0 +1,74 @@ +import os + +from ..cache import set_cache, get_cache +from ..show_error import show_error +from .vcs_upgrader import VcsUpgrader + + +class HgUpgrader(VcsUpgrader): + """ + Allows upgrading a local mercurial-repository-based package + """ + + cli_name = 'hg' + + def retrieve_binary(self): + """ + Returns the path to the hg executable + + :return: The string path to the executable or False on error + """ + + name = 'hg' + if os.name == 'nt': + name += '.exe' + binary = self.find_binary(name) + if binary and os.path.isdir(binary): + full_path = os.path.join(binary, name) + if os.path.exists(full_path): + binary = full_path + if not binary: + show_error((u'Unable to find %s. Please set the hg_binary setting by accessing the ' + + u'Preferences > Package Settings > Package Control > Settings \u2013 User menu entry. ' + + u'The Settings \u2013 Default entry can be used for reference, but changes to that will be ' + + u'overwritten upon next upgrade.') % name) + return False + return binary + + def run(self): + """ + Updates the repository with remote changes + + :return: False or error, or True on success + """ + + binary = self.retrieve_binary() + if not binary: + return False + args = [binary] + args.extend(self.update_command) + args.append('default') + self.execute(args, self.working_copy) + return True + + def incoming(self): + """:return: bool if remote revisions are available""" + + cache_key = self.working_copy + '.incoming' + incoming = get_cache(cache_key) + if incoming != None: + return incoming + + binary = self.retrieve_binary() + if not binary: + return False + + args = [binary, 'in', '-q', 'default'] + output = self.execute(args, self.working_copy) + if output == False: + return False + + incoming = len(output) > 0 + + set_cache(cache_key, incoming, self.cache_length) + return incoming diff --git a/sublime/Packages/Package Control/package_control/upgraders/vcs_upgrader.py b/sublime/Packages/Package Control/package_control/upgraders/vcs_upgrader.py new file mode 100644 index 0000000..d82abe7 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/upgraders/vcs_upgrader.py @@ -0,0 +1,27 @@ +from ..cmd import create_cmd, Cli + + +class VcsUpgrader(Cli): + """ + Base class for updating packages that are a version control repository on local disk + + :param vcs_binary: + The full filesystem path to the executable for the version control + system. May be set to None to allow the code to try and find it. + + :param update_command: + The command to pass to the version control executable to update the + repository. + + :param working_copy: + The local path to the working copy/package directory + + :param cache_length: + The lenth of time to cache if incoming changesets are available + """ + + def __init__(self, vcs_binary, update_command, working_copy, cache_length, debug): + self.update_command = update_command + self.working_copy = working_copy + self.cache_length = cache_length + super(VcsUpgrader, self).__init__(vcs_binary, debug) diff --git a/sublime/Packages/Package Control/package_control/versions.py b/sublime/Packages/Package Control/package_control/versions.py new file mode 100644 index 0000000..90a5ef6 --- /dev/null +++ b/sublime/Packages/Package Control/package_control/versions.py @@ -0,0 +1,81 @@ +import re + +from .semver import SemVer +from .console_write import console_write + + +def semver_compat(v): + if isinstance(v, SemVer): + return str(v) + + # Allowing passing in a dict containing info about a package + if isinstance(v, dict): + if 'version' not in v: + return '0' + v = v['version'] + + # Trim v off of the front + v = re.sub('^v', '', v) + + # We prepend 0 to all date-based version numbers so that developers + # may switch to explicit versioning from GitHub/BitBucket + # versioning based on commit dates. + # + # When translating dates into semver, the way to get each date + # segment into the version is to treat the year and month as + # minor and patch, and then the rest as a numeric build version + # with four different parts. The result looks like: + # 0.2012.11+10.31.23.59 + date_match = re.match('(\d{4})\.(\d{2})\.(\d{2})\.(\d{2})\.(\d{2})\.(\d{2})$', v) + if date_match: + v = '0.%s.%s+%s.%s.%s.%s' % date_match.groups() + + # This handles version that were valid pre-semver with 4+ dotted + # groups, such as 1.6.9.0 + four_plus_match = re.match('(\d+\.\d+\.\d+)[T\.](\d+(\.\d+)*)$', v) + if four_plus_match: + v = '%s+%s' % (four_plus_match.group(1), four_plus_match.group(2)) + + # Semver must have major, minor, patch + elif re.match('^\d+$', v): + v += '.0.0' + elif re.match('^\d+\.\d+$', v): + v += '.0' + return v + + +def version_comparable(string): + return SemVer(semver_compat(string)) + + +def version_exclude_prerelease(versions): + output = [] + for version in versions: + if SemVer(semver_compat(version)).prerelease != None: + continue + output.append(version) + return output + + +def version_filter(versions, allow_prerelease=False): + output = [] + for version in versions: + no_v_version = re.sub('^v', '', version) + if not SemVer.valid(no_v_version): + continue + if not allow_prerelease and SemVer(no_v_version).prerelease != None: + continue + output.append(version) + return output + + +def _version_sort_key(item): + return SemVer(semver_compat(item)) + + +def version_sort(sortable, **kwargs): + try: + return sorted(sortable, key=_version_sort_key, **kwargs) + except (ValueError) as e: + console_write(u"Error sorting versions - %s" % e, True) + return [] diff --git a/sublime/Packages/Package Control/readme.creole b/sublime/Packages/Package Control/readme.creole index eb37565..50b3d69 100644 --- a/sublime/Packages/Package Control/readme.creole +++ b/sublime/Packages/Package Control/readme.creole @@ -1,7 +1,7 @@ = Sublime Package Control -A Sublime Text 2 (http://www.sublimetext.com/2) package manager for easily -discovering, install, upgrading and removing packages. Also includes an +A Sublime Text 2/3 (http://www.sublimetext.com) package manager for easily +discovering, installing, upgrading and removing packages. Also includes an automatic updater and package creation tool. Packages can be installed from GitHub, BitBucket or custom package repositories. @@ -14,10 +14,12 @@ instructions, screenshots and documentation. == License -Sublime Package Control (except for the ntlm library) is licensed under the MIT -license. +Sublime Package Control is licensed under the MIT license. - Copyright (c) 2011-2012 Will Bond +All of the source code (except for package_control/semver.py), is under the +license: + + Copyright (c) 2011-2013 Will Bond Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -37,7 +39,24 @@ license. OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -The ntlm library (for Windows) is from the python-ntlm project -(http://code.google.com/p/python-ntlm/) and is licensed under the GNU Lesser -General Public License (LGPL). Details can be found in the source files -located in lib/windows/ntlm/. \ No newline at end of file +package_control/semver.py is under the license: + + Copyright (c) 2013 Zachary King, FichteFoll + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. diff --git a/sublime/Packages/Theme - Default/arrow_right@2x.png b/sublime/Packages/Theme - Default/arrow_right@2x.png index 9f40cd3..b1d1a11 100644 Binary files a/sublime/Packages/Theme - Default/arrow_right@2x.png and b/sublime/Packages/Theme - Default/arrow_right@2x.png differ diff --git a/sublime/Packages/User/Package Control.last-run b/sublime/Packages/User/Package Control.last-run index 9d0c6f6..1759b27 100644 --- a/sublime/Packages/User/Package Control.last-run +++ b/sublime/Packages/User/Package Control.last-run @@ -1 +1 @@ -1370390066 \ No newline at end of file +1392154952 \ No newline at end of file diff --git a/sublime/Packages/User/Preferences.sublime-settings b/sublime/Packages/User/Preferences.sublime-settings index 6da9b68..649e147 100644 --- a/sublime/Packages/User/Preferences.sublime-settings +++ b/sublime/Packages/User/Preferences.sublime-settings @@ -43,7 +43,7 @@ [ "no_italic" ], - "font_size": 10.0, + "font_size": 11.0, "highlight_line": true, "highlight_modified_tabs": true, "ignored_packages": diff --git a/sublime/Packages/Vintage/vintage.py b/sublime/Packages/Vintage/vintage.py index 5c03361..e5413db 100644 --- a/sublime/Packages/Vintage/vintage.py +++ b/sublime/Packages/Vintage/vintage.py @@ -8,6 +8,7 @@ MOTION_MODE_LINE = 2 # Registers are used for clipboards and macro storage g_registers = {} +REGISTER_NULL = '_' # Represents the current input state. The primary commands that interact with # this are: @@ -831,6 +832,11 @@ class ViPasteLeft(ViPrefixableCommand): 'register': register}) def set_register(view, register, forward): + if register == REGISTER_NULL: + # This is the null register; do nothing. + # More info in Vim: :help "_ + return + delta = 1 if not forward: delta = -1 @@ -865,6 +871,11 @@ def set_register(view, register, forward): g_registers[reg] = text def get_register(view, register): + if register == REGISTER_NULL: + # This is the null register; do nothing. + # More info in Vim: :help "_ + return + use_sys_clipboard = view.settings().get('vintage_use_clipboard', False) == True register = register.lower() if register == '%': @@ -1025,11 +1036,7 @@ class ViSelectBookmark(sublime_plugin.TextCommand): def run(self, edit, character, select_bol=False): self.view.run_command('select_all_bookmarks', {'name': "bookmark_" + character}) if select_bol: - sels = list(self.view.sel()) - self.view.sel().clear() - for r in sels: - start = self.view.line(r.a).begin() - self.view.sel().add(sublime.Region(start, start)) + self.view.run_command('vi_move_to_first_non_white_space_character') g_macro_target = None diff --git a/sublime/Pristine Packages/Default.sublime-package b/sublime/Pristine Packages/Default.sublime-package index 6a877a5..bc7e96e 100644 Binary files a/sublime/Pristine Packages/Default.sublime-package and b/sublime/Pristine Packages/Default.sublime-package differ diff --git a/sublime/Pristine Packages/Package Control.sublime-package b/sublime/Pristine Packages/Package Control.sublime-package new file mode 100644 index 0000000..cc9aa19 Binary files /dev/null and b/sublime/Pristine Packages/Package Control.sublime-package differ diff --git a/sublime/Pristine Packages/Theme - Default.sublime-package b/sublime/Pristine Packages/Theme - Default.sublime-package index 84f197e..6a905dc 100644 Binary files a/sublime/Pristine Packages/Theme - Default.sublime-package and b/sublime/Pristine Packages/Theme - Default.sublime-package differ diff --git a/sublime/Pristine Packages/Vintage.sublime-package b/sublime/Pristine Packages/Vintage.sublime-package index bb82416..a1c9843 100644 Binary files a/sublime/Pristine Packages/Vintage.sublime-package and b/sublime/Pristine Packages/Vintage.sublime-package differ