Copy SSL certificate verification from pip.

Author: trishankkarthik

setup.py

@@ -2,22 +2,28 @@
 
 from distutils.core import setup
 
-setup(name='tuf',
-  version='0.0.0',
+setup(
+  name='tuf',
+  version='0.1',
   description='A secure updater framework for Python',
-  author='numerous',
+  author='https://www.updateframework.com',
   author_email='[email protected]',
   url='https://www.updateframework.com',
-  packages=['tuf',
+  packages=[
+    'evpy',
+    'simplejson',
+    'tuf',
     'tuf.client',
+    'tuf.compatibility',
+    'tuf.interposition',
     'tuf.pushtools',
     'tuf.pushtools.transfer',
-    'tuf.repo',
-    'tuf.interposition',
-    'evpy',
-    'simplejson'],
-  scripts=['quickstart.py',
-    'basic_client.py',
+    'tuf.repo'
+  ],
+  scripts=[
+    'quickstart.py',
     'tuf/pushtools/push.py',
     'tuf/pushtools/receivetools/receive.py',
-    'tuf/repo/signercli.py'])
+    'tuf/repo/signercli.py'
+  ]
+)

tuf/compatibility/__init__.py

@@ -0,0 +1,39 @@
+"""
+We copy some backwards compatibility from pip.
+
+https://github.com/pypa/pip/tree/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/backwardcompat
+"""
+
+
+import sys
+
+
+if sys.version_info >= (3,):
+    import http.client as httplib
+    import urllib.parse as urlparse
+    import urllib.request as urllib2
+else:
+    import httplib
+    import urllib2
+    import urlparse
+
+
+## py25 has no builtin ssl module
+## only >=py32 has ssl.match_hostname and ssl.CertificateError
+try:
+    import ssl
+    try:
+        from ssl import match_hostname, CertificateError
+    except ImportError:
+        from tuf.compatibility.ssl_match_hostname import match_hostname, CertificateError
+except ImportError:
+    ssl = None
+
+
+# patch for py25 socket to work with http://pypi.python.org/pypi/ssl/
+import socket
+if not hasattr(socket, 'create_connection'): # for Python 2.5
+    # monkey-patch socket module
+    from tuf.compatibility.socket_create_connection import create_connection
+    socket.create_connection = create_connection
+

tuf/compatibility/socket_create_connection.py

@@ -0,0 +1,48 @@
+"""
+We copy some functions from the Python 2.7.3 socket module.
+
+http://hg.python.org/releasing/2.7.3/file/7bb96963d067/Lib/socket.py
+"""
+
+
+_GLOBAL_DEFAULT_TIMEOUT = object()
+
+
+def create_connection(address, timeout=_GLOBAL_DEFAULT_TIMEOUT,
+                      source_address=None):
+    """Connect to *address* and return the socket object.
+
+    Convenience function.  Connect to *address* (a 2-tuple ``(host,
+    port)``) and return the socket object.  Passing the optional
+    *timeout* parameter will set the timeout on the socket instance
+    before attempting to connect.  If no *timeout* is supplied, the
+    global default timeout setting returned by :func:`getdefaulttimeout`
+    is used.  If *source_address* is set it must be a tuple of (host, port)
+    for the socket to bind as a source address before making the connection.
+    An host of '' or port 0 tells the OS to use the default.
+    """
+
+    host, port = address
+    err = None
+    for res in getaddrinfo(host, port, 0, SOCK_STREAM):
+        af, socktype, proto, canonname, sa = res
+        sock = None
+        try:
+            sock = socket(af, socktype, proto)
+            if timeout is not _GLOBAL_DEFAULT_TIMEOUT:
+                sock.settimeout(timeout)
+            if source_address:
+                sock.bind(source_address)
+            sock.connect(sa)
+            return sock
+
+        except error as _:
+            err = _
+            if sock is not None:
+                sock.close()
+
+    if err is not None:
+        raise err
+    else:
+        raise error("getaddrinfo returns an empty list")
+

tuf/compatibility/ssl_match_hostname.py

@@ -0,0 +1,70 @@
+"""
+We copy some functions from the Python 3.3.0 ssl module.
+
+http://hg.python.org/releasing/3.3.0/file/1465cbbc8f64/Lib/ssl.py
+"""
+
+
+import re
+
+
+class CertificateError(ValueError):
+    pass
+
+
+def _dnsname_to_pat(dn):
+    pats = []
+    for frag in dn.split(r'.'):
+        if frag == '*':
+            # When '*' is a fragment by itself, it matches a non-empty dotless
+            # fragment.
+            pats.append('[^.]+')
+        else:
+            # Otherwise, '*' matches any dotless fragment.
+            frag = re.escape(frag)
+            pats.append(frag.replace(r'\*', '[^.]*'))
+    return re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE)
+
+
+def match_hostname(cert, hostname):
+    """Verify that *cert* (in decoded format as returned by
+    SSLSocket.getpeercert()) matches the *hostname*.  RFC 2818 rules
+    are mostly followed, but IP addresses are not accepted for *hostname*.
+
+    CertificateError is raised on failure. On success, the function
+    returns nothing.
+    """
+    if not cert:
+        raise ValueError("empty or no certificate")
+    dnsnames = []
+    san = cert.get('subjectAltName', ())
+    for key, value in san:
+        if key == 'DNS':
+            if _dnsname_to_pat(value).match(hostname):
+                return
+            dnsnames.append(value)
+    if not dnsnames:
+        # The subject is only checked when there is no dNSName entry
+        # in subjectAltName
+        for sub in cert.get('subject', ()):
+            for key, value in sub:
+                # XXX according to RFC 2818, the most specific Common Name
+                # must be used.
+                if key == 'commonName':
+                    if _dnsname_to_pat(value).match(hostname):
+                        return
+                    dnsnames.append(value)
+    if len(dnsnames) > 1:
+        raise CertificateError("hostname %r "
+            "doesn't match either of %s"
+            % (hostname, ', '.join(map(repr, dnsnames))))
+    elif len(dnsnames) == 1:
+        raise CertificateError("hostname %r "
+            "doesn't match %r"
+            % (hostname, dnsnames[0]))
+    else:
+        raise CertificateError("no appropriate commonName or "
+            "subjectAltName fields were found")
+
+
+

tuf/conf.py

@@ -31,3 +31,8 @@
 # which already exists and within that directory should have the file
 # 'metadata/current/root.txt'.  This must be set!
 repository_directory = None
+
+# A directory where you may find certificate authorities
+# https://en.wikipedia.org/wiki/Certificate_authority
+# http://docs.python.org/2/library/ssl.html#certificates
+ca_certs = None

tuf/download.py

@@ -22,25 +22,123 @@
   
 """
 
-import urllib2
 import logging
+import os.path
+import socket
 
+import tuf
 import tuf.hash
 import tuf.util
 import tuf.formats
 
+from tuf.compatibility import httplib, ssl, urllib2, urlparse
+if ssl:
+    from tuf.compatibility import match_hostname
+else:
+    raise tuf.Error( "No SSL support!" )    # TODO: degrade gracefully
+
+
 # See 'log.py' to learn how logging is handled in TUF.
 logger = logging.getLogger('tuf.download')
 
 
+class VerifiedHTTPSConnection( httplib.HTTPSConnection ):
+    """
+    A connection that wraps connections with ssl certificate verification.
+
+    https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L72
+    """
+    def connect(self):
+
+        self.connection_kwargs = {}
+
+        #TODO: refactor compatibility logic into tuf.compatibility?
+
+        # for > py2.5
+        if hasattr(self, 'timeout'):
+            self.connection_kwargs.update(timeout = self.timeout)
+
+        # for >= py2.7
+        if hasattr(self, 'source_address'):
+            self.connection_kwargs.update(source_address = self.source_address)
+
+        sock = socket.create_connection((self.host, self.port), **self.connection_kwargs)
+
+        # for >= py2.7
+        if getattr(self, '_tunnel_host', None):
+            self.sock = sock
+            self._tunnel()
+
+        # set location of certificate authorities
+        assert os.path.isfile( tuf.conf.ca_certs )
+        cert_path = tuf.conf.ca_certs
+
+        self.sock = ssl.wrap_socket(sock,
+                                self.key_file,
+                                self.cert_file,
+                                cert_reqs=ssl.CERT_REQUIRED,
+                                ca_certs=cert_path)
+
+        match_hostname(self.sock.getpeercert(), self.host)
+
+
+class VerifiedHTTPSHandler( urllib2.HTTPSHandler ):
+    """
+    A HTTPSHandler that uses our own VerifiedHTTPSConnection.
+
+    https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L109
+    """
+    def __init__(self, connection_class = VerifiedHTTPSConnection):
+        self.specialized_conn_class = connection_class
+        urllib2.HTTPSHandler.__init__(self)
+    def https_open(self, req):
+        return self.do_open(self.specialized_conn_class, req)
+
+
+def _get_request(url):
+    """
+    Wraps the URL to retrieve to protects against "creative"
+    interpretation of the RFC: http://bugs.python.org/issue8732
+
+    https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L147
+    """
+
+    return urllib2.Request(url, headers={'Accept-encoding': 'identity'})
+
+
+def _get_opener( scheme = None ):
+    """
+    Build a urllib2 opener based on whether the user now wants SSL.
+
+    https://github.com/pypa/pip/blob/d0fa66ecc03ab20b7411b35f7c7b423f31f77761/pip/download.py#L178
+    """
+
+    if scheme == "https":
+        assert os.path.isfile( tuf.conf.ca_certs )
+
+        # If we are going over https, use an opener which will provide SSL
+        # certificate verification.
+        https_handler = VerifiedHTTPSHandler()
+        opener = urllib2.build_opener( https_handler )
+
+        # strip out HTTPHandler to prevent MITM spoof
+        for handler in opener.handlers:
+            if isinstance( handler, urllib2.HTTPHandler ):
+                opener.handlers.remove( handler )
+    else:
+        # Otherwise, use the default opener.
+        opener = urllib2.build_opener()
+
+    return opener
+
+
 def _open_connection(url):
   """
   <Purpose>
     Helper function that opens a connection to the url. urllib2 supports http, 
     ftp, and file. In python (2.6+) where the ssl module is available, urllib2 
     also supports https.
-    
-    TODO: Do proper ssl cert/name checking. 
+
     TODO: Disallow SSLv2. 	
     TODO: Support ssl with MCrypto.
     TODO: Determine whether this follows http redirects and decide if we like
@@ -71,11 +169,12 @@ def _open_connection(url):
     # servers do not recognize connections that originates from 
     # Python-urllib/x.y.
 
-    request = urllib2.Request(url)
-    connection = urllib2.urlopen(request)
-    # urllib2.urlopen returns a file-like object: a handle to the remote data.
-    return connection
+    parsed_url = urlparse.urlparse( url )
+    opener = _get_opener( scheme = parsed_url.scheme )
+    request = _get_request( url )
+    return opener.open( request )
   except Exception, e:
+    raise
     raise tuf.DownloadError(e)
 
 

tuf/interposition/README.md

@@ -1,4 +1,4 @@
-## Methods
+## Examples
 
 ```python
 import tuf.interposition
@@ -65,3 +65,5 @@ unspecified path for the given network location.
 
 - The entire `urllib` or `urllib2` contract is not honoured.
 - Downloads are not thread safe.
+- Uses some Python features (e.g. string formatting) not available in earlier versions (e.g. < 2.6).
+- Uses some Python features (e.g. `urllib, urllib2, urlparse`) not available in later versions (e.g. >= 3).