From 267129776d00b76588fa02a732c10d4e80495f5d Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Jun 2017 17:00:05 +0200 Subject: [PATCH 1/7] bpo-22559: Backport MemoryBIO Implementation of the PEP 546: backport ssl.MemoryBIO and ssl.SSLObject from master to the 2.7 branch. Co-Authored-By: Alex Gaynor --- Doc/library/ssl.rst | 271 ++++++++++++++++++++-- Lib/ssl.py | 163 +++++++++++-- Lib/test/test_ssl.py | 163 ++++++++++++- Modules/_ssl.c | 540 +++++++++++++++++++++++++++++++++++++------ 4 files changed, 1010 insertions(+), 127 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 8b41317eb206a9..c85fb791579cf4 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -837,32 +837,73 @@ Constants SSL Sockets ----------- -SSL sockets provide the following methods of :ref:`socket-objects`: - -- :meth:`~socket.socket.accept()` -- :meth:`~socket.socket.bind()` -- :meth:`~socket.socket.close()` -- :meth:`~socket.socket.connect()` -- :meth:`~socket.socket.fileno()` -- :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` -- :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` -- :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, - :meth:`~socket.socket.setblocking()` -- :meth:`~socket.socket.listen()` -- :meth:`~socket.socket.makefile()` -- :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` - (but passing a non-zero ``flags`` argument is not allowed) -- :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with - the same limitation) -- :meth:`~socket.socket.shutdown()` - -However, since the SSL (and TLS) protocol has its own framing atop -of TCP, the SSL sockets abstraction can, in certain respects, diverge from -the specification of normal, OS-level sockets. See especially the -:ref:`notes on non-blocking sockets `. +.. class:: SSLSocket(socket.socket) + + SSL sockets provide the following methods of :ref:`socket-objects`: + + - :meth:`~socket.socket.accept()` + - :meth:`~socket.socket.bind()` + - :meth:`~socket.socket.close()` + - :meth:`~socket.socket.connect()` + - :meth:`~socket.socket.detach()` + - :meth:`~socket.socket.fileno()` + - :meth:`~socket.socket.getpeername()`, :meth:`~socket.socket.getsockname()` + - :meth:`~socket.socket.getsockopt()`, :meth:`~socket.socket.setsockopt()` + - :meth:`~socket.socket.gettimeout()`, :meth:`~socket.socket.settimeout()`, + :meth:`~socket.socket.setblocking()` + - :meth:`~socket.socket.listen()` + - :meth:`~socket.socket.makefile()` + - :meth:`~socket.socket.recv()`, :meth:`~socket.socket.recv_into()` + (but passing a non-zero ``flags`` argument is not allowed) + - :meth:`~socket.socket.send()`, :meth:`~socket.socket.sendall()` (with + the same limitation) + - :meth:`~socket.socket.shutdown()` + + However, since the SSL (and TLS) protocol has its own framing atop + of TCP, the SSL sockets abstraction can, in certain respects, diverge from + the specification of normal, OS-level sockets. See especially the + :ref:`notes on non-blocking sockets `. + + Usually, :class:`SSLSocket` are not created directly, but using the + :func:`wrap_socket` function or the :meth:`SSLContext.wrap_socket` method. SSL sockets also have the following additional methods and attributes: +.. method:: SSLSocket.read(len=0, buffer=None) + + Read up to *len* bytes of data from the SSL socket and return the result as + a ``bytes`` instance. If *buffer* is specified, then read into the buffer + instead, and return the number of bytes read. + + Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is + :ref:`non-blocking ` and the read would block. + + As at any time a re-negotiation is possible, a call to :meth:`read` can also + cause write operations. + +.. method:: SSLSocket.write(buf) + + Write *buf* to the SSL socket and return the number of bytes written. The + *buf* argument must be an object supporting the buffer interface. + + Raise :exc:`SSLWantReadError` or :exc:`SSLWantWriteError` if the socket is + :ref:`non-blocking ` and the write would block. + + As at any time a re-negotiation is possible, a call to :meth:`write` can + also cause read operations. + +.. note:: + + The :meth:`~SSLSocket.read` and :meth:`~SSLSocket.write` methods are the + low-level methods that read and write unencrypted, application-level data + and and decrypt/encrypt it to encrypted, wire-level data. These methods + require an active SSL connection, i.e. the handshake was completed and + :meth:`SSLSocket.unwrap` was not called. + + Normally you should use the socket API methods like + :meth:`~socket.socket.recv` and :meth:`~socket.socket.send` instead of these + methods. + .. method:: SSLSocket.do_handshake() Perform the SSL setup handshake. @@ -1003,6 +1044,11 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 2.7.9 +.. method:: SSLSocket.pending() + + Returns the number of already decrypted bytes available for read, pending on + the connection. + .. attribute:: SSLSocket.context The :class:`SSLContext` object this SSL socket is tied to. If the SSL @@ -1012,6 +1058,36 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 2.7.9 +.. attribute:: SSLSocket.server_side + + A boolean which is ``True`` for server-side sockets and ``False`` for + client-side sockets. + + .. versionadded:: 2.7.9 + +.. attribute:: SSLSocket.server_hostname + + A ``bytes`` instance containing the ``'idna'`` encoded version of the + hostname specified in the *server_hostname* argument in + :meth:`SSLContext.wrap_socket`. If no *server_hostname* was specified, this + attribute will be ``None``. + + .. versionadded:: 2.7.9 + +.. attribute:: SSLSocket.server_side + + A boolean which is ``True`` for server-side sockets and ``False`` for + client-side sockets. + + .. versionadded:: 2.7.9 + +.. attribute:: SSLSocket.server_hostname + + Hostname of the server: :class:`str` type, or ``None`` for server-side + socket or if the hostname was not specified in the constructor. + + .. versionadded:: 2.7.9 + SSL Contexts ------------ @@ -1279,6 +1355,16 @@ to speed up repeated connections from the same clients. Always allow a server_hostname to be passed, even if OpenSSL does not have SNI. +.. method:: SSLContext.wrap_bio(incoming, outgoing, server_side=False, \ + server_hostname=None) + + Create a new :class:`SSLObject` instance by wrapping the BIO objects + *incoming* and *outgoing*. The SSL routines will read input data from the + incoming BIO and write data to the outgoing BIO. + + The *server_side* and *server_hostname* parameters have the same meaning as + in :meth:`SSLContext.wrap_socket`. + .. method:: SSLContext.session_stats() Get statistics about the SSL sessions created or managed by this context. @@ -1647,7 +1733,7 @@ are finished with the client (or the client is finished with you):: And go back to listening for new client connections (of course, a real server would probably handle each client connection in a separate thread, or put -the sockets in non-blocking mode and use an event loop). +the sockets in :ref:`non-blocking mode ` and use an event loop). .. _ssl-nonblocking: @@ -1688,6 +1774,143 @@ to be aware of: except ssl.SSLWantWriteError: select.select([], [sock], []) +.. seealso:: + + The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets + ` and provides a + higher level API. It polls for events using the :mod:`selectors` module and + handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and + :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously + as well. + + +Memory BIO Support +------------------ + +.. versionadded:: 2.7.9 + +Ever since the SSL module was introduced in Python 2.6, the :class:`SSLSocket` +class has provided two related but distinct areas of functionality: + +- SSL protocol handling +- Network IO + +The network IO API is identical to that provided by :class:`socket.socket`, +from which :class:`SSLSocket` also inherits. This allows an SSL socket to be +used as a drop-in replacement for a regular socket, making it very easy to add +SSL support to an existing application. + +Combining SSL protocol handling and network IO usually works well, but there +are some cases where it doesn't. An example is async IO frameworks that want to +use a different IO multiplexing model than the "select/poll on a file +descriptor" (readiness based) model that is assumed by :class:`socket.socket` +and by the internal OpenSSL socket IO routines. This is mostly relevant for +platforms like Windows where this model is not efficient. For this purpose, a +reduced scope variant of :class:`SSLSocket` called :class:`SSLObject` is +provided. + +.. class:: SSLObject + + A reduced-scope variant of :class:`SSLSocket` representing an SSL protocol + instance that does not contain any network IO methods. This class is + typically used by framework authors that want to implement asynchronous IO + for SSL through memory buffers. + + This class implements an interface on top of a low-level SSL object as + implemented by OpenSSL. This object captures the state of an SSL connection + but does not provide any network IO itself. IO needs to be performed through + separate "BIO" objects which are OpenSSL's IO abstraction layer. + + An :class:`SSLObject` instance can be created using the + :meth:`~SSLContext.wrap_bio` method. This method will create the + :class:`SSLObject` instance and bind it to a pair of BIOs. The *incoming* + BIO is used to pass data from Python to the SSL protocol instance, while the + *outgoing* BIO is used to pass data the other way around. + + The following methods are available: + + - :attr:`~SSLSocket.context` + - :attr:`~SSLSocket.server_side` + - :attr:`~SSLSocket.server_hostname` + - :meth:`~SSLSocket.read` + - :meth:`~SSLSocket.write` + - :meth:`~SSLSocket.getpeercert` + - :meth:`~SSLSocket.selected_npn_protocol` + - :meth:`~SSLSocket.cipher` + - :meth:`~SSLSocket.compression` + - :meth:`~SSLSocket.pending` + - :meth:`~SSLSocket.do_handshake` + - :meth:`~SSLSocket.unwrap` + - :meth:`~SSLSocket.get_channel_binding` + + When compared to :class:`SSLSocket`, this object lacks the following + features: + + - Any form of network IO incluging methods such as ``recv()`` and + ``send()``. + + - There is no *do_handshake_on_connect* machinery. You must always manually + call :meth:`~SSLSocket.do_handshake` to start the handshake. + + - There is no handling of *suppress_ragged_eofs*. All end-of-file conditions + that are in violation of the protocol are reported via the + :exc:`SSLEOFError` exception. + + - The method :meth:`~SSLSocket.unwrap` call does not return anything, + unlike for an SSL socket where it returns the underlying socket. + + - The *server_name_callback* callback passed to + :meth:`SSLContext.set_servername_callback` will get an :class:`SSLObject` + instance instead of a :class:`SSLSocket` instance as its first parameter. + + Some notes related to the use of :class:`SSLObject`: + + - All IO on an :class:`SSLObject` is :ref:`non-blocking `. + This means that for example :meth:`~SSLSocket.read` will raise an + :exc:`SSLWantReadError` if it needs more data than the incoming BIO has + available. + + - There is no module-level ``wrap_bio()`` call like there is for + :meth:`~SSLContext.wrap_socket`. An :class:`SSLObject` is always created + via an :class:`SSLContext`. + +An SSLObject communicates with the outside world using memory buffers. The +class :class:`MemoryBIO` provides a memory buffer that can be used for this +purpose. It wraps an OpenSSL memory BIO (Basic IO) object: + +.. class:: MemoryBIO + + A memory buffer that can be used to pass data between Python and an SSL + protocol instance. + + .. attribute:: MemoryBIO.pending + + Return the number of bytes currently in the memory buffer. + + .. attribute:: MemoryBIO.eof + + A boolean indicating whether the memory BIO is current at the end-of-file + position. + + .. method:: MemoryBIO.read(n=-1) + + Read up to *n* bytes from the memory buffer. If *n* is not specified or + negative, all bytes are returned. + + .. method:: MemoryBIO.write(buf) + + Write the bytes from *buf* to the memory BIO. The *buf* argument must be an + object supporting the buffer protocol. + + The return value is the number of bytes written, which is always equal to + the length of *buf*. + + .. method:: MemoryBIO.write_eof() + + Write an EOF marker to the memory BIO. After this method has been called, it + is illegal to call :meth:`~MemoryBIO.write`. The attribute :attr:`eof` will + become true after all data currently in the buffer has been read. + .. _ssl-security: diff --git a/Lib/ssl.py b/Lib/ssl.py index f28c863811fd98..9898e07310cb6c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -98,7 +98,7 @@ import _ssl # if we can't import it, let the error propagate from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION -from _ssl import _SSLContext +from _ssl import _SSLContext, MemoryBIO from _ssl import ( SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, SSLSyscallError, SSLEOFError, @@ -362,6 +362,12 @@ def wrap_socket(self, sock, server_side=False, server_hostname=server_hostname, _context=self) + def wrap_bio(self, incoming, outgoing, server_side=False, + server_hostname=None): + sslobj = self._wrap_bio(incoming, outgoing, server_side=server_side, + server_hostname=server_hostname) + return SSLObject(sslobj) + def set_npn_protocols(self, npn_protocols): protos = bytearray() for protocol in npn_protocols: @@ -521,6 +527,129 @@ def _https_verify_certificates(enable=True): _create_default_https_context = _create_unverified_context + +class SSLObject(object): + """This class implements an interface on top of a low-level SSL object as + implemented by OpenSSL. This object captures the state of an SSL connection + but does not provide any network IO itself. IO needs to be performed + through separate "BIO" objects which are OpenSSL's IO abstraction layer. + + This class does not have a public constructor. Instances are returned by + ``SSLContext.wrap_bio``. This class is typically used by framework authors + that want to implement asynchronous IO for SSL through memory buffers. + + When compared to ``SSLSocket``, this object lacks the following features: + + * Any form of network IO incluging methods such as ``recv`` and ``send``. + * The ``do_handshake_on_connect`` and ``suppress_ragged_eofs`` machinery. + """ + + def __init__(self, sslobj, owner=None): + self._sslobj = sslobj + # Note: _sslobj takes a weak reference to owner + self._sslobj.owner = owner or self + + @property + def context(self): + """The SSLContext that is currently in use.""" + return self._sslobj.context + + @context.setter + def context(self, ctx): + self._sslobj.context = ctx + + @property + def server_side(self): + """Whether this is a server-side socket.""" + return self._sslobj.server_side + + @property + def server_hostname(self): + """The currently set server hostname (for SNI), or ``None`` if no + server hostame is set.""" + return self._sslobj.server_hostname + + def read(self, len=0, buffer=None): + """Read up to 'len' bytes from the SSL object and return them. + + If 'buffer' is provided, read into this buffer and return the number of + bytes read. + """ + if buffer is not None: + v = self._sslobj.read(len, buffer) + else: + v = self._sslobj.read(len or 1024) + return v + + def write(self, data): + """Write 'data' to the SSL object and return the number of bytes + written. + + The 'data' argument must support the buffer interface. + """ + return self._sslobj.write(data) + + def getpeercert(self, binary_form=False): + """Returns a formatted version of the data in the certificate provided + by the other end of the SSL channel. + + Return None if no certificate was provided, {} if a certificate was + provided, but not validated. + """ + return self._sslobj.peer_certificate(binary_form) + + def selected_npn_protocol(self): + """Return the currently selected NPN protocol as a string, or ``None`` + if a next protocol was not negotiated or if NPN is not supported by one + of the peers.""" + if _ssl.HAS_NPN: + return self._sslobj.selected_npn_protocol() + + def cipher(self): + """Return the currently selected cipher as a 3-tuple ``(name, + ssl_version, secret_bits)``.""" + return self._sslobj.cipher() + + def compression(self): + """Return the current compression algorithm in use, or ``None`` if + compression was not negotiated or not supported by one of the peers.""" + return self._sslobj.compression() + + def pending(self): + """Return the number of bytes that can be read immediately.""" + return self._sslobj.pending() + + def do_handshake(self): + """Start the SSL/TLS handshake.""" + self._sslobj.do_handshake() + if self.context.check_hostname: + if not self.server_hostname: + raise ValueError("check_hostname needs server_hostname " + "argument") + match_hostname(self.getpeercert(), self.server_hostname) + + def unwrap(self): + """Start the SSL shutdown handshake.""" + return self._sslobj.shutdown() + + def get_channel_binding(self, cb_type="tls-unique"): + """Get channel binding data for current connection. Raise ValueError + if the requested `cb_type` is not supported. Return bytes of the data + or None if the data is not available (e.g. before the handshake).""" + if cb_type not in CHANNEL_BINDING_TYPES: + raise ValueError("Unsupported channel binding type") + if cb_type != "tls-unique": + raise NotImplementedError( + "{0} channel binding type not implemented" + .format(cb_type)) + return self._sslobj.tls_unique_cb() + + def version(self): + """Return a string identifying the protocol version used by the + current SSL channel. """ + return self._sslobj.version() + + class SSLSocket(socket): """This class implements a subtype of socket.socket that wraps the underlying OS socket in an SSL context when necessary, and @@ -601,8 +730,9 @@ def __init__(self, sock=None, keyfile=None, certfile=None, if connected: # create the SSL object try: - self._sslobj = self._context._wrap_socket(self._sock, server_side, - server_hostname, ssl_sock=self) + sslobj = self._context._wrap_socket(self._sock, server_side, + server_hostname) + self._sslobj = SSLObject(sslobj, owner=self) if do_handshake_on_connect: timeout = self.gettimeout() if timeout == 0.0: @@ -647,11 +777,7 @@ def read(self, len=1024, buffer=None): if not self._sslobj: raise ValueError("Read on closed or unwrapped SSL socket.") try: - if buffer is not None: - v = self._sslobj.read(len, buffer) - else: - v = self._sslobj.read(len) - return v + return self._sslobj.read(len, buffer) except SSLError as x: if x.args[0] == SSL_ERROR_EOF and self.suppress_ragged_eofs: if buffer is not None: @@ -678,7 +804,7 @@ def getpeercert(self, binary_form=False): self._checkClosed() self._check_connected() - return self._sslobj.peer_certificate(binary_form) + return self._sslobj.getpeercert(binary_form) def selected_npn_protocol(self): self._checkClosed() @@ -820,7 +946,7 @@ def close(self): def unwrap(self): if self._sslobj: - s = self._sslobj.shutdown() + s = self._sslobj.unwrap() self._sslobj = None return s else: @@ -841,12 +967,6 @@ def do_handshake(self, block=False): finally: self.settimeout(timeout) - if self.context.check_hostname: - if not self.server_hostname: - raise ValueError("check_hostname needs server_hostname " - "argument") - match_hostname(self.getpeercert(), self.server_hostname) - def _real_connect(self, addr, connect_ex): if self.server_side: raise ValueError("can't connect in server-side mode") @@ -854,7 +974,8 @@ def _real_connect(self, addr, connect_ex): # connected at the time of the call. We connect it, then wrap it. if self._connected: raise ValueError("attempt to connect already-connected SSLSocket!") - self._sslobj = self.context._wrap_socket(self._sock, False, self.server_hostname, ssl_sock=self) + sslobj = self.context._wrap_socket(self._sock, False, self.server_hostname) + self._sslobj = SSLObject(sslobj, owner=self) try: if connect_ex: rc = socket.connect_ex(self, addr) @@ -908,15 +1029,9 @@ def get_channel_binding(self, cb_type="tls-unique"): if the requested `cb_type` is not supported. Return bytes of the data or None if the data is not available (e.g. before the handshake). """ - if cb_type not in CHANNEL_BINDING_TYPES: - raise ValueError("Unsupported channel binding type") - if cb_type != "tls-unique": - raise NotImplementedError( - "{0} channel binding type not implemented" - .format(cb_type)) if self._sslobj is None: return None - return self._sslobj.tls_unique_cb() + return self._sslobj.get_channel_binding(cb_type) def version(self): """ diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 2f98cf145508a4..4e3473466217c7 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -539,9 +539,14 @@ def test_server_side(self): def test_unknown_channel_binding(self): # should raise ValueError for unknown type s = socket.socket(socket.AF_INET) - with closing(ssl.wrap_socket(s)) as ss: + s.bind(('127.0.0.1', 0)) + s.listen(5) + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with closing(ssl.wrap_socket(c, do_handshake_on_connect=False)) as ss: with self.assertRaises(ValueError): ss.get_channel_binding("unknown-type") + s.close() @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, "'tls-unique' channel binding not available") @@ -1317,6 +1322,68 @@ def test_subclass(self): self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + class NetworkedTests(unittest.TestCase): def test_connect(self): @@ -1647,6 +1714,95 @@ def test_context_setget(self): self.assertIs(ss.context, ctx2) self.assertIs(ss._sslobj.context, ctx2) + +class NetworkedBIOTests(unittest.TestCase): + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', 10) + count = 0 + while True: + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + # Note that we get a spurious -1/SSL_ERROR_SYSCALL for + # non-blocking IO. The SSL_shutdown manpage hints at this. + # It *should* be safe to just ignore SYS_ERROR_SYSCALL because + # with a Memory BIO there's no syscalls (for IO at least). + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE, + ssl.SSL_ERROR_SYSCALL): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_handshake(self): + with support.transient_internet("svn.python.org"): + sock = socket.socket(socket.AF_INET) + sock.connect(("svn.python.org", 443)) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_REQUIRED + ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + if ssl.HAS_SNI: + ctx.check_hostname = True + sslobj = ctx.wrap_bio(incoming, outgoing, False, 'svn.python.org') + else: + ctx.check_hostname = False + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertRaises(ValueError, sslobj.getpeercert) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES: + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + sock.close() + + def test_read_write_data(self): + with support.transient_internet("svn.python.org"): + sock = socket.socket(socket.AF_INET) + sock.connect(("svn.python.org", 443)) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'GET / HTTP/1.0\r\n\r\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf[:5], b'HTTP/') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + sock.close() + + try: import threading except ImportError: @@ -2203,7 +2359,7 @@ def test_check_hostname(self): with closing(context.wrap_socket(socket.socket(), server_hostname="invalid")) as s: with self.assertRaisesRegexp(ssl.CertificateError, - "hostname 'invalid' doesn't match u?'localhost'"): + "hostname u?'invalid' doesn't match u?'localhost'"): s.connect((HOST, server.port)) # missing server_hostname arg should cause an exception, too @@ -3170,10 +3326,11 @@ def test_main(verbose=False): if not os.path.exists(filename): raise support.TestFailed("Can't read certificate file %r" % filename) - tests = [ContextTests, BasicTests, BasicSocketTests, SSLErrorTests] + tests = [ContextTests, BasicTests, BasicSocketTests, SSLErrorTests, MemoryBIOTests] if support.is_resource_enabled('network'): tests.append(NetworkedTests) + tests.append(NetworkedBIOTests) if _have_threads: thread_info = support.threading_setup() diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 45a1d0123109eb..5148e092691e4c 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -69,6 +69,7 @@ #include "openssl/ssl.h" #include "openssl/err.h" #include "openssl/rand.h" +#include "openssl/bio.h" /* SSL error object */ static PyObject *PySSLErrorObject; @@ -296,18 +297,26 @@ typedef struct { typedef struct { PyObject_HEAD - PySocketSockObject *Socket; - PyObject *ssl_sock; + PyObject *Socket; /* weakref to socket on which we're layered */ SSL *ssl; PySSLContext *ctx; /* weakref to SSL context */ X509 *peer_cert; char shutdown_seen_zero; char handshake_done; enum py_ssl_server_or_client socket_type; + PyObject *owner; /* Python level "owner" passed to servername callback */ + PyObject *server_hostname; } PySSLSocket; +typedef struct { + PyObject_HEAD + BIO *bio; + int eof_written; +} PySSLMemoryBIO; + static PyTypeObject PySSLContext_Type; static PyTypeObject PySSLSocket_Type; +static PyTypeObject PySSLMemoryBIO_Type; static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args); static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args); @@ -318,6 +327,7 @@ static PyObject *PySSL_cipher(PySSLSocket *self); #define PySSLContext_Check(v) (Py_TYPE(v) == &PySSLContext_Type) #define PySSLSocket_Check(v) (Py_TYPE(v) == &PySSLSocket_Type) +#define PySSLMemoryBIO_Check(v) (Py_TYPE(v) == &PySSLMemoryBIO_Type) typedef enum { SOCKET_IS_NONBLOCKING, @@ -334,6 +344,9 @@ typedef enum { #define ERRSTR1(x,y,z) (x ":" y ": " z) #define ERRSTR(x) ERRSTR1("_ssl.c", STRINGIFY2(__LINE__), x) +/* Get the socket from a PySSLSocket, if it has one */ +#define GET_SOCKET(obj) ((obj)->Socket ? \ + (PySocketSockObject *) PyWeakref_GetObject((obj)->Socket) : NULL) /* * SSL errors. @@ -481,12 +494,12 @@ PySSL_SetError(PySSLSocket *obj, int ret, char *filename, int lineno) case SSL_ERROR_SYSCALL: { if (e == 0) { - PySocketSockObject *s = obj->Socket; - if (ret == 0) { + PySocketSockObject *s = GET_SOCKET(obj); + if (ret == 0 || (((PyObject *)s) == Py_None)) { p = PY_SSL_ERROR_EOF; type = PySSLEOFErrorObject; errstr = "EOF occurred in violation of protocol"; - } else if (ret == -1) { + } else if (s && ret == -1) { /* underlying BIO reported an I/O error */ Py_INCREF(s); ERR_clear_error(); @@ -540,10 +553,12 @@ _setSSLError (char *errstr, int errcode, char *filename, int lineno) { static PySSLSocket * newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, enum py_ssl_server_or_client socket_type, - char *server_hostname, PyObject *ssl_sock) + char *server_hostname, + PySSLMemoryBIO *inbio, PySSLMemoryBIO *outbio) { PySSLSocket *self; SSL_CTX *ctx = sslctx->ctx; + PyObject *hostname; long mode; self = PyObject_New(PySSLSocket, &PySSLSocket_Type); @@ -553,10 +568,21 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, self->peer_cert = NULL; self->ssl = NULL; self->Socket = NULL; - self->ssl_sock = NULL; self->ctx = sslctx; self->shutdown_seen_zero = 0; self->handshake_done = 0; + self->owner = NULL; + if (server_hostname != NULL) { + hostname = PyUnicode_Decode(server_hostname, strlen(server_hostname), + "idna", "strict"); + if (hostname == NULL) { + Py_DECREF(self); + return NULL; + } + self->server_hostname = hostname; + } else + self->server_hostname = NULL; + Py_INCREF(sslctx); /* Make sure the SSL error state is initialized */ @@ -566,8 +592,17 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, PySSL_BEGIN_ALLOW_THREADS self->ssl = SSL_new(ctx); PySSL_END_ALLOW_THREADS - SSL_set_app_data(self->ssl,self); - SSL_set_fd(self->ssl, Py_SAFE_DOWNCAST(sock->sock_fd, SOCKET_T, int)); + SSL_set_app_data(self->ssl, self); + if (sock) { + SSL_set_fd(self->ssl, Py_SAFE_DOWNCAST(sock->sock_fd, SOCKET_T, int)); + } else { + /* BIOs are reference counted and SSL_set_bio borrows our reference. + * To prevent a double free in memory_bio_dealloc() we need to take an + * extra reference here. */ + CRYPTO_add(&inbio->bio->references, 1, CRYPTO_LOCK_BIO); + CRYPTO_add(&outbio->bio->references, 1, CRYPTO_LOCK_BIO); + SSL_set_bio(self->ssl, inbio->bio, outbio->bio); + } mode = SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER; #ifdef SSL_MODE_AUTO_RETRY mode |= SSL_MODE_AUTO_RETRY; @@ -582,7 +617,7 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, /* If the socket is in non-blocking mode or timeout mode, set the BIO * to non-blocking mode (blocking is the default) */ - if (sock->sock_timeout >= 0.0) { + if (sock && sock->sock_timeout >= 0.0) { BIO_set_nbio(SSL_get_rbio(self->ssl), 1); BIO_set_nbio(SSL_get_wbio(self->ssl), 1); } @@ -595,12 +630,11 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, PySSL_END_ALLOW_THREADS self->socket_type = socket_type; - self->Socket = sock; - Py_INCREF(self->Socket); - if (ssl_sock != Py_None) { - self->ssl_sock = PyWeakref_NewRef(ssl_sock, NULL); - if (self->ssl_sock == NULL) { + if (sock != NULL) { + self->Socket = PyWeakref_NewRef((PyObject *) sock, NULL); + if (self->Socket == NULL) { Py_DECREF(self); + Py_XDECREF(self->server_hostname); return NULL; } } @@ -615,14 +649,21 @@ static PyObject *PySSL_SSLdo_handshake(PySSLSocket *self) int ret; int err; int sockstate, nonblocking; - PySocketSockObject *sock = self->Socket; + PySocketSockObject *sock = GET_SOCKET(self); - Py_INCREF(sock); + if (sock) { + if (((PyObject*)sock) == Py_None) { + _setSSLError("Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); - /* just in case the blocking state of the socket has been changed */ - nonblocking = (sock->sock_timeout >= 0.0); - BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); - BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + /* just in case the blocking state of the socket has been changed */ + nonblocking = (sock->sock_timeout >= 0.0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } /* Actually negotiate SSL connection */ /* XXX If SSL_do_handshake() returns 0, it's also a failure. */ @@ -656,7 +697,7 @@ static PyObject *PySSL_SSLdo_handshake(PySSLSocket *self) break; } } while (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE); - Py_DECREF(sock); + Py_XDECREF(sock); if (ret < 1) return PySSL_SetError(self, ret, __FILE__, __LINE__); @@ -671,7 +712,7 @@ static PyObject *PySSL_SSLdo_handshake(PySSLSocket *self) return Py_None; error: - Py_DECREF(sock); + Py_XDECREF(sock); return NULL; } @@ -1583,6 +1624,54 @@ on the SSLContext to change the certificate information associated with the\n\ SSLSocket before the cryptographic exchange handshake messages\n"); +static PyObject * +PySSL_get_server_side(PySSLSocket *self, void *c) +{ + return PyBool_FromLong(self->socket_type == PY_SSL_SERVER); +} + +PyDoc_STRVAR(PySSL_get_server_side_doc, +"Whether this is a server-side socket."); + +static PyObject * +PySSL_get_server_hostname(PySSLSocket *self, void *c) +{ + if (self->server_hostname == NULL) + Py_RETURN_NONE; + Py_INCREF(self->server_hostname); + return self->server_hostname; +} + +PyDoc_STRVAR(PySSL_get_server_hostname_doc, +"The currently set server hostname (for SNI)."); + +static PyObject * +PySSL_get_owner(PySSLSocket *self, void *c) +{ + PyObject *owner; + + if (self->owner == NULL) + Py_RETURN_NONE; + + owner = PyWeakref_GetObject(self->owner); + Py_INCREF(owner); + return owner; +} + +static int +PySSL_set_owner(PySSLSocket *self, PyObject *value, void *c) +{ + Py_XDECREF(self->owner); + self->owner = PyWeakref_NewRef(value, NULL); + if (self->owner == NULL) + return -1; + return 0; +} + +PyDoc_STRVAR(PySSL_get_owner_doc, +"The Python-level owner of this object.\ +Passed as \"self\" in servername callback."); + static void PySSL_dealloc(PySSLSocket *self) { @@ -1591,8 +1680,9 @@ static void PySSL_dealloc(PySSLSocket *self) if (self->ssl) SSL_free(self->ssl); Py_XDECREF(self->Socket); - Py_XDECREF(self->ssl_sock); Py_XDECREF(self->ctx); + Py_XDECREF(self->server_hostname); + Py_XDECREF(self->owner); PyObject_Del(self); } @@ -1609,10 +1699,10 @@ check_socket_and_wait_for_timeout(PySocketSockObject *s, int writing) int rc; /* Nothing to do unless we're in timeout mode (not non-blocking) */ - if (s->sock_timeout < 0.0) - return SOCKET_IS_BLOCKING; - else if (s->sock_timeout == 0.0) + if ((s == NULL) || (s->sock_timeout == 0.0)) return SOCKET_IS_NONBLOCKING; + else if (s->sock_timeout < 0.0) + return SOCKET_IS_BLOCKING; /* Guard against closed socket */ if (s->sock_fd < 0) @@ -1671,12 +1761,19 @@ static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args) int sockstate; int err; int nonblocking; - PySocketSockObject *sock = self->Socket; + PySocketSockObject *sock = GET_SOCKET(self); - Py_INCREF(sock); + if (sock != NULL) { + if (((PyObject*)sock) == Py_None) { + _setSSLError("Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); + } if (!PyArg_ParseTuple(args, "s*:write", &buf)) { - Py_DECREF(sock); + Py_XDECREF(sock); return NULL; } @@ -1686,10 +1783,12 @@ static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args) goto error; } - /* just in case the blocking state of the socket has been changed */ - nonblocking = (sock->sock_timeout >= 0.0); - BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); - BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + if (sock != NULL) { + /* just in case the blocking state of the socket has been changed */ + nonblocking = (sock->sock_timeout >= 0.0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } sockstate = check_socket_and_wait_for_timeout(sock, 1); if (sockstate == SOCKET_HAS_TIMED_OUT) { @@ -1733,7 +1832,7 @@ static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args) } } while (err == SSL_ERROR_WANT_READ || err == SSL_ERROR_WANT_WRITE); - Py_DECREF(sock); + Py_XDECREF(sock); PyBuffer_Release(&buf); if (len > 0) return PyInt_FromLong(len); @@ -1741,7 +1840,7 @@ static PyObject *PySSL_SSLwrite(PySSLSocket *self, PyObject *args) return PySSL_SetError(self, len, __FILE__, __LINE__); error: - Py_DECREF(sock); + Py_XDECREF(sock); PyBuffer_Release(&buf); return NULL; } @@ -1781,9 +1880,16 @@ static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args) int sockstate; int err; int nonblocking; - PySocketSockObject *sock = self->Socket; + PySocketSockObject *sock = GET_SOCKET(self); - Py_INCREF(sock); + if (sock != NULL) { + if (((PyObject*)sock) == Py_None) { + _setSSLError("Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); + } buf.obj = NULL; buf.buf = NULL; @@ -1821,10 +1927,12 @@ static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args) } } - /* just in case the blocking state of the socket has been changed */ - nonblocking = (sock->sock_timeout >= 0.0); - BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); - BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + if (sock != NULL) { + /* just in case the blocking state of the socket has been changed */ + nonblocking = (sock->sock_timeout >= 0.0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } do { PySSL_BEGIN_ALLOW_THREADS @@ -1860,7 +1968,7 @@ static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args) } done: - Py_DECREF(sock); + Py_XDECREF(sock); if (!buf_passed) { _PyBytes_Resize(&dest, count); return dest; @@ -1871,7 +1979,7 @@ static PyObject *PySSL_SSLread(PySSLSocket *self, PyObject *args) } error: - Py_DECREF(sock); + Py_XDECREF(sock); if (!buf_passed) Py_XDECREF(dest); else @@ -1888,20 +1996,22 @@ static PyObject *PySSL_SSLshutdown(PySSLSocket *self) { int err, ssl_err, sockstate, nonblocking; int zeros = 0; - PySocketSockObject *sock = self->Socket; + PySocketSockObject *sock = GET_SOCKET(self); - /* Guard against closed socket */ - if (sock->sock_fd < 0) { - _setSSLError("Underlying socket connection gone", - PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); - return NULL; - } - Py_INCREF(sock); + if (sock != NULL) { + /* Guard against closed socket */ + if ((((PyObject*)sock) == Py_None) || (sock->sock_fd < 0)) { + _setSSLError("Underlying socket connection gone", + PY_SSL_ERROR_NO_SOCKET, __FILE__, __LINE__); + return NULL; + } + Py_INCREF(sock); - /* Just in case the blocking state of the socket has been changed */ - nonblocking = (sock->sock_timeout >= 0.0); - BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); - BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + /* Just in case the blocking state of the socket has been changed */ + nonblocking = (sock->sock_timeout >= 0.0); + BIO_set_nbio(SSL_get_rbio(self->ssl), nonblocking); + BIO_set_nbio(SSL_get_wbio(self->ssl), nonblocking); + } while (1) { PySSL_BEGIN_ALLOW_THREADS @@ -1959,15 +2069,17 @@ static PyObject *PySSL_SSLshutdown(PySSLSocket *self) } if (err < 0) { - Py_DECREF(sock); + Py_XDECREF(sock); return PySSL_SetError(self, err, __FILE__, __LINE__); } - else + if (sock) /* It's already INCREF'ed */ return (PyObject *) sock; + else + Py_RETURN_NONE; error: - Py_DECREF(sock); + Py_XDECREF(sock); return NULL; } @@ -2015,6 +2127,12 @@ If the TLS handshake is not yet complete, None is returned"); static PyGetSetDef ssl_getsetlist[] = { {"context", (getter) PySSL_get_context, (setter) PySSL_set_context, PySSL_set_context_doc}, + {"server_side", (getter) PySSL_get_server_side, NULL, + PySSL_get_server_side_doc}, + {"server_hostname", (getter) PySSL_get_server_hostname, NULL, + PySSL_get_server_hostname_doc}, + {"owner", (getter) PySSL_get_owner, (setter) PySSL_set_owner, + PySSL_get_owner_doc}, {NULL}, /* sentinel */ }; @@ -2997,34 +3115,68 @@ load_dh_params(PySSLContext *self, PyObject *filepath) static PyObject * context_wrap_socket(PySSLContext *self, PyObject *args, PyObject *kwds) { - char *kwlist[] = {"sock", "server_side", "server_hostname", "ssl_sock", NULL}; + char *kwlist[] = {"sock", "server_side", "server_hostname", NULL}; PySocketSockObject *sock; int server_side = 0; char *hostname = NULL; - PyObject *hostname_obj, *ssl_sock = Py_None, *res; + PyObject *hostname_obj, *res; /* server_hostname is either None (or absent), or to be encoded using the idna encoding. */ - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!i|O!O:_wrap_socket", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!i|O!:_wrap_socket", kwlist, PySocketModule.Sock_Type, &sock, &server_side, - Py_TYPE(Py_None), &hostname_obj, - &ssl_sock)) { + Py_TYPE(Py_None), &hostname_obj)) { PyErr_Clear(); - if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!iet|O:_wrap_socket", kwlist, + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!iet:_wrap_socket", kwlist, PySocketModule.Sock_Type, &sock, &server_side, - "idna", &hostname, &ssl_sock)) + "idna", &hostname)) return NULL; } - res = (PyObject *) newPySSLSocket(self, sock, server_side, - hostname, ssl_sock); + res = (PyObject *) newPySSLSocket(self, sock, server_side, hostname, + NULL, NULL); if (hostname != NULL) PyMem_Free(hostname); return res; } +static PyObject * +context_wrap_bio(PySSLContext *self, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {"incoming", "outgoing", "server_side", + "server_hostname", NULL}; + int server_side; + char *hostname = NULL; + PyObject *hostname_obj = Py_None, *res; + PySSLMemoryBIO *incoming, *outgoing; + + /* server_hostname is either None (or absent), or to be encoded + using the idna encoding. */ + if (!PyArg_ParseTupleAndKeywords(args, kwds, "O!O!i|O:_wrap_bio", kwlist, + &PySSLMemoryBIO_Type, &incoming, + &PySSLMemoryBIO_Type, &outgoing, + &server_side, &hostname_obj)) + return NULL; + if (hostname_obj != Py_None) { +#if HAVE_SNI + if (!PyArg_Parse(hostname_obj, "et", "idna", &hostname)) + return NULL; +#else + PyErr_SetString(PyExc_ValueError, "server_hostname is not supported " + "by your OpenSSL library"); + return NULL; +#endif + } + + res = (PyObject *) newPySSLSocket(self, NULL, server_side, hostname, + incoming, outgoing); + + PyMem_Free(hostname); + return res; +} + static PyObject * session_stats(PySSLContext *self, PyObject *unused) { @@ -3135,15 +3287,25 @@ _servername_callback(SSL *s, int *al, void *args) ssl = SSL_get_app_data(s); assert(PySSLSocket_Check(ssl)); - if (ssl->ssl_sock == NULL) { - ssl_socket = Py_None; - } else { - ssl_socket = PyWeakref_GetObject(ssl->ssl_sock); - Py_INCREF(ssl_socket); - } - if (ssl_socket == Py_None) { + + /* The servername callback expects a argument that represents the current + * SSL connection and that has a .context attribute that can be changed to + * identify the requested hostname. Since the official API is the Python + * level API we want to pass the callback a Python level object rather than + * a _ssl.SSLSocket instance. If there's an "owner" (typically an + * SSLObject) that will be passed. Otherwise if there's a socket then that + * will be passed. If both do not exist only then the C-level object is + * passed. */ + if (ssl->owner) + ssl_socket = PyWeakref_GetObject(ssl->owner); + else if (ssl->Socket) + ssl_socket = PyWeakref_GetObject(ssl->Socket); + else + ssl_socket = (PyObject *) ssl; + + Py_INCREF(ssl_socket); + if (ssl_socket == Py_None) goto error; - } if (servername == NULL) { result = PyObject_CallFunctionObjArgs(ssl_ctx->set_hostname, ssl_socket, @@ -3377,6 +3539,8 @@ static PyGetSetDef context_getsetlist[] = { static struct PyMethodDef context_methods[] = { {"_wrap_socket", (PyCFunction) context_wrap_socket, METH_VARARGS | METH_KEYWORDS, NULL}, + {"_wrap_bio", (PyCFunction) context_wrap_bio, + METH_VARARGS | METH_KEYWORDS, NULL}, {"set_ciphers", (PyCFunction) set_ciphers, METH_VARARGS, NULL}, {"_set_alpn_protocols", (PyCFunction) _set_alpn_protocols, @@ -3448,6 +3612,225 @@ static PyTypeObject PySSLContext_Type = { }; +/* + * MemoryBIO objects + */ + +static PyObject * +memory_bio_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +{ + char *kwlist[] = {NULL}; + BIO *bio; + PySSLMemoryBIO *self; + + if (!PyArg_ParseTupleAndKeywords(args, kwds, ":MemoryBIO", kwlist)) + return NULL; + + bio = BIO_new(BIO_s_mem()); + if (bio == NULL) { + PyErr_SetString(PySSLErrorObject, + "failed to allocate BIO"); + return NULL; + } + /* Since our BIO is non-blocking an empty read() does not indicate EOF, + * just that no data is currently available. The SSL routines should retry + * the read, which we can achieve by calling BIO_set_retry_read(). */ + BIO_set_retry_read(bio); + BIO_set_mem_eof_return(bio, -1); + + assert(type != NULL && type->tp_alloc != NULL); + self = (PySSLMemoryBIO *) type->tp_alloc(type, 0); + if (self == NULL) { + BIO_free(bio); + return NULL; + } + self->bio = bio; + self->eof_written = 0; + + return (PyObject *) self; +} + +static void +memory_bio_dealloc(PySSLMemoryBIO *self) +{ + BIO_free(self->bio); + Py_TYPE(self)->tp_free(self); +} + +static PyObject * +memory_bio_get_pending(PySSLMemoryBIO *self, void *c) +{ + return PyLong_FromLong(BIO_ctrl_pending(self->bio)); +} + +PyDoc_STRVAR(PySSL_memory_bio_pending_doc, +"The number of bytes pending in the memory BIO."); + +static PyObject * +memory_bio_get_eof(PySSLMemoryBIO *self, void *c) +{ + return PyBool_FromLong((BIO_ctrl_pending(self->bio) == 0) + && self->eof_written); +} + +PyDoc_STRVAR(PySSL_memory_bio_eof_doc, +"Whether the memory BIO is at EOF."); + +static PyObject * +memory_bio_read(PySSLMemoryBIO *self, PyObject *args) +{ + int len = -1, avail, nbytes; + PyObject *result; + + if (!PyArg_ParseTuple(args, "|i:read", &len)) + return NULL; + + avail = BIO_ctrl_pending(self->bio); + if ((len < 0) || (len > avail)) + len = avail; + + result = PyBytes_FromStringAndSize(NULL, len); + if ((result == NULL) || (len == 0)) + return result; + + nbytes = BIO_read(self->bio, PyBytes_AS_STRING(result), len); + /* There should never be any short reads but check anyway. */ + if ((nbytes < len) && (_PyBytes_Resize(&result, len) < 0)) { + Py_DECREF(result); + return NULL; + } + + return result; +} + +PyDoc_STRVAR(PySSL_memory_bio_read_doc, +"read([len]) -> bytes\n\ +\n\ +Read up to len bytes from the memory BIO.\n\ +\n\ +If len is not specified, read the entire buffer.\n\ +If the return value is an empty bytes instance, this means either\n\ +EOF or that no data is available. Use the \"eof\" property to\n\ +distinguish between the two."); + +static PyObject * +memory_bio_write(PySSLMemoryBIO *self, PyObject *args) +{ + Py_buffer buf; + int nbytes; + + if (!PyArg_ParseTuple(args, "s*:write", &buf)) + return NULL; + + if (buf.len > INT_MAX) { + PyErr_Format(PyExc_OverflowError, + "string longer than %d bytes", INT_MAX); + goto error; + } + + if (self->eof_written) { + PyErr_SetString(PySSLErrorObject, + "cannot write() after write_eof()"); + goto error; + } + + nbytes = BIO_write(self->bio, buf.buf, buf.len); + if (nbytes < 0) { + _setSSLError(NULL, 0, __FILE__, __LINE__); + goto error; + } + + PyBuffer_Release(&buf); + return PyLong_FromLong(nbytes); + +error: + PyBuffer_Release(&buf); + return NULL; +} + +PyDoc_STRVAR(PySSL_memory_bio_write_doc, +"write(b) -> len\n\ +\n\ +Writes the bytes b into the memory BIO. Returns the number\n\ +of bytes written."); + +static PyObject * +memory_bio_write_eof(PySSLMemoryBIO *self, PyObject *args) +{ + self->eof_written = 1; + /* After an EOF is written, a zero return from read() should be a real EOF + * i.e. it should not be retried. Clear the SHOULD_RETRY flag. */ + BIO_clear_retry_flags(self->bio); + BIO_set_mem_eof_return(self->bio, 0); + + Py_RETURN_NONE; +} + +PyDoc_STRVAR(PySSL_memory_bio_write_eof_doc, +"write_eof()\n\ +\n\ +Write an EOF marker to the memory BIO.\n\ +When all data has been read, the \"eof\" property will be True."); + +static PyGetSetDef memory_bio_getsetlist[] = { + {"pending", (getter) memory_bio_get_pending, NULL, + PySSL_memory_bio_pending_doc}, + {"eof", (getter) memory_bio_get_eof, NULL, + PySSL_memory_bio_eof_doc}, + {NULL}, /* sentinel */ +}; + +static struct PyMethodDef memory_bio_methods[] = { + {"read", (PyCFunction) memory_bio_read, + METH_VARARGS, PySSL_memory_bio_read_doc}, + {"write", (PyCFunction) memory_bio_write, + METH_VARARGS, PySSL_memory_bio_write_doc}, + {"write_eof", (PyCFunction) memory_bio_write_eof, + METH_NOARGS, PySSL_memory_bio_write_eof_doc}, + {NULL, NULL} /* sentinel */ +}; + +static PyTypeObject PySSLMemoryBIO_Type = { + PyVarObject_HEAD_INIT(NULL, 0) + "_ssl.MemoryBIO", /*tp_name*/ + sizeof(PySSLMemoryBIO), /*tp_basicsize*/ + 0, /*tp_itemsize*/ + (destructor)memory_bio_dealloc, /*tp_dealloc*/ + 0, /*tp_print*/ + 0, /*tp_getattr*/ + 0, /*tp_setattr*/ + 0, /*tp_reserved*/ + 0, /*tp_repr*/ + 0, /*tp_as_number*/ + 0, /*tp_as_sequence*/ + 0, /*tp_as_mapping*/ + 0, /*tp_hash*/ + 0, /*tp_call*/ + 0, /*tp_str*/ + 0, /*tp_getattro*/ + 0, /*tp_setattro*/ + 0, /*tp_as_buffer*/ + Py_TPFLAGS_DEFAULT, /*tp_flags*/ + 0, /*tp_doc*/ + 0, /*tp_traverse*/ + 0, /*tp_clear*/ + 0, /*tp_richcompare*/ + 0, /*tp_weaklistoffset*/ + 0, /*tp_iter*/ + 0, /*tp_iternext*/ + memory_bio_methods, /*tp_methods*/ + 0, /*tp_members*/ + memory_bio_getsetlist, /*tp_getset*/ + 0, /*tp_base*/ + 0, /*tp_dict*/ + 0, /*tp_descr_get*/ + 0, /*tp_descr_set*/ + 0, /*tp_dictoffset*/ + 0, /*tp_init*/ + 0, /*tp_alloc*/ + memory_bio_new, /*tp_new*/ +}; + #ifdef HAVE_OPENSSL_RAND @@ -4070,6 +4453,8 @@ init_ssl(void) return; if (PyType_Ready(&PySSLSocket_Type) < 0) return; + if (PyType_Ready(&PySSLMemoryBIO_Type) < 0) + return; m = Py_InitModule3("_ssl", PySSL_methods, module_doc); if (m == NULL) @@ -4145,6 +4530,9 @@ init_ssl(void) if (PyDict_SetItemString(d, "_SSLSocket", (PyObject *)&PySSLSocket_Type) != 0) return; + if (PyDict_SetItemString(d, "MemoryBIO", + (PyObject *)&PySSLMemoryBIO_Type) != 0) + return; PyModule_AddIntConstant(m, "SSL_ERROR_ZERO_RETURN", PY_SSL_ERROR_ZERO_RETURN); PyModule_AddIntConstant(m, "SSL_ERROR_WANT_READ", From df550ad2ae196ab3eed4c43cc5b19f3fb4e36c8f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 12 Jun 2017 17:08:07 +0200 Subject: [PATCH 2/7] Update ssl.py from master --- Lib/ssl.py | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/Lib/ssl.py b/Lib/ssl.py index 9898e07310cb6c..10bacadfa74517 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -569,7 +569,7 @@ def server_hostname(self): server hostame is set.""" return self._sslobj.server_hostname - def read(self, len=0, buffer=None): + def read(self, len=1024, buffer=None): """Read up to 'len' bytes from the SSL object and return them. If 'buffer' is provided, read into this buffer and return the number of @@ -578,7 +578,7 @@ def read(self, len=0, buffer=None): if buffer is not None: v = self._sslobj.read(len, buffer) else: - v = self._sslobj.read(len or 1024) + v = self._sslobj.read(len) return v def write(self, data): @@ -605,6 +605,13 @@ def selected_npn_protocol(self): if _ssl.HAS_NPN: return self._sslobj.selected_npn_protocol() + def selected_alpn_protocol(self): + """Return the currently selected ALPN protocol as a string, or ``None`` + if a next protocol was not negotiated or if ALPN is not supported by one + of the peers.""" + if _ssl.HAS_ALPN: + return self._sslobj.selected_alpn_protocol() + def cipher(self): """Return the currently selected cipher as a 3-tuple ``(name, ssl_version, secret_bits)``.""" @@ -841,17 +848,7 @@ def send(self, data, flags=0): raise ValueError( "non-zero flags not allowed in calls to send() on %s" % self.__class__) - try: - v = self._sslobj.write(data) - except SSLError as x: - if x.args[0] == SSL_ERROR_WANT_READ: - return 0 - elif x.args[0] == SSL_ERROR_WANT_WRITE: - return 0 - else: - raise - else: - return v + return self._sslobj.write(data) else: return self._sock.send(data, flags) From 6c0eab963af8cfa9eb5ff15fb6e59db786ad6cf6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 5 Jul 2017 11:04:43 +0200 Subject: [PATCH 3/7] Address Alex Gaynor and my own comments --- Doc/library/ssl.rst | 12 ++---------- Lib/ssl.py | 13 +++++++++++-- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index c85fb791579cf4..61aa5f8a370007 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1774,20 +1774,11 @@ to be aware of: except ssl.SSLWantWriteError: select.select([], [sock], []) -.. seealso:: - - The :mod:`asyncio` module supports :ref:`non-blocking SSL sockets - ` and provides a - higher level API. It polls for events using the :mod:`selectors` module and - handles :exc:`SSLWantWriteError`, :exc:`SSLWantReadError` and - :exc:`BlockingIOError` exceptions. It runs the SSL handshake asynchronously - as well. - Memory BIO Support ------------------ -.. versionadded:: 2.7.9 +.. versionadded:: 2.7.14 Ever since the SSL module was introduced in Python 2.6, the :class:`SSLSocket` class has provided two related but distinct areas of functionality: @@ -1842,6 +1833,7 @@ provided. - :meth:`~SSLSocket.do_handshake` - :meth:`~SSLSocket.unwrap` - :meth:`~SSLSocket.get_channel_binding` + - :meth:`~SSLSocket.selected_alpn_protocol` When compared to :class:`SSLSocket`, this object lacks the following features: diff --git a/Lib/ssl.py b/Lib/ssl.py index 10bacadfa74517..92461369d471f5 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -527,7 +527,6 @@ def _https_verify_certificates(enable=True): _create_default_https_context = _create_unverified_context - class SSLObject(object): """This class implements an interface on top of a low-level SSL object as implemented by OpenSSL. This object captures the state of an SSL connection @@ -848,7 +847,17 @@ def send(self, data, flags=0): raise ValueError( "non-zero flags not allowed in calls to send() on %s" % self.__class__) - return self._sslobj.write(data) + try: + v = self._sslobj.write(data) + except SSLError as x: + if x.args[0] == SSL_ERROR_WANT_READ: + return 0 + elif x.args[0] == SSL_ERROR_WANT_WRITE: + return 0 + else: + raise + else: + return v else: return self._sock.send(data, flags) From f9fe9a94c0eb929c5066ba7820dd7b524856c676 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 21 Jul 2017 02:24:03 +0200 Subject: [PATCH 4/7] test_ssl: Replace svn.python.org with REMOTE_HOST Patch written by Alex Gaynor. --- Lib/test/test_ssl.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index 4e3473466217c7..c99db781c1fe77 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -1757,17 +1757,17 @@ def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): return ret def test_handshake(self): - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): sock = socket.socket(socket.AF_INET) - sock.connect(("svn.python.org", 443)) + sock.connect((REMOTE_HOST, 443)) incoming = ssl.MemoryBIO() outgoing = ssl.MemoryBIO() ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) ctx.verify_mode = ssl.CERT_REQUIRED - ctx.load_verify_locations(SVN_PYTHON_ORG_ROOT_CERT) + ctx.load_verify_locations(REMOTE_ROOT_CERT) if ssl.HAS_SNI: ctx.check_hostname = True - sslobj = ctx.wrap_bio(incoming, outgoing, False, 'svn.python.org') + sslobj = ctx.wrap_bio(incoming, outgoing, False, REMOTE_HOST) else: ctx.check_hostname = False sslobj = ctx.wrap_bio(incoming, outgoing, False) @@ -1786,9 +1786,9 @@ def test_handshake(self): sock.close() def test_read_write_data(self): - with support.transient_internet("svn.python.org"): + with support.transient_internet(REMOTE_HOST): sock = socket.socket(socket.AF_INET) - sock.connect(("svn.python.org", 443)) + sock.connect((REMOTE_HOST, 443)) incoming = ssl.MemoryBIO() outgoing = ssl.MemoryBIO() ctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) From 95c40ccbb88d2abe886e12033e3cc3e1ad10db70 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 21 Jul 2017 02:28:33 +0200 Subject: [PATCH 5/7] Oops, remove duplicated documentation --- Doc/library/ssl.rst | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/Doc/library/ssl.rst b/Doc/library/ssl.rst index 61aa5f8a370007..0d214768066c95 100644 --- a/Doc/library/ssl.rst +++ b/Doc/library/ssl.rst @@ -1065,22 +1065,6 @@ SSL sockets also have the following additional methods and attributes: .. versionadded:: 2.7.9 -.. attribute:: SSLSocket.server_hostname - - A ``bytes`` instance containing the ``'idna'`` encoded version of the - hostname specified in the *server_hostname* argument in - :meth:`SSLContext.wrap_socket`. If no *server_hostname* was specified, this - attribute will be ``None``. - - .. versionadded:: 2.7.9 - -.. attribute:: SSLSocket.server_side - - A boolean which is ``True`` for server-side sockets and ``False`` for - client-side sockets. - - .. versionadded:: 2.7.9 - .. attribute:: SSLSocket.server_hostname Hostname of the server: :class:`str` type, or ``None`` for server-side From 980b3f35603b76d58a6c8388e8f5cbcdff9fe5cf Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Sep 2017 20:00:39 -0700 Subject: [PATCH 6/7] Replace CRYPTO_add() with BIO_up_ref() for OpenSSL 1.1 --- Modules/_ssl.c | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Modules/_ssl.c b/Modules/_ssl.c index 8202fd0262b60d..defb50ed4adcde 100644 --- a/Modules/_ssl.c +++ b/Modules/_ssl.c @@ -171,6 +171,12 @@ static X509 *X509_OBJECT_get0_X509(X509_OBJECT *x) return x->data.x509; } +static int BIO_up_ref(BIO *b) +{ + CRYPTO_add(&b->references, 1, CRYPTO_LOCK_BIO); + return 1; +} + static STACK_OF(X509_OBJECT) *X509_STORE_get0_objects(X509_STORE *store) { return store->objs; } @@ -599,8 +605,8 @@ newPySSLSocket(PySSLContext *sslctx, PySocketSockObject *sock, /* BIOs are reference counted and SSL_set_bio borrows our reference. * To prevent a double free in memory_bio_dealloc() we need to take an * extra reference here. */ - CRYPTO_add(&inbio->bio->references, 1, CRYPTO_LOCK_BIO); - CRYPTO_add(&outbio->bio->references, 1, CRYPTO_LOCK_BIO); + BIO_up_ref(inbio->bio); + BIO_up_ref(outbio->bio); SSL_set_bio(self->ssl, inbio->bio, outbio->bio); } mode = SSL_MODE_ACCEPT_MOVING_WRITE_BUFFER; From d52ab101f5c1e92e8a10db9900ad1ede69a6205f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 7 Sep 2017 20:02:35 -0700 Subject: [PATCH 7/7] Add NEWS entry. --- .../next/Security/2017-09-07-20-02-32.bpo-22559.RGZuaq.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Security/2017-09-07-20-02-32.bpo-22559.RGZuaq.rst diff --git a/Misc/NEWS.d/next/Security/2017-09-07-20-02-32.bpo-22559.RGZuaq.rst b/Misc/NEWS.d/next/Security/2017-09-07-20-02-32.bpo-22559.RGZuaq.rst new file mode 100644 index 00000000000000..00cbd6639b1d3c --- /dev/null +++ b/Misc/NEWS.d/next/Security/2017-09-07-20-02-32.bpo-22559.RGZuaq.rst @@ -0,0 +1 @@ +Implementation of the PEP 546: Backport MemoryBIO.