From 5943fa462c4b2afe297b06aaef911efcf935c0d7 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 11 Aug 2021 16:51:03 +0200 Subject: [PATCH 1/5] Backport PyModule_AddObjectRef as _PyModule_AddObjectRef Having PyModule_AddObjectRef available should make backporting newer patches easier. The new API is much safer. The backport adds an underscore so that we don't break extension modules that define PyModule_AddObjectRef themselves on Python<=3.9 (which would be a virtuous thing to do). --- Include/modsupport.h | 10 ++++++++++ Python/modsupport.c | 13 +++++++++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/Include/modsupport.h b/Include/modsupport.h index 4c4aab65ba..d9fac52521 100644 --- a/Include/modsupport.h +++ b/Include/modsupport.h @@ -136,7 +136,17 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords( void _PyArg_Fini(void); #endif /* Py_LIMITED_API */ +// Add an attribute with name 'name' and value 'obj' to the module 'mod. +// On success, return 0 on success. +// On error, raise an exception and return -1. +// Backported from Python 3.10, where it's available without the underscore +// in the name, to ease porting patches to RHEL +PyAPI_FUNC(int) _PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value); + +// Similar to PyModule_AddObjectRef() but steal a reference to 'obj' +// (Py_DECREF(obj)) on success (if it returns 0). PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *); + PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long); PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *); #if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000 diff --git a/Python/modsupport.c b/Python/modsupport.c index 13482c6508..fca1083e2d 100644 --- a/Python/modsupport.c +++ b/Python/modsupport.c @@ -631,7 +631,7 @@ va_build_stack(PyObject **small_stack, Py_ssize_t small_stack_len, int -PyModule_AddObject(PyObject *m, const char *name, PyObject *o) +_PyModule_AddObjectRef(PyObject *m, const char *name, PyObject *o) { PyObject *dict; if (!PyModule_Check(m)) { @@ -655,10 +655,19 @@ PyModule_AddObject(PyObject *m, const char *name, PyObject *o) } if (PyDict_SetItemString(dict, name, o)) return -1; - Py_DECREF(o); return 0; } +int +PyModule_AddObject(PyObject *mod, const char *name, PyObject *value) +{ + int res = _PyModule_AddObjectRef(mod, name, value); + if (res == 0) { + Py_DECREF(value); + } + return res; +} + int PyModule_AddIntConstant(PyObject *m, const char *name, long value) { -- 2.31.1 From d6ecb3ac9a626af2ecd782d6c7b452251713c4a6 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Fri, 13 Aug 2021 13:16:43 +0200 Subject: [PATCH 2/5] _hashopenssl: Uncomment and use initialization function list This simplifies backporting of future changes. We use this change instead of Python 3.10's: bpo-1635741: Port _hashlib to multiphase initialization (GH-23358) --- Modules/_hashopenssl.c | 26 +++++--------------------- 1 file changed, 5 insertions(+), 21 deletions(-) diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index ff3a1aef5e..db661dc5f7 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -2085,7 +2085,6 @@ hashlib_init_hmactype(PyObject *module) return 0; } -#if 0 static PyModuleDef_Slot hashlib_slots[] = { /* OpenSSL 1.0.2 and LibreSSL */ {Py_mod_exec, hashlib_openssl_legacy_init}, @@ -2095,7 +2094,6 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_md_meth_names}, {0, NULL} }; -#endif static struct PyModuleDef _hashlibmodule = { PyModuleDef_HEAD_INIT, @@ -2123,25 +2121,11 @@ PyInit__hashlib(void) return NULL; } - if (hashlib_openssl_legacy_init(m) < 0) { - Py_DECREF(m); - return NULL; - } - if (hashlib_init_evptype(m) < 0) { - Py_DECREF(m); - return NULL; - } - if (hashlib_init_evpxoftype(m) < 0) { - Py_DECREF(m); - return NULL; - } - if (hashlib_init_hmactype(m) < 0) { - Py_DECREF(m); - return NULL; - } - if (hashlib_md_meth_names(m) == -1) { - Py_DECREF(m); - return NULL; + for (int i=0; hashlib_slots[i].slot; i++) { + if (((int (*)(PyObject*))hashlib_slots[i].value)(m) < 0) { + Py_DECREF(m); + return NULL; + } } return m; -- 2.31.1 From ce54104e69b9f3e9b3f49cec2054b66e84a5cbb8 Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sat, 27 Mar 2021 14:55:03 +0100 Subject: [PATCH 3/5] bpo-40645: use C implementation of HMAC (GH-24920, GH-25063, GH-26079) This backports the feature and 2 subsequent bugfixes from: https://bugs.python.org/issue40645 Signed-off-by: Christian Heimes Co-authored-by: Erlend Egeberg Aasland Co-Authored-By: Pablo Galindo --- Lib/hashlib.py | 1 + Lib/hmac.py | 86 ++++++---- Lib/test/test_hmac.py | 114 ++++++++----- .../2021-03-29-11-55-06.bpo-40645.PhaT-B.rst | 2 + .../2021-03-19-10-22-17.bpo-40645.5pXhb-.rst | 2 + Modules/_hashopenssl.c | 158 ++++++++++++++++-- Modules/clinic/_hashopenssl.c.h | 38 +---- 7 files changed, 276 insertions(+), 125 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst create mode 100644 Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst diff --git a/Lib/hashlib.py b/Lib/hashlib.py index 58c340d56e..ffa3be049a 100644 --- a/Lib/hashlib.py +++ b/Lib/hashlib.py @@ -173,6 +173,7 @@ try: algorithms_available = algorithms_available.union( _hashlib.openssl_md_meth_names) except ImportError: + _hashlib = None new = __py_new __get_hash = __get_builtin_constructor diff --git a/Lib/hmac.py b/Lib/hmac.py index 180bc378b5..8b4f920db9 100644 --- a/Lib/hmac.py +++ b/Lib/hmac.py @@ -8,11 +8,12 @@ try: import _hashlib as _hashopenssl except ImportError: _hashopenssl = None - _openssl_md_meths = None + _functype = None from _operator import _compare_digest as compare_digest else: - _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names) compare_digest = _hashopenssl.compare_digest + _functype = type(_hashopenssl.openssl_sha256) # builtin type + import hashlib as _hashlib trans_5C = bytes((x ^ 0x5C) for x in range(256)) @@ -23,7 +24,6 @@ trans_36 = bytes((x ^ 0x36) for x in range(256)) digest_size = None - class HMAC: """RFC 2104 HMAC class. Also complies with RFC 4231. @@ -32,7 +32,7 @@ class HMAC: blocksize = 64 # 512-bit HMAC; can be changed in subclasses. __slots__ = ( - "_digest_cons", "_inner", "_outer", "block_size", "digest_size" + "_hmac", "_inner", "_outer", "block_size", "digest_size" ) def __init__(self, key, msg=None, digestmod=''): @@ -55,15 +55,30 @@ class HMAC: if not digestmod: raise TypeError("Missing required parameter 'digestmod'.") + if _hashopenssl and isinstance(digestmod, (str, _functype)): + try: + self._init_hmac(key, msg, digestmod) + except _hashopenssl.UnsupportedDigestmodError: + self._init_old(key, msg, digestmod) + else: + self._init_old(key, msg, digestmod) + + def _init_hmac(self, key, msg, digestmod): + self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod) + self.digest_size = self._hmac.digest_size + self.block_size = self._hmac.block_size + + def _init_old(self, key, msg, digestmod): if callable(digestmod): - self._digest_cons = digestmod + digest_cons = digestmod elif isinstance(digestmod, str): - self._digest_cons = lambda d=b'': _hashlib.new(digestmod, d) + digest_cons = lambda d=b'': _hashlib.new(digestmod, d) else: - self._digest_cons = lambda d=b'': digestmod.new(d) + digest_cons = lambda d=b'': digestmod.new(d) - self._outer = self._digest_cons() - self._inner = self._digest_cons() + self._hmac = None + self._outer = digest_cons() + self._inner = digest_cons() self.digest_size = self._inner.digest_size if hasattr(self._inner, 'block_size'): @@ -79,13 +94,13 @@ class HMAC: RuntimeWarning, 2) blocksize = self.blocksize + if len(key) > blocksize: + key = digest_cons(key).digest() + # self.blocksize is the default blocksize. self.block_size is # effective block size as well as the public API attribute. self.block_size = blocksize - if len(key) > blocksize: - key = self._digest_cons(key).digest() - key = key.ljust(blocksize, b'\0') self._outer.update(key.translate(trans_5C)) self._inner.update(key.translate(trans_36)) @@ -94,23 +109,15 @@ class HMAC: @property def name(self): - return "hmac-" + self._inner.name - - @property - def digest_cons(self): - return self._digest_cons - - @property - def inner(self): - return self._inner - - @property - def outer(self): - return self._outer + if self._hmac: + return self._hmac.name + else: + return f"hmac-{self._inner.name}" def update(self, msg): """Feed data from msg into this hashing object.""" - self._inner.update(msg) + inst = self._hmac or self._inner + inst.update(msg) def copy(self): """Return a separate copy of this hashing object. @@ -119,10 +126,14 @@ class HMAC: """ # Call __new__ directly to avoid the expensive __init__. other = self.__class__.__new__(self.__class__) - other._digest_cons = self._digest_cons other.digest_size = self.digest_size - other._inner = self._inner.copy() - other._outer = self._outer.copy() + if self._hmac: + other._hmac = self._hmac.copy() + other._inner = other._outer = None + else: + other._hmac = None + other._inner = self._inner.copy() + other._outer = self._outer.copy() return other def _current(self): @@ -130,9 +141,12 @@ class HMAC: To be used only internally with digest() and hexdigest(). """ - h = self._outer.copy() - h.update(self._inner.digest()) - return h + if self._hmac: + return self._hmac + else: + h = self._outer.copy() + h.update(self._inner.digest()) + return h def digest(self): """Return the hash value of this hashing object. @@ -179,9 +193,11 @@ def digest(key, msg, digest): A hashlib constructor returning a new hash object. *OR* A module supporting PEP 247. """ - if (_hashopenssl is not None and - isinstance(digest, str) and digest in _openssl_md_meths): - return _hashopenssl.hmac_digest(key, msg, digest) + if _hashopenssl is not None and isinstance(digest, (str, _functype)): + try: + return _hashopenssl.hmac_digest(key, msg, digest) + except _hashopenssl.UnsupportedDigestmodError: + pass if callable(digest): digest_cons = digest diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py index 6daf22ca06..adf52adbf2 100644 --- a/Lib/test/test_hmac.py +++ b/Lib/test/test_hmac.py @@ -11,14 +11,21 @@ from test.support import hashlib_helper from _operator import _compare_digest as operator_compare_digest try: + import _hashlib as _hashopenssl from _hashlib import HMAC as C_HMAC from _hashlib import hmac_new as c_hmac_new from _hashlib import compare_digest as openssl_compare_digest except ImportError: + _hashopenssl = None C_HMAC = None c_hmac_new = None openssl_compare_digest = None +try: + import _sha256 as sha256_module +except ImportError: + sha256_module = None + def ignore_warning(func): @functools.wraps(func) @@ -32,22 +39,27 @@ def ignore_warning(func): class TestVectorsTestCase(unittest.TestCase): - def asssert_hmac( - self, key, data, digest, hashfunc, hashname, digest_size, block_size + def assert_hmac_internals( + self, h, digest, hashname, digest_size, block_size ): - h = hmac.HMAC(key, data, digestmod=hashfunc) self.assertEqual(h.hexdigest().upper(), digest.upper()) self.assertEqual(h.digest(), binascii.unhexlify(digest)) self.assertEqual(h.name, f"hmac-{hashname}") self.assertEqual(h.digest_size, digest_size) self.assertEqual(h.block_size, block_size) + def assert_hmac( + self, key, data, digest, hashfunc, hashname, digest_size, block_size + ): + h = hmac.HMAC(key, data, digestmod=hashfunc) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) + h = hmac.HMAC(key, data, digestmod=hashname) - self.assertEqual(h.hexdigest().upper(), digest.upper()) - self.assertEqual(h.digest(), binascii.unhexlify(digest)) - self.assertEqual(h.name, f"hmac-{hashname}") - self.assertEqual(h.digest_size, digest_size) - self.assertEqual(h.block_size, block_size) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) h = hmac.HMAC(key, digestmod=hashname) h2 = h.copy() @@ -56,11 +68,9 @@ class TestVectorsTestCase(unittest.TestCase): self.assertEqual(h.hexdigest().upper(), digest.upper()) h = hmac.new(key, data, digestmod=hashname) - self.assertEqual(h.hexdigest().upper(), digest.upper()) - self.assertEqual(h.digest(), binascii.unhexlify(digest)) - self.assertEqual(h.name, f"hmac-{hashname}") - self.assertEqual(h.digest_size, digest_size) - self.assertEqual(h.block_size, block_size) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) h = hmac.new(key, None, digestmod=hashname) h.update(data) @@ -81,23 +91,18 @@ class TestVectorsTestCase(unittest.TestCase): hmac.digest(key, data, digest=hashfunc), binascii.unhexlify(digest) ) - with unittest.mock.patch('hmac._openssl_md_meths', {}): - self.assertEqual( - hmac.digest(key, data, digest=hashname), - binascii.unhexlify(digest) - ) - self.assertEqual( - hmac.digest(key, data, digest=hashfunc), - binascii.unhexlify(digest) - ) + + h = hmac.HMAC.__new__(hmac.HMAC) + h._init_old(key, data, digestmod=hashname) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) if c_hmac_new is not None: h = c_hmac_new(key, data, digestmod=hashname) - self.assertEqual(h.hexdigest().upper(), digest.upper()) - self.assertEqual(h.digest(), binascii.unhexlify(digest)) - self.assertEqual(h.name, f"hmac-{hashname}") - self.assertEqual(h.digest_size, digest_size) - self.assertEqual(h.block_size, block_size) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) h = c_hmac_new(key, digestmod=hashname) h2 = h.copy() @@ -105,12 +110,24 @@ class TestVectorsTestCase(unittest.TestCase): h.update(data) self.assertEqual(h.hexdigest().upper(), digest.upper()) + func = getattr(_hashopenssl, f"openssl_{hashname}") + h = c_hmac_new(key, data, digestmod=func) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) + + h = hmac.HMAC.__new__(hmac.HMAC) + h._init_hmac(key, data, digestmod=hashname) + self.assert_hmac_internals( + h, digest, hashname, digest_size, block_size + ) + @hashlib_helper.requires_hashdigest('md5', openssl=True) def test_md5_vectors(self): # Test the HMAC module against test vectors from the RFC. def md5test(key, data, digest): - self.asssert_hmac( + self.assert_hmac( key, data, digest, hashfunc=hashlib.md5, hashname="md5", @@ -150,7 +167,7 @@ class TestVectorsTestCase(unittest.TestCase): @hashlib_helper.requires_hashdigest('sha1', openssl=True) def test_sha_vectors(self): def shatest(key, data, digest): - self.asssert_hmac( + self.assert_hmac( key, data, digest, hashfunc=hashlib.sha1, hashname="sha1", @@ -191,7 +208,7 @@ class TestVectorsTestCase(unittest.TestCase): def hmactest(key, data, hexdigests): digest = hexdigests[hashfunc] - self.asssert_hmac( + self.assert_hmac( key, data, digest, hashfunc=hashfunc, hashname=hash_name, @@ -427,6 +444,15 @@ class ConstructorTestCase(unittest.TestCase): ): C_HMAC() + @unittest.skipUnless(sha256_module is not None, 'need _sha256') + def test_with_sha256_module(self): + h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256) + self.assertEqual(h.hexdigest(), self.expected) + self.assertEqual(h.name, "hmac-sha256") + + digest = hmac.digest(b"key", b"hash this!", sha256_module.sha256) + self.assertEqual(digest, binascii.unhexlify(self.expected)) + class SanityTestCase(unittest.TestCase): @@ -447,21 +473,21 @@ class SanityTestCase(unittest.TestCase): class CopyTestCase(unittest.TestCase): @hashlib_helper.requires_hashdigest('sha256') - def test_attributes(self): + def test_attributes_old(self): # Testing if attributes are of same type. - h1 = hmac.HMAC(b"key", digestmod="sha256") + h1 = hmac.HMAC.__new__(hmac.HMAC) + h1._init_old(b"key", b"msg", digestmod="sha256") h2 = h1.copy() - self.assertTrue(h1._digest_cons == h2._digest_cons, - "digest constructors don't match.") self.assertEqual(type(h1._inner), type(h2._inner), "Types of inner don't match.") self.assertEqual(type(h1._outer), type(h2._outer), "Types of outer don't match.") @hashlib_helper.requires_hashdigest('sha256') - def test_realcopy(self): + def test_realcopy_old(self): # Testing if the copy method created a real copy. - h1 = hmac.HMAC(b"key", digestmod="sha256") + h1 = hmac.HMAC.__new__(hmac.HMAC) + h1._init_old(b"key", b"msg", digestmod="sha256") h2 = h1.copy() # Using id() in case somebody has overridden __eq__/__ne__. self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.") @@ -469,17 +495,15 @@ class CopyTestCase(unittest.TestCase): "No real copy of the attribute 'inner'.") self.assertTrue(id(h1._outer) != id(h2._outer), "No real copy of the attribute 'outer'.") - self.assertEqual(h1._inner, h1.inner) - self.assertEqual(h1._outer, h1.outer) - self.assertEqual(h1._digest_cons, h1.digest_cons) + self.assertIs(h1._hmac, None) + @unittest.skipIf(_hashopenssl is None, "test requires _hashopenssl") @hashlib_helper.requires_hashdigest('sha256') - def test_properties(self): - # deprecated properties - h1 = hmac.HMAC(b"key", digestmod="sha256") - self.assertEqual(h1._inner, h1.inner) - self.assertEqual(h1._outer, h1.outer) - self.assertEqual(h1._digest_cons, h1.digest_cons) + def test_realcopy_hmac(self): + h1 = hmac.HMAC.__new__(hmac.HMAC) + h1._init_hmac(b"key", b"msg", digestmod="sha256") + h2 = h1.copy() + self.assertTrue(id(h1._hmac) != id(h2._hmac)) @hashlib_helper.requires_hashdigest('sha256') def test_equality(self): diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst b/Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst new file mode 100644 index 0000000000..9ca9843947 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst @@ -0,0 +1,2 @@ +Fix reference leak in the :mod:`_hashopenssl` extension. Patch by Pablo +Galindo. diff --git a/Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst b/Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst new file mode 100644 index 0000000000..a9ab1c0915 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst @@ -0,0 +1,2 @@ +The :mod:`hmac` module now uses OpenSSL's HMAC implementation when digestmod +argument is a hash name or builtin hash function. diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index db661dc5f7..fd3b3c0719 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -92,6 +92,8 @@ typedef struct { #ifdef PY_OPENSSL_HAS_SHAKE PyTypeObject *EVPXOFtype; #endif + PyObject *constructs; + PyObject *unsupported_digestmod_error; } _hashlibstate; static inline _hashlibstate* @@ -295,9 +297,56 @@ py_digest_by_name(const char *name) #endif } + if (digest == NULL) { + PyErr_Format(PyExc_ValueError, "unsupported hash type %s", name); + return NULL; + } + return digest; } +/* Get digest EVP from object + * + * * string + * * _hashopenssl builtin function + * + * on error returns NULL with exception set. + */ +static const EVP_MD* +py_digest_by_digestmod(PyObject *module, PyObject *digestmod) { + const EVP_MD* evp; + PyObject *name_obj = NULL; + const char *name; + + if (PyUnicode_Check(digestmod)) { + name_obj = digestmod; + } else { + _hashlibstate *state = get_hashlib_state(module); + // borrowed ref + name_obj = PyDict_GetItem(state->constructs, digestmod); + } + if (name_obj == NULL) { + _hashlibstate *state = get_hashlib_state(module); + PyErr_Clear(); + PyErr_Format( + state->unsupported_digestmod_error, + "Unsupported digestmod %R", digestmod); + return NULL; + } + + name = PyUnicode_AsUTF8(name_obj); + if (name == NULL) { + return NULL; + } + + evp = py_digest_by_name(name); + if (evp == NULL) { + return NULL; + } + + return evp; +} + static EVPobject * newEVPobject(PyTypeObject *type) { @@ -822,7 +871,7 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, /*[clinic end generated code: output=ddd5053f92dffe90 input=c24554d0337be1b0]*/ { Py_buffer view = { 0 }; - PyObject *ret_obj; + PyObject *ret_obj = NULL; char *name; const EVP_MD *digest = NULL; @@ -835,11 +884,15 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj, GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view); digest = py_digest_by_name(name); + if (digest == NULL) { + goto exit; + } ret_obj = EVPnew(module, digest, (unsigned char*)view.buf, view.len, usedforsecurity); +exit: if (data_obj) PyBuffer_Release(&view); return ret_obj; @@ -1130,7 +1183,6 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name, digest = py_digest_by_name(hash_name); if (digest == NULL) { - PyErr_SetString(PyExc_ValueError, "unsupported hash type"); goto end; } @@ -1334,26 +1386,26 @@ _hashlib.hmac_digest as _hashlib_hmac_singleshot key: Py_buffer msg: Py_buffer - digest: str + digest: object Single-shot HMAC. [clinic start generated code]*/ static PyObject * _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, - Py_buffer *msg, const char *digest) -/*[clinic end generated code: output=15658ede5ab98185 input=019dffc571909a46]*/ + Py_buffer *msg, PyObject *digest) +/*[clinic end generated code: output=82f19965d12706ac input=0a0790cc3db45c2e]*/ { unsigned char md[EVP_MAX_MD_SIZE] = {0}; unsigned int md_len = 0; unsigned char *result; const EVP_MD *evp; - evp = py_digest_by_name(digest); + evp = py_digest_by_digestmod(module, digest); if (evp == NULL) { - PyErr_SetString(PyExc_ValueError, "unsupported hash type"); return NULL; } + if (key->len > INT_MAX) { PyErr_SetString(PyExc_OverflowError, "key is too long."); @@ -1391,15 +1443,15 @@ _hashlib.hmac_new key: Py_buffer msg as msg_obj: object(c_default="NULL") = b'' - digestmod: str(c_default="NULL") = None + digestmod: object(c_default="NULL") = None Return a new hmac object. [clinic start generated code]*/ static PyObject * _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, - const char *digestmod) -/*[clinic end generated code: output=9a35673be0cbea1b input=a0878868eb190134]*/ + PyObject *digestmod) +/*[clinic end generated code: output=c20d9e4d9ed6d219 input=5f4071dcc7f34362]*/ { PyTypeObject *type = get_hashlib_state(module)->HMACtype; const EVP_MD *digest; @@ -1413,15 +1465,14 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, return NULL; } - if ((digestmod == NULL) || !strlen(digestmod)) { + if (digestmod == NULL) { PyErr_SetString( PyExc_TypeError, "Missing required parameter 'digestmod'."); return NULL; } - digest = py_digest_by_name(digestmod); - if (!digest) { - PyErr_SetString(PyExc_ValueError, "unknown hash function"); + digest = py_digest_by_digestmod(module, digestmod); + if (digest == NULL) { return NULL; } @@ -1992,6 +2043,8 @@ hashlib_traverse(PyObject *m, visitproc visit, void *arg) #ifdef PY_OPENSSL_HAS_SHAKE Py_VISIT(state->EVPXOFtype); #endif + Py_VISIT(state->constructs); + Py_VISIT(state->unsupported_digestmod_error); return 0; } @@ -2004,6 +2057,8 @@ hashlib_clear(PyObject *m) #ifdef PY_OPENSSL_HAS_SHAKE Py_CLEAR(state->EVPXOFtype); #endif + Py_CLEAR(state->constructs); + Py_CLEAR(state->unsupported_digestmod_error); return 0; } @@ -2085,6 +2140,79 @@ hashlib_init_hmactype(PyObject *module) return 0; } +static int +hashlib_init_constructors(PyObject *module) +{ + /* Create dict from builtin openssl_hash functions to name + * {_hashlib.openssl_sha256: "sha256", ...} + */ + PyModuleDef *mdef; + PyMethodDef *fdef; + PyObject *proxy; + PyObject *func, *name_obj; + _hashlibstate *state = get_hashlib_state(module); + + mdef = PyModule_GetDef(module); + if (mdef == NULL) { + return -1; + } + + state->constructs = PyDict_New(); + if (state->constructs == NULL) { + return -1; + } + + for (fdef = mdef->m_methods; fdef->ml_name != NULL; fdef++) { + if (strncmp(fdef->ml_name, "openssl_", 8)) { + continue; + } + name_obj = PyUnicode_FromString(fdef->ml_name + 8); + if (name_obj == NULL) { + return -1; + } + func = PyObject_GetAttrString(module, fdef->ml_name); + if (func == NULL) { + Py_DECREF(name_obj); + return -1; + } + int rc = PyDict_SetItem(state->constructs, func, name_obj); + Py_DECREF(func); + Py_DECREF(name_obj); + if (rc < 0) { + return -1; + } + } + + proxy = PyDictProxy_New(state->constructs); + if (proxy == NULL) { + return -1; + } + + int rc = _PyModule_AddObjectRef(module, "_constructors", proxy); + Py_DECREF(proxy); + if (rc < 0) { + return -1; + } + return 0; +} + +static int +hashlib_exception(PyObject *module) +{ + _hashlibstate *state = get_hashlib_state(module); + state->unsupported_digestmod_error = PyErr_NewException( + "_hashlib.UnsupportedDigestmodError", PyExc_ValueError, NULL); + if (state->unsupported_digestmod_error == NULL) { + return -1; + } + if (_PyModule_AddObjectRef(module, "UnsupportedDigestmodError", + state->unsupported_digestmod_error) < 0) { + return -1; + } + return 0; +} + + static PyModuleDef_Slot hashlib_slots[] = { /* OpenSSL 1.0.2 and LibreSSL */ {Py_mod_exec, hashlib_openssl_legacy_init}, @@ -2092,6 +2220,8 @@ static PyModuleDef_Slot hashlib_slots[] = { {Py_mod_exec, hashlib_init_evpxoftype}, {Py_mod_exec, hashlib_init_hmactype}, {Py_mod_exec, hashlib_md_meth_names}, + {Py_mod_exec, hashlib_init_constructors}, + {Py_mod_exec, hashlib_exception}, {0, NULL} }; diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 68aa765e52..4466ec4330 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -1106,7 +1106,7 @@ PyDoc_STRVAR(_hashlib_hmac_singleshot__doc__, static PyObject * _hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key, - Py_buffer *msg, const char *digest); + Py_buffer *msg, PyObject *digest); static PyObject * _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1117,7 +1117,7 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar PyObject *argsbuf[3]; Py_buffer key = {NULL, NULL}; Py_buffer msg = {NULL, NULL}; - const char *digest; + PyObject *digest; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf); if (!args) { @@ -1137,19 +1137,7 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar _PyArg_BadArgument("hmac_digest", "argument 'msg'", "contiguous buffer", args[1]); goto exit; } - if (!PyUnicode_Check(args[2])) { - _PyArg_BadArgument("hmac_digest", "argument 'digest'", "str", args[2]); - goto exit; - } - Py_ssize_t digest_length; - digest = PyUnicode_AsUTF8AndSize(args[2], &digest_length); - if (digest == NULL) { - goto exit; - } - if (strlen(digest) != (size_t)digest_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + digest = args[2]; return_value = _hashlib_hmac_singleshot_impl(module, &key, &msg, digest); exit: @@ -1176,7 +1164,7 @@ PyDoc_STRVAR(_hashlib_hmac_new__doc__, static PyObject * _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj, - const char *digestmod); + PyObject *digestmod); static PyObject * _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1188,7 +1176,7 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; Py_buffer key = {NULL, NULL}; PyObject *msg_obj = NULL; - const char *digestmod = NULL; + PyObject *digestmod = NULL; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf); if (!args) { @@ -1210,19 +1198,7 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO goto skip_optional_pos; } } - if (!PyUnicode_Check(args[2])) { - _PyArg_BadArgument("hmac_new", "argument 'digestmod'", "str", args[2]); - goto exit; - } - Py_ssize_t digestmod_length; - digestmod = PyUnicode_AsUTF8AndSize(args[2], &digestmod_length); - if (digestmod == NULL) { - goto exit; - } - if (strlen(digestmod) != (size_t)digestmod_length) { - PyErr_SetString(PyExc_ValueError, "embedded null character"); - goto exit; - } + digestmod = args[2]; skip_optional_pos: return_value = _hashlib_hmac_new_impl(module, &key, msg_obj, digestmod); @@ -1442,4 +1418,4 @@ exit: #ifndef _HASHLIB_GET_FIPS_MODE_METHODDEF #define _HASHLIB_GET_FIPS_MODE_METHODDEF #endif /* !defined(_HASHLIB_GET_FIPS_MODE_METHODDEF) */ -/*[clinic end generated code: output=b6b280e46bf0b139 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/ -- 2.31.1 From 547952d3ae9d94d9e38b71d4e309c1d4c2ec963b Mon Sep 17 00:00:00 2001 From: Charalampos Stratakis Date: Thu, 12 Dec 2019 16:58:31 +0100 Subject: [PATCH 4/5] Expose blake2b and blake2s hashes from OpenSSL These aren't as powerful as Python's own implementation, but they can be used under FIPS. --- Lib/test/test_hashlib.py | 6 ++ Modules/_hashopenssl.c | 43 +++++++++++++ Modules/clinic/_hashopenssl.c.h | 106 +++++++++++++++++++++++++++++++- 3 files changed, 154 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py index 86f31a5587..f843022b48 100644 --- a/Lib/test/test_hashlib.py +++ b/Lib/test/test_hashlib.py @@ -354,6 +354,12 @@ class HashLibTestCase(unittest.TestCase): # 2 is for hashlib.name(...) and hashlib.new(name, ...) self.assertGreaterEqual(len(constructors), 2) for hash_object_constructor in constructors: + + # OpenSSL's blake2s & blake2d don't support `key` + _name = hash_object_constructor.__name__ + if 'key' in kwargs and _name.startswith('openssl_blake2'): + return + m = hash_object_constructor(data, **kwargs) computed = m.hexdigest() if not shake else m.hexdigest(length) self.assertEqual( diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c index fd3b3c0719..065c6c0612 100644 --- a/Modules/_hashopenssl.c +++ b/Modules/_hashopenssl.c @@ -294,6 +294,12 @@ py_digest_by_name(const char *name) else if (!strcmp(name, "blake2b512")) { digest = EVP_blake2b512(); } + else if (!strcmp(name, "blake2s")) { + digest = EVP_blake2s256(); + } + else if (!strcmp(name, "blake2b")) { + digest = EVP_blake2b512(); + } #endif } @@ -1038,6 +1044,41 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj, } +/*[clinic input] +_hashlib.openssl_blake2b + string as data_obj: object(py_default="b''") = NULL + * + usedforsecurity: bool = True +Returns a blake2b hash object; optionally initialized with a string +[clinic start generated code]*/ + +static PyObject * +_hashlib_openssl_blake2b_impl(PyObject *module, PyObject *data_obj, + int usedforsecurity) +/*[clinic end generated code: output=7a838b1643cde13e input=4ad7fd54268f3689]*/ + +{ + return EVP_fast_new(module, data_obj, EVP_blake2b512(), usedforsecurity); +} + +/*[clinic input] +_hashlib.openssl_blake2s + string as data_obj: object(py_default="b''") = NULL + * + usedforsecurity: bool = True +Returns a blake2s hash object; optionally initialized with a string +[clinic start generated code]*/ + +static PyObject * +_hashlib_openssl_blake2s_impl(PyObject *module, PyObject *data_obj, + int usedforsecurity) +/*[clinic end generated code: output=4eda6b40757471da input=1ed39481ffa4e26a]*/ + +{ + return EVP_fast_new(module, data_obj, EVP_blake2s256(), usedforsecurity); +} + + #ifdef PY_OPENSSL_HAS_SHA3 /*[clinic input] @@ -2022,6 +2063,8 @@ static struct PyMethodDef EVP_functions[] = { _HASHLIB_OPENSSL_SHA256_METHODDEF _HASHLIB_OPENSSL_SHA384_METHODDEF _HASHLIB_OPENSSL_SHA512_METHODDEF + _HASHLIB_OPENSSL_BLAKE2B_METHODDEF + _HASHLIB_OPENSSL_BLAKE2S_METHODDEF _HASHLIB_OPENSSL_SHA3_224_METHODDEF _HASHLIB_OPENSSL_SHA3_256_METHODDEF _HASHLIB_OPENSSL_SHA3_384_METHODDEF diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h index 4466ec4330..54c22b25d3 100644 --- a/Modules/clinic/_hashopenssl.c.h +++ b/Modules/clinic/_hashopenssl.c.h @@ -540,6 +540,110 @@ exit: return return_value; } +PyDoc_STRVAR(_hashlib_openssl_blake2b__doc__, +"openssl_blake2b($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Returns a blake2b hash object; optionally initialized with a string"); + +#define _HASHLIB_OPENSSL_BLAKE2B_METHODDEF \ + {"openssl_blake2b", (PyCFunction)(void(*)(void))_hashlib_openssl_blake2b, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_blake2b__doc__}, + +static PyObject * +_hashlib_openssl_blake2b_impl(PyObject *module, PyObject *data_obj, + int usedforsecurity); + +static PyObject * +_hashlib_openssl_blake2b(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "openssl_blake2b", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *data_obj = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + data_obj = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _hashlib_openssl_blake2b_impl(module, data_obj, usedforsecurity); + +exit: + return return_value; +} + +PyDoc_STRVAR(_hashlib_openssl_blake2s__doc__, +"openssl_blake2s($module, /, string=b\'\', *, usedforsecurity=True)\n" +"--\n" +"\n" +"Returns a blake2s hash object; optionally initialized with a string"); + +#define _HASHLIB_OPENSSL_BLAKE2S_METHODDEF \ + {"openssl_blake2s", (PyCFunction)(void(*)(void))_hashlib_openssl_blake2s, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_blake2s__doc__}, + +static PyObject * +_hashlib_openssl_blake2s_impl(PyObject *module, PyObject *data_obj, + int usedforsecurity); + +static PyObject * +_hashlib_openssl_blake2s(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + static const char * const _keywords[] = {"string", "usedforsecurity", NULL}; + static _PyArg_Parser _parser = {NULL, _keywords, "openssl_blake2s", 0}; + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0; + PyObject *data_obj = NULL; + int usedforsecurity = 1; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf); + if (!args) { + goto exit; + } + if (!noptargs) { + goto skip_optional_pos; + } + if (args[0]) { + data_obj = args[0]; + if (!--noptargs) { + goto skip_optional_pos; + } + } +skip_optional_pos: + if (!noptargs) { + goto skip_optional_kwonly; + } + usedforsecurity = PyObject_IsTrue(args[1]); + if (usedforsecurity < 0) { + goto exit; + } +skip_optional_kwonly: + return_value = _hashlib_openssl_blake2s_impl(module, data_obj, usedforsecurity); + +exit: + return return_value; +} + #if defined(PY_OPENSSL_HAS_SHA3) PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__, @@ -1418,4 +1522,4 @@ exit: #ifndef _HASHLIB_GET_FIPS_MODE_METHODDEF #define _HASHLIB_GET_FIPS_MODE_METHODDEF #endif /* !defined(_HASHLIB_GET_FIPS_MODE_METHODDEF) */ -/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/ +/*[clinic end generated code: output=fab05055e982f112 input=a9049054013a1b77]*/ -- 2.31.1 From 4d2a433760548bb9813139844e574c06ea3fdb7a Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Thu, 1 Aug 2019 17:57:05 +0200 Subject: [PATCH 5/5] Use a stronger hash in multiprocessing handshake Adapted from patch by David Malcolm, https://bugs.python.org/issue17258 --- Lib/multiprocessing/connection.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index 510e4b5aba..b68f2fb837 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -42,6 +42,10 @@ BUFSIZE = 8192 # A very generous timeout when it comes to local connections... CONNECTION_TIMEOUT = 20. +# The hmac module implicitly defaults to using MD5. +# Support using a stronger algorithm for the challenge/response code: +HMAC_DIGEST_NAME='sha256' + _mmap_counter = itertools.count() default_family = 'AF_INET' @@ -741,7 +745,7 @@ def deliver_challenge(connection, authkey): "Authkey must be bytes, not {0!s}".format(type(authkey))) message = os.urandom(MESSAGE_LENGTH) connection.send_bytes(CHALLENGE + message) - digest = hmac.new(authkey, message, 'md5').digest() + digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() response = connection.recv_bytes(256) # reject large message if response == digest: connection.send_bytes(WELCOME) @@ -757,7 +761,7 @@ def answer_challenge(connection, authkey): message = connection.recv_bytes(256) # reject large message assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message message = message[len(CHALLENGE):] - digest = hmac.new(authkey, message, 'md5').digest() + digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest() connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message if response != WELCOME: -- 2.31.1