From 5943fa462c4b2afe297b06aaef911efcf935c0d7 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Wed, 11 Aug 2021 16:51:03 +0200
Subject: [PATCH 1/5] Backport PyModule_AddObjectRef as _PyModule_AddObjectRef
Having PyModule_AddObjectRef available should make backporting
newer patches easier. The new API is much safer.
The backport adds an underscore so that we don't break extension
modules that define PyModule_AddObjectRef themselves on Python<=3.9
(which would be a virtuous thing to do).
---
Include/modsupport.h | 10 ++++++++++
Python/modsupport.c | 13 +++++++++++--
2 files changed, 21 insertions(+), 2 deletions(-)
diff --git a/Include/modsupport.h b/Include/modsupport.h
index 4c4aab65ba..d9fac52521 100644
--- a/Include/modsupport.h
+++ b/Include/modsupport.h
@@ -136,7 +136,17 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywords(
void _PyArg_Fini(void);
#endif /* Py_LIMITED_API */
+// Add an attribute with name 'name' and value 'obj' to the module 'mod.
+// On success, return 0 on success.
+// On error, raise an exception and return -1.
+// Backported from Python 3.10, where it's available without the underscore
+// in the name, to ease porting patches to RHEL
+PyAPI_FUNC(int) _PyModule_AddObjectRef(PyObject *mod, const char *name, PyObject *value);
+
+// Similar to PyModule_AddObjectRef() but steal a reference to 'obj'
+// (Py_DECREF(obj)) on success (if it returns 0).
PyAPI_FUNC(int) PyModule_AddObject(PyObject *, const char *, PyObject *);
+
PyAPI_FUNC(int) PyModule_AddIntConstant(PyObject *, const char *, long);
PyAPI_FUNC(int) PyModule_AddStringConstant(PyObject *, const char *, const char *);
#if !defined(Py_LIMITED_API) || Py_LIMITED_API+0 >= 0x03090000
diff --git a/Python/modsupport.c b/Python/modsupport.c
index 13482c6508..fca1083e2d 100644
--- a/Python/modsupport.c
+++ b/Python/modsupport.c
@@ -631,7 +631,7 @@ va_build_stack(PyObject **small_stack, Py_ssize_t small_stack_len,
int
-PyModule_AddObject(PyObject *m, const char *name, PyObject *o)
+_PyModule_AddObjectRef(PyObject *m, const char *name, PyObject *o)
{
PyObject *dict;
if (!PyModule_Check(m)) {
@@ -655,10 +655,19 @@ PyModule_AddObject(PyObject *m, const char *name, PyObject *o)
}
if (PyDict_SetItemString(dict, name, o))
return -1;
- Py_DECREF(o);
return 0;
}
+int
+PyModule_AddObject(PyObject *mod, const char *name, PyObject *value)
+{
+ int res = _PyModule_AddObjectRef(mod, name, value);
+ if (res == 0) {
+ Py_DECREF(value);
+ }
+ return res;
+}
+
int
PyModule_AddIntConstant(PyObject *m, const char *name, long value)
{
--
2.31.1
From d6ecb3ac9a626af2ecd782d6c7b452251713c4a6 Mon Sep 17 00:00:00 2001
From: Petr Viktorin <encukou@gmail.com>
Date: Fri, 13 Aug 2021 13:16:43 +0200
Subject: [PATCH 2/5] _hashopenssl: Uncomment and use initialization function
list
This simplifies backporting of future changes.
We use this change instead of Python 3.10's:
bpo-1635741: Port _hashlib to multiphase initialization (GH-23358)
---
Modules/_hashopenssl.c | 26 +++++---------------------
1 file changed, 5 insertions(+), 21 deletions(-)
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index ff3a1aef5e..db661dc5f7 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -2085,7 +2085,6 @@ hashlib_init_hmactype(PyObject *module)
return 0;
}
-#if 0
static PyModuleDef_Slot hashlib_slots[] = {
/* OpenSSL 1.0.2 and LibreSSL */
{Py_mod_exec, hashlib_openssl_legacy_init},
@@ -2095,7 +2094,6 @@ static PyModuleDef_Slot hashlib_slots[] = {
{Py_mod_exec, hashlib_md_meth_names},
{0, NULL}
};
-#endif
static struct PyModuleDef _hashlibmodule = {
PyModuleDef_HEAD_INIT,
@@ -2123,25 +2121,11 @@ PyInit__hashlib(void)
return NULL;
}
- if (hashlib_openssl_legacy_init(m) < 0) {
- Py_DECREF(m);
- return NULL;
- }
- if (hashlib_init_evptype(m) < 0) {
- Py_DECREF(m);
- return NULL;
- }
- if (hashlib_init_evpxoftype(m) < 0) {
- Py_DECREF(m);
- return NULL;
- }
- if (hashlib_init_hmactype(m) < 0) {
- Py_DECREF(m);
- return NULL;
- }
- if (hashlib_md_meth_names(m) == -1) {
- Py_DECREF(m);
- return NULL;
+ for (int i=0; hashlib_slots[i].slot; i++) {
+ if (((int (*)(PyObject*))hashlib_slots[i].value)(m) < 0) {
+ Py_DECREF(m);
+ return NULL;
+ }
}
return m;
--
2.31.1
From ce54104e69b9f3e9b3f49cec2054b66e84a5cbb8 Mon Sep 17 00:00:00 2001
From: Christian Heimes <christian@python.org>
Date: Sat, 27 Mar 2021 14:55:03 +0100
Subject: [PATCH 3/5] bpo-40645: use C implementation of HMAC (GH-24920,
GH-25063, GH-26079)
This backports the feature and 2 subsequent bugfixes
from: https://bugs.python.org/issue40645
Signed-off-by: Christian Heimes <christian@python.org>
Co-authored-by: Erlend Egeberg Aasland <erlend.aasland@innova.no>
Co-Authored-By: Pablo Galindo <Pablogsal@gmail.com>
---
Lib/hashlib.py | 1 +
Lib/hmac.py | 86 ++++++----
Lib/test/test_hmac.py | 114 ++++++++-----
.../2021-03-29-11-55-06.bpo-40645.PhaT-B.rst | 2 +
.../2021-03-19-10-22-17.bpo-40645.5pXhb-.rst | 2 +
Modules/_hashopenssl.c | 158 ++++++++++++++++--
Modules/clinic/_hashopenssl.c.h | 38 +----
7 files changed, 276 insertions(+), 125 deletions(-)
create mode 100644 Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst
create mode 100644 Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst
diff --git a/Lib/hashlib.py b/Lib/hashlib.py
index 58c340d56e..ffa3be049a 100644
--- a/Lib/hashlib.py
+++ b/Lib/hashlib.py
@@ -173,6 +173,7 @@ try:
algorithms_available = algorithms_available.union(
_hashlib.openssl_md_meth_names)
except ImportError:
+ _hashlib = None
new = __py_new
__get_hash = __get_builtin_constructor
diff --git a/Lib/hmac.py b/Lib/hmac.py
index 180bc378b5..8b4f920db9 100644
--- a/Lib/hmac.py
+++ b/Lib/hmac.py
@@ -8,11 +8,12 @@ try:
import _hashlib as _hashopenssl
except ImportError:
_hashopenssl = None
- _openssl_md_meths = None
+ _functype = None
from _operator import _compare_digest as compare_digest
else:
- _openssl_md_meths = frozenset(_hashopenssl.openssl_md_meth_names)
compare_digest = _hashopenssl.compare_digest
+ _functype = type(_hashopenssl.openssl_sha256) # builtin type
+
import hashlib as _hashlib
trans_5C = bytes((x ^ 0x5C) for x in range(256))
@@ -23,7 +24,6 @@ trans_36 = bytes((x ^ 0x36) for x in range(256))
digest_size = None
-
class HMAC:
"""RFC 2104 HMAC class. Also complies with RFC 4231.
@@ -32,7 +32,7 @@ class HMAC:
blocksize = 64 # 512-bit HMAC; can be changed in subclasses.
__slots__ = (
- "_digest_cons", "_inner", "_outer", "block_size", "digest_size"
+ "_hmac", "_inner", "_outer", "block_size", "digest_size"
)
def __init__(self, key, msg=None, digestmod=''):
@@ -55,15 +55,30 @@ class HMAC:
if not digestmod:
raise TypeError("Missing required parameter 'digestmod'.")
+ if _hashopenssl and isinstance(digestmod, (str, _functype)):
+ try:
+ self._init_hmac(key, msg, digestmod)
+ except _hashopenssl.UnsupportedDigestmodError:
+ self._init_old(key, msg, digestmod)
+ else:
+ self._init_old(key, msg, digestmod)
+
+ def _init_hmac(self, key, msg, digestmod):
+ self._hmac = _hashopenssl.hmac_new(key, msg, digestmod=digestmod)
+ self.digest_size = self._hmac.digest_size
+ self.block_size = self._hmac.block_size
+
+ def _init_old(self, key, msg, digestmod):
if callable(digestmod):
- self._digest_cons = digestmod
+ digest_cons = digestmod
elif isinstance(digestmod, str):
- self._digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
+ digest_cons = lambda d=b'': _hashlib.new(digestmod, d)
else:
- self._digest_cons = lambda d=b'': digestmod.new(d)
+ digest_cons = lambda d=b'': digestmod.new(d)
- self._outer = self._digest_cons()
- self._inner = self._digest_cons()
+ self._hmac = None
+ self._outer = digest_cons()
+ self._inner = digest_cons()
self.digest_size = self._inner.digest_size
if hasattr(self._inner, 'block_size'):
@@ -79,13 +94,13 @@ class HMAC:
RuntimeWarning, 2)
blocksize = self.blocksize
+ if len(key) > blocksize:
+ key = digest_cons(key).digest()
+
# self.blocksize is the default blocksize. self.block_size is
# effective block size as well as the public API attribute.
self.block_size = blocksize
- if len(key) > blocksize:
- key = self._digest_cons(key).digest()
-
key = key.ljust(blocksize, b'\0')
self._outer.update(key.translate(trans_5C))
self._inner.update(key.translate(trans_36))
@@ -94,23 +109,15 @@ class HMAC:
@property
def name(self):
- return "hmac-" + self._inner.name
-
- @property
- def digest_cons(self):
- return self._digest_cons
-
- @property
- def inner(self):
- return self._inner
-
- @property
- def outer(self):
- return self._outer
+ if self._hmac:
+ return self._hmac.name
+ else:
+ return f"hmac-{self._inner.name}"
def update(self, msg):
"""Feed data from msg into this hashing object."""
- self._inner.update(msg)
+ inst = self._hmac or self._inner
+ inst.update(msg)
def copy(self):
"""Return a separate copy of this hashing object.
@@ -119,10 +126,14 @@ class HMAC:
"""
# Call __new__ directly to avoid the expensive __init__.
other = self.__class__.__new__(self.__class__)
- other._digest_cons = self._digest_cons
other.digest_size = self.digest_size
- other._inner = self._inner.copy()
- other._outer = self._outer.copy()
+ if self._hmac:
+ other._hmac = self._hmac.copy()
+ other._inner = other._outer = None
+ else:
+ other._hmac = None
+ other._inner = self._inner.copy()
+ other._outer = self._outer.copy()
return other
def _current(self):
@@ -130,9 +141,12 @@ class HMAC:
To be used only internally with digest() and hexdigest().
"""
- h = self._outer.copy()
- h.update(self._inner.digest())
- return h
+ if self._hmac:
+ return self._hmac
+ else:
+ h = self._outer.copy()
+ h.update(self._inner.digest())
+ return h
def digest(self):
"""Return the hash value of this hashing object.
@@ -179,9 +193,11 @@ def digest(key, msg, digest):
A hashlib constructor returning a new hash object. *OR*
A module supporting PEP 247.
"""
- if (_hashopenssl is not None and
- isinstance(digest, str) and digest in _openssl_md_meths):
- return _hashopenssl.hmac_digest(key, msg, digest)
+ if _hashopenssl is not None and isinstance(digest, (str, _functype)):
+ try:
+ return _hashopenssl.hmac_digest(key, msg, digest)
+ except _hashopenssl.UnsupportedDigestmodError:
+ pass
if callable(digest):
digest_cons = digest
diff --git a/Lib/test/test_hmac.py b/Lib/test/test_hmac.py
index 6daf22ca06..adf52adbf2 100644
--- a/Lib/test/test_hmac.py
+++ b/Lib/test/test_hmac.py
@@ -11,14 +11,21 @@ from test.support import hashlib_helper
from _operator import _compare_digest as operator_compare_digest
try:
+ import _hashlib as _hashopenssl
from _hashlib import HMAC as C_HMAC
from _hashlib import hmac_new as c_hmac_new
from _hashlib import compare_digest as openssl_compare_digest
except ImportError:
+ _hashopenssl = None
C_HMAC = None
c_hmac_new = None
openssl_compare_digest = None
+try:
+ import _sha256 as sha256_module
+except ImportError:
+ sha256_module = None
+
def ignore_warning(func):
@functools.wraps(func)
@@ -32,22 +39,27 @@ def ignore_warning(func):
class TestVectorsTestCase(unittest.TestCase):
- def asssert_hmac(
- self, key, data, digest, hashfunc, hashname, digest_size, block_size
+ def assert_hmac_internals(
+ self, h, digest, hashname, digest_size, block_size
):
- h = hmac.HMAC(key, data, digestmod=hashfunc)
self.assertEqual(h.hexdigest().upper(), digest.upper())
self.assertEqual(h.digest(), binascii.unhexlify(digest))
self.assertEqual(h.name, f"hmac-{hashname}")
self.assertEqual(h.digest_size, digest_size)
self.assertEqual(h.block_size, block_size)
+ def assert_hmac(
+ self, key, data, digest, hashfunc, hashname, digest_size, block_size
+ ):
+ h = hmac.HMAC(key, data, digestmod=hashfunc)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
+
h = hmac.HMAC(key, data, digestmod=hashname)
- self.assertEqual(h.hexdigest().upper(), digest.upper())
- self.assertEqual(h.digest(), binascii.unhexlify(digest))
- self.assertEqual(h.name, f"hmac-{hashname}")
- self.assertEqual(h.digest_size, digest_size)
- self.assertEqual(h.block_size, block_size)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
h = hmac.HMAC(key, digestmod=hashname)
h2 = h.copy()
@@ -56,11 +68,9 @@ class TestVectorsTestCase(unittest.TestCase):
self.assertEqual(h.hexdigest().upper(), digest.upper())
h = hmac.new(key, data, digestmod=hashname)
- self.assertEqual(h.hexdigest().upper(), digest.upper())
- self.assertEqual(h.digest(), binascii.unhexlify(digest))
- self.assertEqual(h.name, f"hmac-{hashname}")
- self.assertEqual(h.digest_size, digest_size)
- self.assertEqual(h.block_size, block_size)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
h = hmac.new(key, None, digestmod=hashname)
h.update(data)
@@ -81,23 +91,18 @@ class TestVectorsTestCase(unittest.TestCase):
hmac.digest(key, data, digest=hashfunc),
binascii.unhexlify(digest)
)
- with unittest.mock.patch('hmac._openssl_md_meths', {}):
- self.assertEqual(
- hmac.digest(key, data, digest=hashname),
- binascii.unhexlify(digest)
- )
- self.assertEqual(
- hmac.digest(key, data, digest=hashfunc),
- binascii.unhexlify(digest)
- )
+
+ h = hmac.HMAC.__new__(hmac.HMAC)
+ h._init_old(key, data, digestmod=hashname)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
if c_hmac_new is not None:
h = c_hmac_new(key, data, digestmod=hashname)
- self.assertEqual(h.hexdigest().upper(), digest.upper())
- self.assertEqual(h.digest(), binascii.unhexlify(digest))
- self.assertEqual(h.name, f"hmac-{hashname}")
- self.assertEqual(h.digest_size, digest_size)
- self.assertEqual(h.block_size, block_size)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
h = c_hmac_new(key, digestmod=hashname)
h2 = h.copy()
@@ -105,12 +110,24 @@ class TestVectorsTestCase(unittest.TestCase):
h.update(data)
self.assertEqual(h.hexdigest().upper(), digest.upper())
+ func = getattr(_hashopenssl, f"openssl_{hashname}")
+ h = c_hmac_new(key, data, digestmod=func)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
+
+ h = hmac.HMAC.__new__(hmac.HMAC)
+ h._init_hmac(key, data, digestmod=hashname)
+ self.assert_hmac_internals(
+ h, digest, hashname, digest_size, block_size
+ )
+
@hashlib_helper.requires_hashdigest('md5', openssl=True)
def test_md5_vectors(self):
# Test the HMAC module against test vectors from the RFC.
def md5test(key, data, digest):
- self.asssert_hmac(
+ self.assert_hmac(
key, data, digest,
hashfunc=hashlib.md5,
hashname="md5",
@@ -150,7 +167,7 @@ class TestVectorsTestCase(unittest.TestCase):
@hashlib_helper.requires_hashdigest('sha1', openssl=True)
def test_sha_vectors(self):
def shatest(key, data, digest):
- self.asssert_hmac(
+ self.assert_hmac(
key, data, digest,
hashfunc=hashlib.sha1,
hashname="sha1",
@@ -191,7 +208,7 @@ class TestVectorsTestCase(unittest.TestCase):
def hmactest(key, data, hexdigests):
digest = hexdigests[hashfunc]
- self.asssert_hmac(
+ self.assert_hmac(
key, data, digest,
hashfunc=hashfunc,
hashname=hash_name,
@@ -427,6 +444,15 @@ class ConstructorTestCase(unittest.TestCase):
):
C_HMAC()
+ @unittest.skipUnless(sha256_module is not None, 'need _sha256')
+ def test_with_sha256_module(self):
+ h = hmac.HMAC(b"key", b"hash this!", digestmod=sha256_module.sha256)
+ self.assertEqual(h.hexdigest(), self.expected)
+ self.assertEqual(h.name, "hmac-sha256")
+
+ digest = hmac.digest(b"key", b"hash this!", sha256_module.sha256)
+ self.assertEqual(digest, binascii.unhexlify(self.expected))
+
class SanityTestCase(unittest.TestCase):
@@ -447,21 +473,21 @@ class SanityTestCase(unittest.TestCase):
class CopyTestCase(unittest.TestCase):
@hashlib_helper.requires_hashdigest('sha256')
- def test_attributes(self):
+ def test_attributes_old(self):
# Testing if attributes are of same type.
- h1 = hmac.HMAC(b"key", digestmod="sha256")
+ h1 = hmac.HMAC.__new__(hmac.HMAC)
+ h1._init_old(b"key", b"msg", digestmod="sha256")
h2 = h1.copy()
- self.assertTrue(h1._digest_cons == h2._digest_cons,
- "digest constructors don't match.")
self.assertEqual(type(h1._inner), type(h2._inner),
"Types of inner don't match.")
self.assertEqual(type(h1._outer), type(h2._outer),
"Types of outer don't match.")
@hashlib_helper.requires_hashdigest('sha256')
- def test_realcopy(self):
+ def test_realcopy_old(self):
# Testing if the copy method created a real copy.
- h1 = hmac.HMAC(b"key", digestmod="sha256")
+ h1 = hmac.HMAC.__new__(hmac.HMAC)
+ h1._init_old(b"key", b"msg", digestmod="sha256")
h2 = h1.copy()
# Using id() in case somebody has overridden __eq__/__ne__.
self.assertTrue(id(h1) != id(h2), "No real copy of the HMAC instance.")
@@ -469,17 +495,15 @@ class CopyTestCase(unittest.TestCase):
"No real copy of the attribute 'inner'.")
self.assertTrue(id(h1._outer) != id(h2._outer),
"No real copy of the attribute 'outer'.")
- self.assertEqual(h1._inner, h1.inner)
- self.assertEqual(h1._outer, h1.outer)
- self.assertEqual(h1._digest_cons, h1.digest_cons)
+ self.assertIs(h1._hmac, None)
+ @unittest.skipIf(_hashopenssl is None, "test requires _hashopenssl")
@hashlib_helper.requires_hashdigest('sha256')
- def test_properties(self):
- # deprecated properties
- h1 = hmac.HMAC(b"key", digestmod="sha256")
- self.assertEqual(h1._inner, h1.inner)
- self.assertEqual(h1._outer, h1.outer)
- self.assertEqual(h1._digest_cons, h1.digest_cons)
+ def test_realcopy_hmac(self):
+ h1 = hmac.HMAC.__new__(hmac.HMAC)
+ h1._init_hmac(b"key", b"msg", digestmod="sha256")
+ h2 = h1.copy()
+ self.assertTrue(id(h1._hmac) != id(h2._hmac))
@hashlib_helper.requires_hashdigest('sha256')
def test_equality(self):
diff --git a/Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst b/Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst
new file mode 100644
index 0000000000..9ca9843947
--- /dev/null
+++ b/Misc/NEWS.d/next/Core and Builtins/2021-03-29-11-55-06.bpo-40645.PhaT-B.rst
@@ -0,0 +1,2 @@
+Fix reference leak in the :mod:`_hashopenssl` extension. Patch by Pablo
+Galindo.
diff --git a/Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst b/Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst
new file mode 100644
index 0000000000..a9ab1c0915
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2021-03-19-10-22-17.bpo-40645.5pXhb-.rst
@@ -0,0 +1,2 @@
+The :mod:`hmac` module now uses OpenSSL's HMAC implementation when digestmod
+argument is a hash name or builtin hash function.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index db661dc5f7..fd3b3c0719 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -92,6 +92,8 @@ typedef struct {
#ifdef PY_OPENSSL_HAS_SHAKE
PyTypeObject *EVPXOFtype;
#endif
+ PyObject *constructs;
+ PyObject *unsupported_digestmod_error;
} _hashlibstate;
static inline _hashlibstate*
@@ -295,9 +297,56 @@ py_digest_by_name(const char *name)
#endif
}
+ if (digest == NULL) {
+ PyErr_Format(PyExc_ValueError, "unsupported hash type %s", name);
+ return NULL;
+ }
+
return digest;
}
+/* Get digest EVP from object
+ *
+ * * string
+ * * _hashopenssl builtin function
+ *
+ * on error returns NULL with exception set.
+ */
+static const EVP_MD*
+py_digest_by_digestmod(PyObject *module, PyObject *digestmod) {
+ const EVP_MD* evp;
+ PyObject *name_obj = NULL;
+ const char *name;
+
+ if (PyUnicode_Check(digestmod)) {
+ name_obj = digestmod;
+ } else {
+ _hashlibstate *state = get_hashlib_state(module);
+ // borrowed ref
+ name_obj = PyDict_GetItem(state->constructs, digestmod);
+ }
+ if (name_obj == NULL) {
+ _hashlibstate *state = get_hashlib_state(module);
+ PyErr_Clear();
+ PyErr_Format(
+ state->unsupported_digestmod_error,
+ "Unsupported digestmod %R", digestmod);
+ return NULL;
+ }
+
+ name = PyUnicode_AsUTF8(name_obj);
+ if (name == NULL) {
+ return NULL;
+ }
+
+ evp = py_digest_by_name(name);
+ if (evp == NULL) {
+ return NULL;
+ }
+
+ return evp;
+}
+
static EVPobject *
newEVPobject(PyTypeObject *type)
{
@@ -822,7 +871,7 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj,
/*[clinic end generated code: output=ddd5053f92dffe90 input=c24554d0337be1b0]*/
{
Py_buffer view = { 0 };
- PyObject *ret_obj;
+ PyObject *ret_obj = NULL;
char *name;
const EVP_MD *digest = NULL;
@@ -835,11 +884,15 @@ EVP_new_impl(PyObject *module, PyObject *name_obj, PyObject *data_obj,
GET_BUFFER_VIEW_OR_ERROUT(data_obj, &view);
digest = py_digest_by_name(name);
+ if (digest == NULL) {
+ goto exit;
+ }
ret_obj = EVPnew(module, digest,
(unsigned char*)view.buf, view.len,
usedforsecurity);
+exit:
if (data_obj)
PyBuffer_Release(&view);
return ret_obj;
@@ -1130,7 +1183,6 @@ pbkdf2_hmac_impl(PyObject *module, const char *hash_name,
digest = py_digest_by_name(hash_name);
if (digest == NULL) {
- PyErr_SetString(PyExc_ValueError, "unsupported hash type");
goto end;
}
@@ -1334,26 +1386,26 @@ _hashlib.hmac_digest as _hashlib_hmac_singleshot
key: Py_buffer
msg: Py_buffer
- digest: str
+ digest: object
Single-shot HMAC.
[clinic start generated code]*/
static PyObject *
_hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key,
- Py_buffer *msg, const char *digest)
-/*[clinic end generated code: output=15658ede5ab98185 input=019dffc571909a46]*/
+ Py_buffer *msg, PyObject *digest)
+/*[clinic end generated code: output=82f19965d12706ac input=0a0790cc3db45c2e]*/
{
unsigned char md[EVP_MAX_MD_SIZE] = {0};
unsigned int md_len = 0;
unsigned char *result;
const EVP_MD *evp;
- evp = py_digest_by_name(digest);
+ evp = py_digest_by_digestmod(module, digest);
if (evp == NULL) {
- PyErr_SetString(PyExc_ValueError, "unsupported hash type");
return NULL;
}
+
if (key->len > INT_MAX) {
PyErr_SetString(PyExc_OverflowError,
"key is too long.");
@@ -1391,15 +1443,15 @@ _hashlib.hmac_new
key: Py_buffer
msg as msg_obj: object(c_default="NULL") = b''
- digestmod: str(c_default="NULL") = None
+ digestmod: object(c_default="NULL") = None
Return a new hmac object.
[clinic start generated code]*/
static PyObject *
_hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
- const char *digestmod)
-/*[clinic end generated code: output=9a35673be0cbea1b input=a0878868eb190134]*/
+ PyObject *digestmod)
+/*[clinic end generated code: output=c20d9e4d9ed6d219 input=5f4071dcc7f34362]*/
{
PyTypeObject *type = get_hashlib_state(module)->HMACtype;
const EVP_MD *digest;
@@ -1413,15 +1465,14 @@ _hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
return NULL;
}
- if ((digestmod == NULL) || !strlen(digestmod)) {
+ if (digestmod == NULL) {
PyErr_SetString(
PyExc_TypeError, "Missing required parameter 'digestmod'.");
return NULL;
}
- digest = py_digest_by_name(digestmod);
- if (!digest) {
- PyErr_SetString(PyExc_ValueError, "unknown hash function");
+ digest = py_digest_by_digestmod(module, digestmod);
+ if (digest == NULL) {
return NULL;
}
@@ -1992,6 +2043,8 @@ hashlib_traverse(PyObject *m, visitproc visit, void *arg)
#ifdef PY_OPENSSL_HAS_SHAKE
Py_VISIT(state->EVPXOFtype);
#endif
+ Py_VISIT(state->constructs);
+ Py_VISIT(state->unsupported_digestmod_error);
return 0;
}
@@ -2004,6 +2057,8 @@ hashlib_clear(PyObject *m)
#ifdef PY_OPENSSL_HAS_SHAKE
Py_CLEAR(state->EVPXOFtype);
#endif
+ Py_CLEAR(state->constructs);
+ Py_CLEAR(state->unsupported_digestmod_error);
return 0;
}
@@ -2085,6 +2140,79 @@ hashlib_init_hmactype(PyObject *module)
return 0;
}
+static int
+hashlib_init_constructors(PyObject *module)
+{
+ /* Create dict from builtin openssl_hash functions to name
+ * {_hashlib.openssl_sha256: "sha256", ...}
+ */
+ PyModuleDef *mdef;
+ PyMethodDef *fdef;
+ PyObject *proxy;
+ PyObject *func, *name_obj;
+ _hashlibstate *state = get_hashlib_state(module);
+
+ mdef = PyModule_GetDef(module);
+ if (mdef == NULL) {
+ return -1;
+ }
+
+ state->constructs = PyDict_New();
+ if (state->constructs == NULL) {
+ return -1;
+ }
+
+ for (fdef = mdef->m_methods; fdef->ml_name != NULL; fdef++) {
+ if (strncmp(fdef->ml_name, "openssl_", 8)) {
+ continue;
+ }
+ name_obj = PyUnicode_FromString(fdef->ml_name + 8);
+ if (name_obj == NULL) {
+ return -1;
+ }
+ func = PyObject_GetAttrString(module, fdef->ml_name);
+ if (func == NULL) {
+ Py_DECREF(name_obj);
+ return -1;
+ }
+ int rc = PyDict_SetItem(state->constructs, func, name_obj);
+ Py_DECREF(func);
+ Py_DECREF(name_obj);
+ if (rc < 0) {
+ return -1;
+ }
+ }
+
+ proxy = PyDictProxy_New(state->constructs);
+ if (proxy == NULL) {
+ return -1;
+ }
+
+ int rc = _PyModule_AddObjectRef(module, "_constructors", proxy);
+ Py_DECREF(proxy);
+ if (rc < 0) {
+ return -1;
+ }
+ return 0;
+}
+
+static int
+hashlib_exception(PyObject *module)
+{
+ _hashlibstate *state = get_hashlib_state(module);
+ state->unsupported_digestmod_error = PyErr_NewException(
+ "_hashlib.UnsupportedDigestmodError", PyExc_ValueError, NULL);
+ if (state->unsupported_digestmod_error == NULL) {
+ return -1;
+ }
+ if (_PyModule_AddObjectRef(module, "UnsupportedDigestmodError",
+ state->unsupported_digestmod_error) < 0) {
+ return -1;
+ }
+ return 0;
+}
+
+
static PyModuleDef_Slot hashlib_slots[] = {
/* OpenSSL 1.0.2 and LibreSSL */
{Py_mod_exec, hashlib_openssl_legacy_init},
@@ -2092,6 +2220,8 @@ static PyModuleDef_Slot hashlib_slots[] = {
{Py_mod_exec, hashlib_init_evpxoftype},
{Py_mod_exec, hashlib_init_hmactype},
{Py_mod_exec, hashlib_md_meth_names},
+ {Py_mod_exec, hashlib_init_constructors},
+ {Py_mod_exec, hashlib_exception},
{0, NULL}
};
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 68aa765e52..4466ec4330 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -1106,7 +1106,7 @@ PyDoc_STRVAR(_hashlib_hmac_singleshot__doc__,
static PyObject *
_hashlib_hmac_singleshot_impl(PyObject *module, Py_buffer *key,
- Py_buffer *msg, const char *digest);
+ Py_buffer *msg, PyObject *digest);
static PyObject *
_hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
@@ -1117,7 +1117,7 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar
PyObject *argsbuf[3];
Py_buffer key = {NULL, NULL};
Py_buffer msg = {NULL, NULL};
- const char *digest;
+ PyObject *digest;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 3, 3, 0, argsbuf);
if (!args) {
@@ -1137,19 +1137,7 @@ _hashlib_hmac_singleshot(PyObject *module, PyObject *const *args, Py_ssize_t nar
_PyArg_BadArgument("hmac_digest", "argument 'msg'", "contiguous buffer", args[1]);
goto exit;
}
- if (!PyUnicode_Check(args[2])) {
- _PyArg_BadArgument("hmac_digest", "argument 'digest'", "str", args[2]);
- goto exit;
- }
- Py_ssize_t digest_length;
- digest = PyUnicode_AsUTF8AndSize(args[2], &digest_length);
- if (digest == NULL) {
- goto exit;
- }
- if (strlen(digest) != (size_t)digest_length) {
- PyErr_SetString(PyExc_ValueError, "embedded null character");
- goto exit;
- }
+ digest = args[2];
return_value = _hashlib_hmac_singleshot_impl(module, &key, &msg, digest);
exit:
@@ -1176,7 +1164,7 @@ PyDoc_STRVAR(_hashlib_hmac_new__doc__,
static PyObject *
_hashlib_hmac_new_impl(PyObject *module, Py_buffer *key, PyObject *msg_obj,
- const char *digestmod);
+ PyObject *digestmod);
static PyObject *
_hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
@@ -1188,7 +1176,7 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO
Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1;
Py_buffer key = {NULL, NULL};
PyObject *msg_obj = NULL;
- const char *digestmod = NULL;
+ PyObject *digestmod = NULL;
args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 3, 0, argsbuf);
if (!args) {
@@ -1210,19 +1198,7 @@ _hashlib_hmac_new(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO
goto skip_optional_pos;
}
}
- if (!PyUnicode_Check(args[2])) {
- _PyArg_BadArgument("hmac_new", "argument 'digestmod'", "str", args[2]);
- goto exit;
- }
- Py_ssize_t digestmod_length;
- digestmod = PyUnicode_AsUTF8AndSize(args[2], &digestmod_length);
- if (digestmod == NULL) {
- goto exit;
- }
- if (strlen(digestmod) != (size_t)digestmod_length) {
- PyErr_SetString(PyExc_ValueError, "embedded null character");
- goto exit;
- }
+ digestmod = args[2];
skip_optional_pos:
return_value = _hashlib_hmac_new_impl(module, &key, msg_obj, digestmod);
@@ -1442,4 +1418,4 @@ exit:
#ifndef _HASHLIB_GET_FIPS_MODE_METHODDEF
#define _HASHLIB_GET_FIPS_MODE_METHODDEF
#endif /* !defined(_HASHLIB_GET_FIPS_MODE_METHODDEF) */
-/*[clinic end generated code: output=b6b280e46bf0b139 input=a9049054013a1b77]*/
+/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/
--
2.31.1
From 547952d3ae9d94d9e38b71d4e309c1d4c2ec963b Mon Sep 17 00:00:00 2001
From: Charalampos Stratakis <cstratak@redhat.com>
Date: Thu, 12 Dec 2019 16:58:31 +0100
Subject: [PATCH 4/5] Expose blake2b and blake2s hashes from OpenSSL
These aren't as powerful as Python's own implementation, but they can be
used under FIPS.
---
Lib/test/test_hashlib.py | 6 ++
Modules/_hashopenssl.c | 43 +++++++++++++
Modules/clinic/_hashopenssl.c.h | 106 +++++++++++++++++++++++++++++++-
3 files changed, 154 insertions(+), 1 deletion(-)
diff --git a/Lib/test/test_hashlib.py b/Lib/test/test_hashlib.py
index 86f31a5587..f843022b48 100644
--- a/Lib/test/test_hashlib.py
+++ b/Lib/test/test_hashlib.py
@@ -354,6 +354,12 @@ class HashLibTestCase(unittest.TestCase):
# 2 is for hashlib.name(...) and hashlib.new(name, ...)
self.assertGreaterEqual(len(constructors), 2)
for hash_object_constructor in constructors:
+
+ # OpenSSL's blake2s & blake2d don't support `key`
+ _name = hash_object_constructor.__name__
+ if 'key' in kwargs and _name.startswith('openssl_blake2'):
+ return
+
m = hash_object_constructor(data, **kwargs)
computed = m.hexdigest() if not shake else m.hexdigest(length)
self.assertEqual(
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index fd3b3c0719..065c6c0612 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -294,6 +294,12 @@ py_digest_by_name(const char *name)
else if (!strcmp(name, "blake2b512")) {
digest = EVP_blake2b512();
}
+ else if (!strcmp(name, "blake2s")) {
+ digest = EVP_blake2s256();
+ }
+ else if (!strcmp(name, "blake2b")) {
+ digest = EVP_blake2b512();
+ }
#endif
}
@@ -1038,6 +1044,41 @@ _hashlib_openssl_sha512_impl(PyObject *module, PyObject *data_obj,
}
+/*[clinic input]
+_hashlib.openssl_blake2b
+ string as data_obj: object(py_default="b''") = NULL
+ *
+ usedforsecurity: bool = True
+Returns a blake2b hash object; optionally initialized with a string
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_blake2b_impl(PyObject *module, PyObject *data_obj,
+ int usedforsecurity)
+/*[clinic end generated code: output=7a838b1643cde13e input=4ad7fd54268f3689]*/
+
+{
+ return EVP_fast_new(module, data_obj, EVP_blake2b512(), usedforsecurity);
+}
+
+/*[clinic input]
+_hashlib.openssl_blake2s
+ string as data_obj: object(py_default="b''") = NULL
+ *
+ usedforsecurity: bool = True
+Returns a blake2s hash object; optionally initialized with a string
+[clinic start generated code]*/
+
+static PyObject *
+_hashlib_openssl_blake2s_impl(PyObject *module, PyObject *data_obj,
+ int usedforsecurity)
+/*[clinic end generated code: output=4eda6b40757471da input=1ed39481ffa4e26a]*/
+
+{
+ return EVP_fast_new(module, data_obj, EVP_blake2s256(), usedforsecurity);
+}
+
+
#ifdef PY_OPENSSL_HAS_SHA3
/*[clinic input]
@@ -2022,6 +2063,8 @@ static struct PyMethodDef EVP_functions[] = {
_HASHLIB_OPENSSL_SHA256_METHODDEF
_HASHLIB_OPENSSL_SHA384_METHODDEF
_HASHLIB_OPENSSL_SHA512_METHODDEF
+ _HASHLIB_OPENSSL_BLAKE2B_METHODDEF
+ _HASHLIB_OPENSSL_BLAKE2S_METHODDEF
_HASHLIB_OPENSSL_SHA3_224_METHODDEF
_HASHLIB_OPENSSL_SHA3_256_METHODDEF
_HASHLIB_OPENSSL_SHA3_384_METHODDEF
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 4466ec4330..54c22b25d3 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -540,6 +540,110 @@ exit:
return return_value;
}
+PyDoc_STRVAR(_hashlib_openssl_blake2b__doc__,
+"openssl_blake2b($module, /, string=b\'\', *, usedforsecurity=True)\n"
+"--\n"
+"\n"
+"Returns a blake2b hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_BLAKE2B_METHODDEF \
+ {"openssl_blake2b", (PyCFunction)(void(*)(void))_hashlib_openssl_blake2b, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_blake2b__doc__},
+
+static PyObject *
+_hashlib_openssl_blake2b_impl(PyObject *module, PyObject *data_obj,
+ int usedforsecurity);
+
+static PyObject *
+_hashlib_openssl_blake2b(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
+ static _PyArg_Parser _parser = {NULL, _keywords, "openssl_blake2b", 0};
+ PyObject *argsbuf[2];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
+ PyObject *data_obj = NULL;
+ int usedforsecurity = 1;
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[0]) {
+ data_obj = args[0];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+skip_optional_pos:
+ if (!noptargs) {
+ goto skip_optional_kwonly;
+ }
+ usedforsecurity = PyObject_IsTrue(args[1]);
+ if (usedforsecurity < 0) {
+ goto exit;
+ }
+skip_optional_kwonly:
+ return_value = _hashlib_openssl_blake2b_impl(module, data_obj, usedforsecurity);
+
+exit:
+ return return_value;
+}
+
+PyDoc_STRVAR(_hashlib_openssl_blake2s__doc__,
+"openssl_blake2s($module, /, string=b\'\', *, usedforsecurity=True)\n"
+"--\n"
+"\n"
+"Returns a blake2s hash object; optionally initialized with a string");
+
+#define _HASHLIB_OPENSSL_BLAKE2S_METHODDEF \
+ {"openssl_blake2s", (PyCFunction)(void(*)(void))_hashlib_openssl_blake2s, METH_FASTCALL|METH_KEYWORDS, _hashlib_openssl_blake2s__doc__},
+
+static PyObject *
+_hashlib_openssl_blake2s_impl(PyObject *module, PyObject *data_obj,
+ int usedforsecurity);
+
+static PyObject *
+_hashlib_openssl_blake2s(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames)
+{
+ PyObject *return_value = NULL;
+ static const char * const _keywords[] = {"string", "usedforsecurity", NULL};
+ static _PyArg_Parser _parser = {NULL, _keywords, "openssl_blake2s", 0};
+ PyObject *argsbuf[2];
+ Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 0;
+ PyObject *data_obj = NULL;
+ int usedforsecurity = 1;
+
+ args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 0, 1, 0, argsbuf);
+ if (!args) {
+ goto exit;
+ }
+ if (!noptargs) {
+ goto skip_optional_pos;
+ }
+ if (args[0]) {
+ data_obj = args[0];
+ if (!--noptargs) {
+ goto skip_optional_pos;
+ }
+ }
+skip_optional_pos:
+ if (!noptargs) {
+ goto skip_optional_kwonly;
+ }
+ usedforsecurity = PyObject_IsTrue(args[1]);
+ if (usedforsecurity < 0) {
+ goto exit;
+ }
+skip_optional_kwonly:
+ return_value = _hashlib_openssl_blake2s_impl(module, data_obj, usedforsecurity);
+
+exit:
+ return return_value;
+}
+
#if defined(PY_OPENSSL_HAS_SHA3)
PyDoc_STRVAR(_hashlib_openssl_sha3_224__doc__,
@@ -1418,4 +1522,4 @@ exit:
#ifndef _HASHLIB_GET_FIPS_MODE_METHODDEF
#define _HASHLIB_GET_FIPS_MODE_METHODDEF
#endif /* !defined(_HASHLIB_GET_FIPS_MODE_METHODDEF) */
-/*[clinic end generated code: output=7ff9aad0bd53e7ce input=a9049054013a1b77]*/
+/*[clinic end generated code: output=fab05055e982f112 input=a9049054013a1b77]*/
--
2.31.1
From 4d2a433760548bb9813139844e574c06ea3fdb7a Mon Sep 17 00:00:00 2001
From: Petr Viktorin <pviktori@redhat.com>
Date: Thu, 1 Aug 2019 17:57:05 +0200
Subject: [PATCH 5/5] Use a stronger hash in multiprocessing handshake
Adapted from patch by David Malcolm,
https://bugs.python.org/issue17258
---
Lib/multiprocessing/connection.py | 8 ++++++--
1 file changed, 6 insertions(+), 2 deletions(-)
diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py
index 510e4b5aba..b68f2fb837 100644
--- a/Lib/multiprocessing/connection.py
+++ b/Lib/multiprocessing/connection.py
@@ -42,6 +42,10 @@ BUFSIZE = 8192
# A very generous timeout when it comes to local connections...
CONNECTION_TIMEOUT = 20.
+# The hmac module implicitly defaults to using MD5.
+# Support using a stronger algorithm for the challenge/response code:
+HMAC_DIGEST_NAME='sha256'
+
_mmap_counter = itertools.count()
default_family = 'AF_INET'
@@ -741,7 +745,7 @@ def deliver_challenge(connection, authkey):
"Authkey must be bytes, not {0!s}".format(type(authkey)))
message = os.urandom(MESSAGE_LENGTH)
connection.send_bytes(CHALLENGE + message)
- digest = hmac.new(authkey, message, 'md5').digest()
+ digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest()
response = connection.recv_bytes(256) # reject large message
if response == digest:
connection.send_bytes(WELCOME)
@@ -757,7 +761,7 @@ def answer_challenge(connection, authkey):
message = connection.recv_bytes(256) # reject large message
assert message[:len(CHALLENGE)] == CHALLENGE, 'message = %r' % message
message = message[len(CHALLENGE):]
- digest = hmac.new(authkey, message, 'md5').digest()
+ digest = hmac.new(authkey, message, HMAC_DIGEST_NAME).digest()
connection.send_bytes(digest)
response = connection.recv_bytes(256) # reject large message
if response != WELCOME:
--
2.31.1