From c25a14ab6ce15d18338a7499e5849225aea03a7d Mon Sep 17 00:00:00 2001

From: Nikos Mavrogiannopoulos <nmav@redhat.com>

Date: Tue, 14 Oct 2014 13:57:33 +0200

Subject: [PATCH] FIPS140-2 RSA key generation changes to account for seed

 starting with null byte

---

 lib/nettle/int/dsa-fips.h           |  2 ++

 lib/nettle/int/provable-prime.c     | 25 ++++++++++++++++++-------

 lib/nettle/int/rsa-keygen-fips186.c | 33 ++++++++++++++++++++-------------

 3 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/lib/nettle/int/dsa-fips.h b/lib/nettle/int/dsa-fips.h

index 571bc0a..08fac25 100644

--- a/lib/nettle/int/dsa-fips.h

+++ b/lib/nettle/int/dsa-fips.h

@@ -115,4 +115,6 @@ hash (uint8_t digest[DIGEST_SIZE], unsigned length, void *data)

   return;

 }

+unsigned mpz_seed_sizeinbase_256_u(mpz_t s, unsigned nominal);

+

 #endif /* DSA_FIPS_H_INCLUDED */

diff --git a/lib/nettle/int/provable-prime.c b/lib/nettle/int/provable-prime.c

index 3bb46aa..e4a4325 100644

--- a/lib/nettle/int/provable-prime.c

+++ b/lib/nettle/int/provable-prime.c

@@ -992,6 +992,18 @@ static unsigned small_prime_check(unsigned x)

 	return 1;

 }

+/* The seed in FIPS186-3 is used either as an integer or blob,

+ * but when used as an integer it must not be trunacated below

+ * the "nominal" seed size. This function returns the size

+ * that way. */

+unsigned mpz_seed_sizeinbase_256_u(mpz_t s, unsigned nominal)

+{

+	unsigned ret = nettle_mpz_sizeinbase_256_u(s);

+	if (ret < nominal)

+		return nominal;

+	return ret;

+}

+

 static int st_provable_prime_small(mpz_t p,

 				   unsigned *prime_seed_length,

 				   void *prime_seed,

@@ -1018,7 +1030,7 @@ static int st_provable_prime_small(mpz_t p,

 	nettle_mpz_set_str_256_u(s, seed_length, seed);

  retry:

-	tseed_length = nettle_mpz_sizeinbase_256_u(s);

+	tseed_length = mpz_seed_sizeinbase_256_u(s, seed_length);

 	if (tseed_length > sizeof(tseed)) {

 		goto fail;

 	}

@@ -1030,7 +1042,7 @@ static int st_provable_prime_small(mpz_t p,

 	mpz_add_ui(s, s, 1);

-	tseed_length = nettle_mpz_sizeinbase_256_u(s);

+	tseed_length = mpz_seed_sizeinbase_256_u(s, seed_length);

 	if (tseed_length > sizeof(tseed))

 		goto fail;

@@ -1071,7 +1083,7 @@ static int st_provable_prime_small(mpz_t p,

 	mpz_set_ui(p, c);

 	if (prime_seed != NULL) {

-		tseed_length = nettle_mpz_sizeinbase_256_u(s);

+		tseed_length = mpz_seed_sizeinbase_256_u(s, tseed_length);

 		if (*prime_seed_length < tseed_length)

 			goto fail;

@@ -1161,7 +1173,7 @@ st_provable_prime(mpz_t p,

 			goto fail;

 		for (i = 0; i < iterations; i++) {

-			tseed_length = nettle_mpz_sizeinbase_256_u(s);

+			tseed_length = mpz_seed_sizeinbase_256_u(s, pseed_length);

 			if (tseed_length > sizeof(tseed))

 				goto fail;

 			nettle_mpz_get_str_256(tseed_length, tseed, s);

@@ -1212,9 +1224,8 @@ st_provable_prime(mpz_t p,

 	mpz_set_ui(r, 0); /* a = 0 */

 	if (iterations > 0) {

-

 		for (i = 0; i < iterations; i++) {

-			tseed_length = nettle_mpz_sizeinbase_256_u(s);

+			tseed_length = mpz_seed_sizeinbase_256_u(s, pseed_length);

 			if (tseed_length > sizeof(tseed))

 				goto fail;

@@ -1249,7 +1260,7 @@ st_provable_prime(mpz_t p,

 			mpz_set(p, c);

 			if (prime_seed != NULL) {

-				tseed_length = nettle_mpz_sizeinbase_256_u(s);

+				tseed_length = mpz_seed_sizeinbase_256_u(s, pseed_length);

 				if (*prime_seed_length < tseed_length)

 					goto fail;

diff --git a/lib/nettle/int/rsa-keygen-fips186.c b/lib/nettle/int/rsa-keygen-fips186.c

index 754842a..624aa36 100644

--- a/lib/nettle/int/rsa-keygen-fips186.c

+++ b/lib/nettle/int/rsa-keygen-fips186.c

@@ -53,7 +53,7 @@ unsigned iterations;

 unsigned storage_length = 0, i;

 uint8_t *storage = NULL;

 uint8_t pseed[MAX_PVP_SEED_SIZE+1];

-unsigned pseed_length = sizeof(pseed);

+unsigned pseed_length = sizeof(pseed), tseed_length;

 unsigned max = bits*5;

 	mpz_init(p0);

@@ -85,11 +85,13 @@ unsigned max = bits*5;

 		nettle_mpz_set_str_256_u(s, pseed_length, pseed);

 		for (i = 0; i < iterations; i++) {

-			pseed_length = nettle_mpz_sizeinbase_256_u(s);

-			nettle_mpz_get_str_256(pseed_length, pseed, s);

+			tseed_length = mpz_seed_sizeinbase_256_u(s, pseed_length);

+			if (tseed_length > sizeof(pseed))

+				goto fail;

+			nettle_mpz_get_str_256(tseed_length, pseed, s);

 			hash(&storage[(iterations - i - 1) * DIGEST_SIZE],

-			     pseed_length, pseed);

+			     tseed_length, pseed);

 			mpz_add_ui(s, s, 1);

 		}

@@ -170,11 +172,13 @@ unsigned max = bits*5;

 		mpz_set_ui(x, 0); /* a = 0 */

 		if (iterations > 0) {

 			for (i = 0; i < iterations; i++) {

-				pseed_length = nettle_mpz_sizeinbase_256_u(s);

-				nettle_mpz_get_str_256(pseed_length, pseed, s);

+				tseed_length = mpz_seed_sizeinbase_256_u(s, pseed_length);

+				if (tseed_length > sizeof(pseed))

+					goto fail;

+				nettle_mpz_get_str_256(tseed_length, pseed, s);

 				hash(&storage[(iterations - i - 1) * DIGEST_SIZE],

-				     pseed_length, pseed);

+				     tseed_length, pseed);

 				mpz_add_ui(s, s, 1);

 			}

@@ -203,16 +207,19 @@ unsigned max = bits*5;

 			mpz_powm(r1, r2, p0, p);

 			if (mpz_cmp_ui(r1, 1) == 0) {

 				if (prime_seed_length != NULL) {

-					pseed_length = nettle_mpz_sizeinbase_256_u(s);

-					nettle_mpz_get_str_256(pseed_length, pseed, s);

+					tseed_length = mpz_seed_sizeinbase_256_u(s, pseed_length);

+					if (tseed_length > sizeof(pseed))

+						goto fail;

+

+					nettle_mpz_get_str_256(tseed_length, pseed, s);

-					if (*prime_seed_length < pseed_length) {

-						*prime_seed_length = pseed_length;

+					if (*prime_seed_length < tseed_length) {

+						*prime_seed_length = tseed_length;

 						goto fail;

 					}

-					*prime_seed_length = pseed_length;

+					*prime_seed_length = tseed_length;

 					if (prime_seed != NULL)

-						memcpy(prime_seed, pseed, pseed_length);

+						memcpy(prime_seed, pseed, tseed_length);

 				}

 				ret = 1;

 				goto cleanup;

-- 

1.9.3