Disable FIPS mode

FIPS mode is not supported for compat-openssl11. Apply a minimal patch

that will reject explicit enabling of FIPS mode and disable automatic

activation of FIPS mode.

To avoid regressions, keep the rest of the library as it was.

diff -up openssl-1.1.1k/crypto/fips/fips.c.disable-fips openssl-1.1.1k/crypto/fips/fips.c

--- openssl-1.1.1k/crypto/fips/fips.c.disable-fips	2022-05-30 17:05:28.604500582 +0200

+++ openssl-1.1.1k/crypto/fips/fips.c	2022-05-30 17:09:46.129110042 +0200

@@ -405,13 +405,8 @@ static int verify_checksums(void)

 int FIPS_module_installed(void)

 {

-    int rv;

-    rv = access(FIPS_MODULE_PATH, F_OK);

-    if (rv < 0 && errno != ENOENT)

-        rv = 0;

-

     /* Installed == true */

-    return !rv || FIPS_module_mode();

+    return 0;

 }

 int FIPS_module_mode_set(int onoff)

diff -up openssl-1.1.1k/crypto/o_fips.c.disable-fips openssl-1.1.1k/crypto/o_fips.c

--- openssl-1.1.1k/crypto/o_fips.c.disable-fips	2022-05-30 17:05:37.411658179 +0200

+++ openssl-1.1.1k/crypto/o_fips.c	2022-05-30 17:06:25.279514707 +0200

@@ -12,24 +12,14 @@

 int FIPS_mode(void)

 {

-#ifdef OPENSSL_FIPS

-    return FIPS_module_mode();

-#else

     /* This version of the library does not support FIPS mode. */

     return 0;

-#endif

 }

 int FIPS_mode_set(int r)

 {

-#ifdef OPENSSL_FIPS

-    if (r && FIPS_module_mode()) /* can be implicitly initialized by OPENSSL_init() */

-        return 1;

-    return FIPS_module_mode_set(r);

-#else

     if (r == 0)

         return 1;

     CRYPTOerr(CRYPTO_F_FIPS_MODE_SET, CRYPTO_R_FIPS_MODE_NOT_SUPPORTED);

     return 0;

-#endif

 }

diff -up openssl-1.1.1k/crypto/o_init.c.disable-fips openssl-1.1.1k/crypto/o_init.c

--- openssl-1.1.1k/crypto/o_init.c.disable-fips	2022-05-30 17:06:58.250104676 +0200

+++ openssl-1.1.1k/crypto/o_init.c	2022-05-30 17:17:12.369135344 +0200

@@ -7,55 +7,9 @@

  * https://www.openssl.org/source/license.html

  */

-/* for secure_getenv */

-#define _GNU_SOURCE

 #include "e_os.h"

 #include <openssl/err.h>

 #ifdef OPENSSL_FIPS

-# include <sys/types.h>

-# include <sys/stat.h>

-# include <fcntl.h>

-# include <unistd.h>

-# include <errno.h>

-# include <stdlib.h>

-# include <openssl/rand.h>

-# include <openssl/fips.h>

-# include "crypto/fips.h"

-

-# define FIPS_MODE_SWITCH_FILE "/proc/sys/crypto/fips_enabled"

-

-static void init_fips_mode(void)

-{

-    char buf[2] = "0";

-    int fd;

-

-    if (secure_getenv("OPENSSL_FORCE_FIPS_MODE") != NULL) {

-        buf[0] = '1';

-    } else if ((fd = open(FIPS_MODE_SWITCH_FILE, O_RDONLY)) >= 0) {

-        while (read(fd, buf, sizeof(buf)) < 0 && errno == EINTR) ;

-        close(fd);

-    }

-

-    if (buf[0] != '1' && !FIPS_module_installed())

-        return;

-

-    /* Ensure the selftests always run */

-    /* XXX: TO SOLVE - premature initialization due to selftests */

-    FIPS_mode_set(1);

-

-    /* Failure reading the fips mode switch file means just not

-     * switching into FIPS mode. We would break too many things

-     * otherwise..

-     */

-

-    if (buf[0] != '1') {

-        /* drop down to non-FIPS mode if it is not requested */

-        FIPS_mode_set(0);

-    } else {

-        /* abort if selftest failed */

-        FIPS_selftest_check();

-    }

-}

 /*

  * Perform FIPS module power on selftest and automatic FIPS mode switch.

@@ -63,11 +17,6 @@ static void init_fips_mode(void)

 void __attribute__ ((constructor)) OPENSSL_init_library(void)

 {

-    static int done = 0;

-    if (done)

-        return;

-    done = 1;

-    init_fips_mode();

 }

 #endif