Details
-
Bug
-
Status: Closed (View Workflow)
-
Major
-
Resolution: Fixed
-
6.4.0
-
None
-
RHEL7
Description
A 512bit length RSA key is not FIPS compliant. In MaxScale 6.4.0 and earlier, inside ./server/core/ssl.cc the following code block
if (rsa_512 == NULL && (rsa_512 = create_rsa(512)) == NULL) |
{
|
MXS_ERROR("512-bit RSA key generation failed."); |
return false; |
}
|
else if (rsa_1024 == NULL && (rsa_1024 = create_rsa(1024)) == NULL) |
{
|
MXS_ERROR("1024-bit RSA key generation failed."); |
return false; |
}
|
else |
{
|
mxb_assert(rsa_512 && rsa_1024);
|
SSL_CTX_set_tmp_rsa_callback(m_ctx, tmp_rsa_callback);
|
}
|
Will always cause an error like the following if SSL is enabled in MaxScale on a RHEL/CentOS7 FIPS system w/OpenSSL 1.0.1:
2022-06-30 09:46:34 error : (server1); 512-bit RSA key generation failed.
|
2022-06-30 09:46:34 error : (server1); Unable to initialize SSL for server 'server1'
|
2022-06-30 09:46:34 error : Failed to create a new server.
|
It is easy to confirm that on a RHEL7 based system with FIPS enabled,
RSA_generate_key
|
for 512bit length will always fail:
[root@mgt2 tmp]# cat /etc/redhat-release
|
Red Hat Enterprise Linux Server release 7.9 (Maipo)
|
|
|
[root@mgt2 tmp]# fipscheck
|
usage: fipscheck [-s <hmac-suffix>] <paths-to-files>
|
fips mode is on
|
|
|
[root@mgt2 tmp]# /bin/cat << 'EOF' > test.c
|
#include <stdio.h>
|
#include <stdlib.h>
|
#include <openssl/rsa.h>
|
|
|
void main() {
|
printf("512: %d\n",RSA_generate_key(512, RSA_F4, NULL, NULL));
|
printf("1024: %d\n",RSA_generate_key(1024, RSA_F4, NULL, NULL));
|
printf("2048: %d\n",RSA_generate_key(2048, RSA_F4, NULL, NULL));
|
}
|
EOF
|
|
|
[root@mgt2 tmp]# gcc -o test test.c -lssl -lcrypto
|
|
|
[root@mgt2 tmp]# ./test
|
512: 0
|
1024: 32893344
|
2048: 32889648
|
This is not an issue on RHEL8 systems w/OpenSSL 1.1.1 and
RSA_generate_key_ex
|
[root@smtools tmp]# cat /etc/redhat-release
|
Red Hat Enterprise Linux release 8.6 (Ootpa)
|
|
|
[root@smtools tmp]# fipscheck
|
usage: fipscheck [-s <hmac-suffix>] <paths-to-files>
|
fips mode is on
|
|
|
[root@smtools tmp]# /bin/cat << 'EOF' > test.c
|
#include <stdio.h>
|
#include <stdlib.h>
|
#include <openssl/rsa.h>
|
|
|
static RSA* create_rsa(int bits)
|
{
|
BIGNUM* bn = BN_new();
|
BN_set_word(bn, RSA_F4);
|
RSA* rsa = RSA_new();
|
RSA_generate_key_ex(rsa, bits, bn, NULL);
|
BN_free(bn);
|
return rsa;
|
}
|
|
|
void main() {
|
|
|
printf("512: %d\n",create_rsa(512));
|
printf("1024: %d\n",create_rsa(1024));
|
printf("2048: %d\n",create_rsa(2048));
|
}
|
EOF
|
|
|
[root@smtools tmp]# gcc -o test test.c -lssl -lcrypto
|
|
|
[root@smtools tmp]# ./test
|
512: 33744400
|
1024: 33758944
|
2048: 33626144
|
If I am missing something someplace in the guide that lets me override the keylength value in
./server/core/ssl.cc
|
please let me know. Thanks.