BN_mod_inverse undocumented behaviour #6004
Description
BIGNUM *BN_mod_inverse(BIGNUM *r, BIGNUM *a, const BIGNUM *n, BN_CTX *ctx);
Documentation says: BN_mod_inverse() computes the inverse of a modulo n places the result in r ("(a*r)%n==1")
This is true as long n is not 1 and n is not -1.
It would be appropriate to return NULL if n is 1 or -1.
PoC:
#include <openssl/bn.h>
BIGNUM *zero = NULL;
BN_CTX *ctx = NULL;
void test_BN_mod_inverse(const BIGNUM *A, const BIGNUM *N)
{
BIGNUM* inv = BN_new();
BIGNUM* one = BN_new();
BN_set_word(one, 1);
if ( BN_cmp(A, zero) < 0 || BN_cmp(N, zero) == 0 ) {
goto end;
}
if ( BN_mod_inverse(inv, A, N, ctx) == NULL ) {
goto end;
}
if ( BN_mul(inv, inv, A, ctx) != 1 ) {
goto end;
}
if ( BN_mod(inv, inv, N, ctx) != 1 ) {
goto end;
}
if ( BN_cmp(inv, one) != 0 ) {
printf("result is %s\n", BN_bn2dec(inv));
abort();
}
end:
BN_free(inv);
BN_free(one);
}
int main(void)
{
BIGNUM *A = NULL, *N = NULL;
int ret;
if ( (ctx = BN_CTX_new()) == NULL ) {
abort();
}
zero = BN_new();
BN_zero(zero);
ret = BN_dec2bn(&A, "2");
if ( ret == 0 ) abort();
ret = BN_dec2bn(&N, "1");
if ( ret == 0 ) abort();
test_BN_mod_inverse(A, N);
}Found by OSS-Fuzz, see also https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=7752