Fix a ** 0 mod 1 = 0 for real this time.

Commit 2b0180c37fa6ffc48ee40caa831ca398b828e680 attempted to do this but
only hit one of many BN_mod_exp codepaths. Fix remaining variants and add
a test for each method.

Thanks to Hanno Boeck for reporting this issue.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Dr. Stephen Henson <steve@openssl.org>

crypto/bn/bn_exp.c

@@ -282,9 +282,14 @@ int BN_mod_exp_recp(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
     }
 
     bits = BN_num_bits(p);
-
     if (bits == 0) {
-        ret = BN_one(r);
+        /* x**0 mod 1 is still zero. */
+        if (BN_is_one(m)) {
+            ret = 1;
+            BN_zero(r);
+        } else {
+            ret = BN_one(r);
+        }
         return ret;
     }
 
@@ -418,7 +423,13 @@ int BN_mod_exp_mont(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
     }
     bits = BN_num_bits(p);
     if (bits == 0) {
-        ret = BN_one(rr);
+        /* x**0 mod 1 is still zero. */
+        if (BN_is_one(m)) {
+            ret = 1;
+            BN_zero(rr);
+        } else {
+            ret = BN_one(rr);
+        }
         return ret;
     }
 
@@ -671,7 +682,13 @@ int BN_mod_exp_mont_consttime(BIGNUM *rr, const BIGNUM *a, const BIGNUM *p,
 
     bits = BN_num_bits(p);
     if (bits == 0) {
-        ret = BN_one(rr);
+        /* x**0 mod 1 is still zero. */
+        if (BN_is_one(m)) {
+            ret = 1;
+            BN_zero(rr);
+        } else {
+            ret = BN_one(rr);
+        }
         return ret;
     }
 
@@ -1180,8 +1197,9 @@ int BN_mod_exp_mont_word(BIGNUM *rr, BN_ULONG a, const BIGNUM *p,
         if (BN_is_one(m)) {
             ret = 1;
             BN_zero(rr);
-        } else
+        } else {
             ret = BN_one(rr);
+        }
         return ret;
     }
     if (a == 0) {
@@ -1295,9 +1313,14 @@ int BN_mod_exp_simple(BIGNUM *r, const BIGNUM *a, const BIGNUM *p,
     }
 
     bits = BN_num_bits(p);
-
+   if (bits == 0) {
-    if (bits == 0) {
+        /* x**0 mod 1 is still zero. */
-        ret = BN_one(r);
+        if (BN_is_one(m)) {
+            ret = 1;
+            BN_zero(r);
+        } else {
+            ret = BN_one(r);
+        }
         return ret;
     }
 

test/exptest.c

@@ -72,6 +72,25 @@
 static const char rnd_seed[] =
     "string to make the random number generator think it has entropy";
 
+/*
+ * Test that r == 0 in test_exp_mod_zero(). Returns one on success,
+ * returns zero and prints debug output otherwise.
+ */
+static int a_is_zero_mod_one(const char *method, const BIGNUM *r,
+                             const BIGNUM *a) {
+    if (!BN_is_zero(r)) {
+        fprintf(stderr, "%s failed:\n", method);
+        fprintf(stderr, "a ** 0 mod 1 = r (should be 0)\n");
+        fprintf(stderr, "a = ");
+        BN_print_fp(stderr, a);
+        fprintf(stderr, "\nr = ");
+        BN_print_fp(stderr, r);
+        fprintf(stderr, "\n");
+        return 0;
+    }
+    return 1;
+}
+
 /*
  * test_exp_mod_zero tests that x**0 mod 1 == 0. It returns zero on success.
  */
@@ -79,8 +98,9 @@ static int test_exp_mod_zero()
 {
     BIGNUM *a = NULL, *p = NULL, *m = NULL;
     BIGNUM *r = NULL;
+    BN_ULONG one_word = 1;
     BN_CTX *ctx = BN_CTX_new();
-    int ret = 1;
+    int ret = 1, failed = 0;
 
     m = BN_new();
     if (!m)
@@ -100,22 +120,65 @@ static int test_exp_mod_zero()
     r = BN_new();
     if (!r)
         goto err;
-    BN_mod_exp(r, a, p, m, ctx);
-    BN_CTX_free(ctx);
 
-    if (BN_is_zero(r))
+    if (!BN_rand(a, 1024, 0, 0))
-        ret = 0;
+        goto err;
-    else {
+
-        printf("1**0 mod 1 = ");
+    if (!BN_mod_exp(r, a, p, m, ctx))
-        BN_print_fp(stdout, r);
+        goto err;
-        printf(", should be 0\n");
+
+    if (!a_is_zero_mod_one("BN_mod_exp", r, a))
+        failed = 1;
+
+    if (!BN_mod_exp_recp(r, a, p, m, ctx))
+        goto err;
+
+    if (!a_is_zero_mod_one("BN_mod_exp_recp", r, a))
+        failed = 1;
+
+    if (!BN_mod_exp_simple(r, a, p, m, ctx))
+        goto err;
+
+    if (!a_is_zero_mod_one("BN_mod_exp_simple", r, a))
+        failed = 1;
+
+    if (!BN_mod_exp_mont(r, a, p, m, ctx, NULL))
+        goto err;
+
+    if (!a_is_zero_mod_one("BN_mod_exp_mont", r, a))
+        failed = 1;
+
+    if (!BN_mod_exp_mont_consttime(r, a, p, m, ctx, NULL)) {
+        goto err;
     }
 
+    if (!a_is_zero_mod_one("BN_mod_exp_mont_consttime", r, a))
+        failed = 1;
+
+    /*
+     * A different codepath exists for single word multiplication
+     * in non-constant-time only.
+     */
+    if (!BN_mod_exp_mont_word(r, one_word, p, m, ctx, NULL))
+        goto err;
+
+    if (!BN_is_zero(r)) {
+        fprintf(stderr, "BN_mod_exp_mont_word failed:\n");
+        fprintf(stderr, "1 ** 0 mod 1 = r (should be 0)\n");
+        fprintf(stderr, "r = ");
+        BN_print_fp(stderr, r);
+        fprintf(stderr, "\n");
+        return 0;
+    }
+
+    ret = failed;
+
  err:
     BN_free(r);
     BN_free(a);
     BN_free(p);
     BN_free(m);
+    BN_CTX_free(ctx);
 
     return ret;
 }