Return error in even moduli in sp_exptmod_nct for RSA public-only builds

.github/configs/os-check-linux.json

@@ -562,6 +562,14 @@
  "configure": ["--enable-cryptonly", "--enable-asn=no", "--disable-rsa",
   "--disable-dh", "--disable-fpecc", "--with-eccminsz=160",
   "--enable-ecccustcurves=all"]},
+{"name": "cryptonly-rsapub-no-dh-public-mp", "minutes": 1.0,
+ "comment": "RSA public only with DH off, so sp_exptmod_ex hands every modulus to sp_exptmod_nct. WOLFSSL_PUBLIC_MP lets wolfcrypt test run mp_test, whose even modulus checks confirm both return MP_VAL instead of a wrong Montgomery result.",
+ "configure": ["--enable-cryptonly", "--enable-rsapub", "--disable-dh",
+  "--disable-examples", "CPPFLAGS=-DWOLFSSL_PUBLIC_MP"]},
+{"name": "cryptonly-rsapub-public-mp", "minutes": 1.0,
+ "comment": "RSA public only with DH at its default (on). sp_exptmod_ex keeps its existing route for an even modulus to the generic exponentiation, while sp_exptmod_nct returns MP_VAL. WOLFSSL_PUBLIC_MP lets wolfcrypt test run mp_test's even modulus checks in this shape.",
+ "configure": ["--enable-cryptonly", "--enable-rsapub",
+  "--disable-examples", "CPPFLAGS=-DWOLFSSL_PUBLIC_MP"]},
 {"name": "pkcs12-pbkdf-mp-api", "minutes": 1.0,
  "comment": "WC_PKCS12_PBKDF_USING_MP_API selects a second wc_PKCS12_PBKDF_ex implementation that no configure option reaches.",
  "configure": ["--enable-pkcs12", "--enable-des3", "--enable-opensslextra",

wolfcrypt/src/sp_int.c

@@ -14199,7 +14199,8 @@ static int _sp_exptmod_base_2(const sp_int* e, int digits, const sp_int* m,
  *
  * @return  MP_OKAY on success.
  * @return  MP_VAL when b, e, m or r is NULL, digits is negative, or m <= 0 or
- *          e is negative.
+ *          e is negative; or m is even in an RSA verify/public only build
+ *          without DH.
  * @return  MP_MEM when dynamic memory allocation fails.
  */
 int sp_exptmod_ex(const sp_int* b, const sp_int* e, int digits, const sp_int* m,
@@ -14379,7 +14380,8 @@ int sp_exptmod_ex(const sp_int* b, const sp_int* e, int digits, const sp_int* m,
  * @param [out] r  SP integer to hold result.
  *
  * @return  MP_OKAY on success.
- * @return  MP_VAL when b, e, m or r is NULL; or m <= 0 or e is negative.
+ * @return  MP_VAL when b, e, m or r is NULL; or m <= 0 or e is negative; or
+ *          m is even in an RSA verify/public only build without DH.
  * @return  MP_MEM when dynamic memory allocation fails.
  */
 int sp_exptmod(const sp_int* b, const sp_int* e, const sp_int* m, sp_int* r)
@@ -14839,7 +14841,9 @@ static int _sp_exptmod_nct(const sp_int* b, const sp_int* e, const sp_int* m,
  * @param [out] r  SP integer to hold result.
  *
  * @return  MP_OKAY on success.
- * @return  MP_VAL when b, e, m or r is NULL; or m <= 0 or e is negative.
+ * @return  MP_VAL when b, e, m or r is NULL; or m <= 0 or e is negative; or
+ *          m is even in an RSA verify/public only build, even where
+ *          sp_exptmod_ex accepts it (DH builds).
  * @return  MP_MEM when dynamic memory allocation fails.
  */
 int sp_exptmod_nct(const sp_int* b, const sp_int* e, const sp_int* m, sp_int* r)
@@ -14886,11 +14890,14 @@ int sp_exptmod_nct(const sp_int* b, const sp_int* e, const sp_int* m, sp_int* r)
     else if (m->used * 2 >= r->size) {
         err = MP_VAL;
     }
-#if !defined(WOLFSSL_RSA_VERIFY_ONLY) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)
     else if (mp_iseven(m)) {
+#if !defined(WOLFSSL_RSA_VERIFY_ONLY) && !defined(WOLFSSL_RSA_PUBLIC_ONLY)
         err = _sp_exptmod_ex(b, e, (int)(e->used * SP_WORD_SIZE), m, r);
-    }
+#else
+        /* Montgomery form requires an odd modulus. */
+        err = MP_VAL;
 #endif
+    }
     else {
         err = _sp_exptmod_nct(b, e, m, r);
     }

wolfcrypt/test/test.c

@@ -96192,9 +96192,11 @@ static wc_test_ret_t mp_test_exptmod(mp_int* b, mp_int* e, mp_int* m, mp_int* r,
         return WC_TEST_RET_ENC_NC;
     }
 
-    /* Even, multi-word modulus uses the generic exponentiation path for
-     * both mp_exptmod_ex and mp_exptmod_nct.  Verify b^6 mod m against
-     * repeated modular multiplication. */
+    /* Even, multi-word modulus.  Verify b^6 mod m against repeated modular
+     * multiplication.  RSA verify/public only builds reject an even modulus
+     * in mp_exptmod_nct rather than return a wrong Montgomery result.
+     * Without DH, mp_exptmod_ex hands every modulus to mp_exptmod_nct, so it
+     * rejects it too; with DH it uses the generic exponentiation. */
     if ((ret = mp_set(m, 1)) != MP_OKAY ||
             (ret = mp_mul_2d(m, 521, m)) != MP_OKAY ||   /* m = 2^521 (even) */
             (ret = mp_set(b, 7)) != MP_OKAY ||
@@ -96207,18 +96209,33 @@ static wc_test_ret_t mp_test_exptmod(mp_int* b, mp_int* e, mp_int* m, mp_int* r,
             return WC_TEST_RET_ENC_EC(ret);
         }
     }
+#if (defined(WOLFSSL_RSA_VERIFY_ONLY) || defined(WOLFSSL_RSA_PUBLIC_ONLY)) && \
+    defined(NO_DH)
+    ret = mp_exptmod_ex(b, e, (int)e->used, m, r);
+    if (ret != WC_NO_ERR_TRACE(MP_VAL)) {
+        return WC_TEST_RET_ENC_EC(ret);
+    }
+#else
     if ((ret = mp_exptmod_ex(b, e, (int)e->used, m, r)) != MP_OKAY) {
         return WC_TEST_RET_ENC_EC(ret);
     }
     if (mp_cmp(r, t) != MP_EQ) {
         return WC_TEST_RET_ENC_NC;
     }
+#endif
+#if defined(WOLFSSL_RSA_VERIFY_ONLY) || defined(WOLFSSL_RSA_PUBLIC_ONLY)
+    ret = mp_exptmod_nct(b, e, m, r);
+    if (ret != WC_NO_ERR_TRACE(MP_VAL)) {
+        return WC_TEST_RET_ENC_EC(ret);
+    }
+#else
     if ((ret = mp_exptmod_nct(b, e, m, r)) != MP_OKAY) {
         return WC_TEST_RET_ENC_EC(ret);
     }
     if (mp_cmp(r, t) != MP_EQ) {
         return WC_TEST_RET_ENC_NC;
     }
+#endif
 
 #if !defined(HAVE_FIPS) || FIPS_VERSION3_GE(7,0,0)