From 3ef5c3034e5c545f34d6929568f3f2b10ac4bdf0 Mon Sep 17 00:00:00 2001
From: Tomas Mraz <tomas@openssl.org>
Date: Mon, 28 Feb 2022 18:26:35 +0100
Subject: [PATCH] Add a negative testcase for BN_mod_sqrt

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
---
 test/bntest.c                          | 11 ++++++++++-
 test/recipes/10-test_bn_data/bnmod.txt | 12 ++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/test/bntest.c b/test/bntest.c
index 390dd80073..1cab660bca 100644
--- a/test/bntest.c
+++ b/test/bntest.c
@@ -1729,8 +1729,17 @@ static int file_modsqrt(STANZA *s)
             || !TEST_ptr(ret2 = BN_new()))
         goto err;
 
+    if (BN_is_negative(mod_sqrt)) {
+        /* A negative testcase */
+        if (!TEST_ptr_null(BN_mod_sqrt(ret, a, p, ctx)))
+            goto err;
+
+        st = 1;
+        goto err;
+    }
+
     /* There are two possible answers. */
-    if (!TEST_true(BN_mod_sqrt(ret, a, p, ctx))
+    if (!TEST_ptr(BN_mod_sqrt(ret, a, p, ctx))
             || !TEST_true(BN_sub(ret2, p, ret)))
         goto err;
 
diff --git a/test/recipes/10-test_bn_data/bnmod.txt b/test/recipes/10-test_bn_data/bnmod.txt
index 5ea4d031f2..e28cc6bfb0 100644
--- a/test/recipes/10-test_bn_data/bnmod.txt
+++ b/test/recipes/10-test_bn_data/bnmod.txt
@@ -2799,3 +2799,15 @@ P = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951f
 ModSqrt = a1d52989f12f204d3d2167d9b1e6c8a6174c0c786a979a5952383b7b8bd186
 A = 2eee37cf06228a387788188e650bc6d8a2ff402931443f69156a29155eca07dcb45f3aac238d92943c0c25c896098716baa433f25bd696a142f5a69d5d937e81
 P = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951f
+
+# Negative testcases for BN_mod_sqrt()
+
+# This one triggers an infinite loop with unfixed implementation
+# It should just fail.
+ModSqrt = -1
+A = 20a7ee
+P = 460201
+
+ModSqrt = -1
+A = 65bebdb00a96fc814ec44b81f98b59fba3c30203928fa5214c51e0a97091645280c947b005847f239758482b9bfc45b066fde340d1fe32fc9c1bf02e1b2d0ed
+P = 9df9d6cc20b8540411af4e5357ef2b0353cb1f2ab5ffc3e246b41c32f71e951f
-- 
2.27.0