Merge pull request #8931 from julek-wolfssl/MAX_ENCODED_SIG_SZ-detected

Detect correct `MAX_ENCODED_SIG_SZ` based on max support in math lib

Author: SparkiDev

wolfssl/wolfcrypt/types.h

@@ -2035,7 +2035,12 @@ enum Max_ASN {
 #elif defined(HAVE_FALCON) || defined(HAVE_DILITHIUM)
     MAX_ENCODED_SIG_SZ  = 5120,
 #elif !defined(NO_RSA)
-#ifdef WOLFSSL_HAPROXY
+#if defined(USE_FAST_MATH) && defined(FP_MAX_BITS)
+    MAX_ENCODED_SIG_SZ  = FP_MAX_BITS / 8,
+#elif (defined(WOLFSSL_SP_MATH_ALL) || defined(WOLFSSL_SP_MATH)) && \
+    defined(SP_INT_BITS)
+    MAX_ENCODED_SIG_SZ  = (SP_INT_BITS + 7) / 8,
+#elif defined(WOLFSSL_HAPROXY)
     MAX_ENCODED_SIG_SZ  = 1024,    /* Supports 8192 bit keys */
 #else
     MAX_ENCODED_SIG_SZ  = 512,     /* Supports 4096 bit keys */