Reliably generate positive or negative NaN again

...after e5c80bb69a30dfb0a3daf6061ab127d92f8142d6 "Purge out setNan from
math.cxx" had dropped the use of rtl::math::setNan and sign bit fiddling, and
relied on the implicit assumption that std::numeric_limits<double>::quiet_NaN
would produce a positive NaN (but which does not seem to be guaranteed by the
C++ standard) and on the expressed hope that multiplying such a positive NaN by
-1 would generate a negative NaN (but which does not seem to be guaranteed by
IEEE 754: while it mandates that a NaN's payload is preserved across such an
operation, the result's sign bit appears to be unspecified)

Change-Id: I12215c888a1cb8de6b3f046a836c550cb21b5a85
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/118604
Tested-by: Jenkins
Reviewed-by: Stephan Bergmann <sbergman@redhat.com>

2 files changed, 18 insertions, 2 deletions

diff --git a/sal/qa/rtl/math/test-rtl-math.cxx b/sal/qa/rtl/math/test-rtl-math.cxx
index d5bbac1a684e..f4df71e78ac3 100644
--- a/sal/qa/rtl/math/test-rtl-math.cxx
+++ b/sal/qa/rtl/math/test-rtl-math.cxx
@@ -75,6 +75,22 @@ public:
         CPPUNIT_ASSERT(std::isnan(res));
 
         res = rtl::math::stringToDouble(
+                OUString("+1.#NAN"),
+                '.', ',', &status, &end);
+        CPPUNIT_ASSERT_EQUAL(rtl_math_ConversionStatus_Ok, status);
+        CPPUNIT_ASSERT_EQUAL(sal_Int32(7), end);
+        CPPUNIT_ASSERT(std::isnan(res));
+        CPPUNIT_ASSERT(!std::signbit(res));
+
+        res = rtl::math::stringToDouble(
+                OUString("-1.#NAN"),
+                '.', ',', &status, &end);
+        CPPUNIT_ASSERT_EQUAL(rtl_math_ConversionStatus_Ok, status);
+        CPPUNIT_ASSERT_EQUAL(sal_Int32(7), end);
+        CPPUNIT_ASSERT(std::isnan(res));
+        CPPUNIT_ASSERT(std::signbit(res));
+
+        res = rtl::math::stringToDouble(
                 OUString("INF"),
                 '.', ',', &status, &end);
         CPPUNIT_ASSERT_EQUAL(rtl_math_ConversionStatus_OutOfRange, status);
diff --git a/sal/rtl/math.cxx b/sal/rtl/math.cxx
index e9b50a1b4638..5d4ed6061f97 100644
--- a/sal/rtl/math.cxx
+++ b/sal/rtl/math.cxx
@@ -1004,8 +1004,8 @@ double stringToDouble(CharT const * pBegin, CharT const * pEnd,
             {
                 // "1.#NAN", "+1.#NAN", "-1.#NAN"
                 p += 4;
-                fVal = std::numeric_limits<double>::quiet_NaN();
-                // bSign will cause negation of fVal in the end, producing a negative NAN.
+                fVal = std::copysign(std::numeric_limits<double>::quiet_NaN(), bSign ? -1.0 : 1.0);
+                bSign = false; // don't negate again
 
                 // Eat any further digits:
                 while (p != pEnd && rtl::isAsciiDigit(*p))