From 520949f17a91c531ea0c8b3856ffcf3c7ac8a3b2 Mon Sep 17 00:00:00 2001 From: Eike Rathke Date: Thu, 3 Dec 2020 18:18:23 +0100 Subject: Better accuracy in rtl_math_approxValue(), tdf#138360 related Similar to commit 5abb1890ffafe5a2212076208a1c6e226f1ffa4e for rtl_math_round() use the reciprocal value in an inverse operation for negative exponents to not use the inexact 1e-1 0.10000000000000001 and so on factors. Change-Id: I05b852e06f2c31d6e0ce622b07277a81a5690833 Reviewed-on: https://gerrit.libreoffice.org/c/core/+/107172 Reviewed-by: Eike Rathke Tested-by: Jenkins --- sal/rtl/math.cxx | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'sal') diff --git a/sal/rtl/math.cxx b/sal/rtl/math.cxx index a296927635bf..46a3e925b95b 100644 --- a/sal/rtl/math.cxx +++ b/sal/rtl/math.cxx @@ -1299,16 +1299,24 @@ double SAL_CALL rtl_math_approxValue( double fValue ) SAL_THROW_EXTERN_C() int nExp = static_cast< int >(floor(log10(fValue))); nExp = 14 - nExp; - double fExpValue = getN10Exp(nExp); + double fExpValue = getN10Exp(abs(nExp)); + + if (nExp < 0) + fValue /= fExpValue; + else + fValue *= fExpValue; - fValue *= fExpValue; // If the original value was near DBL_MIN we got an overflow. Restore and // bail out. if (!std::isfinite(fValue)) return fOrigValue; fValue = std::round(fValue); - fValue /= fExpValue; + + if (nExp < 0) + fValue *= fExpValue; + else + fValue /= fExpValue; // If the original value was near DBL_MAX we got an overflow. Restore and // bail out. -- cgit