diff options
| author | Mike Kaganski <mike.kaganski@collabora.com> | 2019-12-25 10:23:41 +0300 |
|---|---|---|
| committer | Mike Kaganski <mike.kaganski@collabora.com> | 2020-01-08 15:57:19 +0100 |
| commit | 8f46501233c164ff91d77a7f5adf74ea16cd0165 (patch) | |
| tree | 40eaad424c349171bb5b03f9611c91fac8c6ab23 /sc | |
| parent | 3c9288edd5bbdbac239a236f96f6c5f83c28de10 (diff) | |
tdf#129606: limit precision in ScTable::FillAnalyse
... to 16th significant digit of least precise argument. This follows
the practice to only consider 16 significant digits of user-provided
values.
Change-Id: Ic44fff82396f4f383c96343f9b2f7d35ccce9070
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/85795
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
Diffstat (limited to 'sc')
| -rw-r--r-- | sc/source/core/data/table4.cxx | 7 |
1 files changed, 5 insertions, 2 deletions
diff --git a/sc/source/core/data/table4.cxx b/sc/source/core/data/table4.cxx index c79e81af397c..5aab834a1fed 100644 --- a/sc/source/core/data/table4.cxx +++ b/sc/source/core/data/table4.cxx @@ -202,8 +202,11 @@ double approxDiff( double a, double b ) // We now have two subtractions with a similar but not equal error. Obtain // the exponent of the error magnitude and round accordingly. const double e = fabs(d - c); - const double fExp = floor( log10( e)); - return rtl::math::round( c, -static_cast<int>(fExp)-1); + const int nExp = static_cast<int>(floor(log10(e))) + 1; + // tdf#129606: Limit precision to the 16th significant digit of the least precise argument. + // Cf. mnMaxGeneralPrecision in sc/source/core/data/column3.cxx. + const int nExpArg = static_cast<int>(floor(log10(std::max(aa, ab)))) - 15; + return rtl::math::round(c, -std::max(nExp, nExpArg)); } } |