From 191100ff3ab38f6487d0da3b23b8544175b61d33 Mon Sep 17 00:00:00 2001 From: Eike Rathke Date: Mon, 13 Jun 2016 13:49:43 +0200 Subject: Resolves: tdf#90419 diminish precision error in Series Fill There may be more elegant ways to accomplish this, go and find one.. Change-Id: Iceaa0783db9cf3d3e1aa20f075fe7e0618a1feb6 (cherry picked from commit e89c0e4fb783bd36d5f5fea154ee8608e542dae4) Reviewed-on: https://gerrit.libreoffice.org/26220 Tested-by: Jenkins Reviewed-by: Christian Lohmaier --- sc/source/core/data/table4.cxx | 43 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 4 deletions(-) diff --git a/sc/source/core/data/table4.cxx b/sc/source/core/data/table4.cxx index 532d47c0097b..b2bb0544dac4 100644 --- a/sc/source/core/data/table4.cxx +++ b/sc/source/core/data/table4.cxx @@ -188,6 +188,41 @@ void setSuffixCell( } +namespace { +/* TODO: move this to rtl::math::approxDiff() ? Though the name is funny, the + * approx is expected to be more correct than the raw diff. */ +/** Calculate a-b trying to diminish precision errors such as for 0.11-0.12 + not return -0.009999999999999995 but -0.01 instead. + */ +double approxDiff( double a, double b ) +{ + if (a == b) + return 0.0; + if (a == 0.0) + return -b; + if (b == 0.0) + return a; + const double c = a - b; + const double aa = fabs(a); + const double ab = fabs(b); + if (aa < 1e-16 || aa > 1e+16 || ab < 1e-16 || ab > 1e+16) + // This is going nowhere, live with the result. + return c; + + const double q = aa < ab ? b / a : a / b; + const double d = (a * q - b * q) / q; + if (d == c) + // No differing error, live with the result. + return c; + + // We now have two subtractions with a similar but not equal error. Obtain + // the exponent of the error magnitude and round accordingly. + const double e = fabs(d - c); + const double fExp = floor( log10( e)); + return rtl::math::round( c, -(static_cast(fExp))-1); +} +} + void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, FillCmd& rCmd, FillDateCmd& rDateCmd, double& rInc, sal_uInt16& rMinDigits, @@ -314,7 +349,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, { double nVal1 = aFirstCell.mfValue; double nVal2 = GetValue(nCol+nAddX, nRow+nAddY); - rInc = nVal2 - nVal1; + rInc = approxDiff( nVal2, nVal1); nCol = sal::static_int_cast( nCol + nAddX ); nRow = sal::static_int_cast( nRow + nAddY ); bool bVal = true; @@ -324,7 +359,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, if (aCell.meType == CELLTYPE_VALUE) { nVal2 = aCell.mfValue; - double nDiff = nVal2 - nVal1; + double nDiff = approxDiff( nVal2, nVal1); if ( !::rtl::math::approxEqual( nDiff, rInc, 13 ) ) bVal = false; nVal1 = nVal2; @@ -389,7 +424,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, short nFlag2 = lcl_DecompValueString( aStr, nVal2, &rMinDigits ); if ( nFlag1 == nFlag2 ) { - rInc = (double)nVal2 - (double)nVal1; + rInc = approxDiff( nVal2, nVal1); nCol = sal::static_int_cast( nCol + nAddX ); nRow = sal::static_int_cast( nRow + nAddY ); bool bVal = true; @@ -403,7 +438,7 @@ void ScTable::FillAnalyse( SCCOL nCol1, SCROW nRow1, SCCOL nCol2, SCROW nRow2, nFlag2 = lcl_DecompValueString( aStr, nVal2, &rMinDigits ); if ( nFlag1 == nFlag2 ) { - double nDiff = (double)nVal2 - (double)nVal1; + double nDiff = approxDiff( nVal2, nVal1); if ( !::rtl::math::approxEqual( nDiff, rInc, 13 ) ) bVal = false; nVal1 = nVal2; -- cgit