tdf#157476 tdf#150861 - sc fix autoFilter is filtering incorrectly

caused by rounding problem with duplicated values.

We need to store the rounded values, based on the numberformat
precision, otherwise later can cause problems at removing the duplicated
values.

Regression from: f6b143a57d9bd8f5d7b29febcb4e01ee1eb2ff1d

Change-Id: I19f5248122ffca1e52adb96714df79dd9c64b23c
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/178683
Tested-by: Gabor Kelemen <gabor.kelemen.extern@allotropia.de>
Reviewed-by: Balazs Varga <balazs.varga.extern@allotropia.de>
Tested-by: Jenkins
Signed-off-by: Xisco Fauli <xiscofauli@libreoffice.org>
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/178737
(cherry picked from commit 97bab05d04393bf4f04ff4df17aac8b6843f1a85)
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/178793

3 files changed, 55 insertions, 5 deletions

diff --git a/sc/qa/uitest/autofilter2/tdf157476.py b/sc/qa/uitest/autofilter2/tdf157476.py
new file mode 100644
index 000000000000..557d9a73ab83
--- /dev/null
+++ b/sc/qa/uitest/autofilter2/tdf157476.py
@@ -0,0 +1,54 @@
+# -*- tab-width: 4; indent-tabs-mode: nil; py-indent-offset: 4 -*-
+#
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+from uitest.framework import UITestCase
+from uitest.uihelper.common import get_state_as_dict, get_url_for_data_file
+from libreoffice.uno.propertyvalue import mkPropertyValues
+from libreoffice.calc.document import is_row_hidden
+
+class tdf157476(UITestCase):
+
+    def test_tdf157476(self):
+
+        with self.ui_test.load_file(get_url_for_data_file("tdf157476.ods")) as doc:
+
+            xGridWin = self.xUITest.getTopFocusWindow().getChild("grid_window")
+            xGridWin.executeAction("LAUNCH", mkPropertyValues({"AUTOFILTER": "", "COL": "0", "ROW": "0"}))
+            xFloatWindow = self.xUITest.getFloatWindow()
+
+            xCheckListMenu = xFloatWindow.getChild("FilterDropDown")
+            xTreeList = xCheckListMenu.getChild("check_list_box")
+            self.assertEqual(6, len(xTreeList.getChildren()))
+
+            self.assertEqual("(empty)", get_state_as_dict(xTreeList.getChild('0'))['Text'])
+            self.assertEqual("0.32", get_state_as_dict(xTreeList.getChild('1'))['Text'])
+            self.assertEqual("0.65", get_state_as_dict(xTreeList.getChild('2'))['Text'])
+            self.assertEqual("1.33", get_state_as_dict(xTreeList.getChild('3'))['Text'])
+            self.assertEqual("2.00", get_state_as_dict(xTreeList.getChild('4'))['Text'])
+            self.assertEqual("3.00", get_state_as_dict(xTreeList.getChild('5'))['Text'])
+
+            xFirstEntry = xTreeList.getChild("0")
+            xFirstEntry.executeAction("CLICK", tuple())
+            xFirstEntry = xTreeList.getChild("2")
+            xFirstEntry.executeAction("CLICK", tuple())
+
+            xOkBtn = xFloatWindow.getChild("ok")
+            xOkBtn.executeAction("CLICK", tuple())
+
+            self.assertFalse(is_row_hidden(doc, 0))
+            self.assertFalse(is_row_hidden(doc, 1))
+            self.assertFalse(is_row_hidden(doc, 2))
+            self.assertFalse(is_row_hidden(doc, 3))
+            self.assertTrue(is_row_hidden(doc, 4))
+            self.assertFalse(is_row_hidden(doc, 5))
+            self.assertFalse(is_row_hidden(doc, 6))
+            self.assertTrue(is_row_hidden(doc, 7))
+            self.assertTrue(is_row_hidden(doc, 8))
+            self.assertTrue(is_row_hidden(doc, 9))
+
+# vim: set shiftwidth=4 softtabstop=4 expandtab:
diff --git a/sc/qa/uitest/data/autofilter/tdf157476.ods b/sc/qa/uitest/data/autofilter/tdf157476.ods
new file mode 100644
index 000000000000..5b0a62a3fba2
--- /dev/null
+++ b/sc/qa/uitest/data/autofilter/tdf157476.ods
diff --git a/sc/source/core/data/documen4.cxx b/sc/source/core/data/documen4.cxx
index 077af2b64ecd..d161492e6538 100644
--- a/sc/source/core/data/documen4.cxx
+++ b/sc/source/core/data/documen4.cxx
@@ -686,11 +686,7 @@ double ScDocument::RoundValueAsShown( double fVal, sal_uInt32 nFormat, const ScI
             if (nPrecision == static_cast<short>(SvNumberFormatter::UNLIMITED_PRECISION))
                 return fVal;
         }
-        double fRound = ::rtl::math::round( fVal, nPrecision );
-        if ( ::rtl::math::approxEqual( fVal, fRound ) )
-            return fVal;        // rounding might introduce some error
-        else
-            return fRound;
+        return ::rtl::math::round( fVal, nPrecision );
     }
     else
         return fVal;