tdf#107953 - Extend significant digits

According to
https://randomascii.wordpress.com/2012/03/08/float-precisionfrom-zero-to-100-digits-2/
show at least 9 / 17 significant digits for a single / double data type
in order to identify the contained value.

Change-Id: Id2833fc51ca005bc78b68d6b4cca28f2d95f5a85
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/119526
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
Reviewed-by: Andreas Heinisch <andreas.heinisch@yahoo.de>

6 files changed, 63 insertions, 52 deletions

diff --git a/basic/qa/basic_coverage/test_optional_paramters_basic.bas b/basic/qa/basic_coverage/test_optional_paramters_basic.bas
index 24da0ab20a3b..ed2cdc7720b2 100644
--- a/basic/qa/basic_coverage/test_optional_paramters_basic.bas
+++ b/basic/qa/basic_coverage/test_optional_paramters_basic.bas
@@ -21,21 +21,21 @@ Sub verify_testOptionalsBasic()
 
     ' optionals with variant datatypes (ByRef and ByVal)
     TestUtil.AssertEqual(TestOptVariantByRefByVal(), 0, "TestOptVariantByRefByVal()")
-    TestUtil.AssertEqual(TestOptVariantByRefByVal(123), 123, "TestOptVariantByRefByVal(123)")
-    TestUtil.AssertEqual(TestOptVariantByRefByVal(, 456), 456, "TestOptVariantByRefByVal(, 456)")
-    TestUtil.AssertEqual(TestOptVariantByRefByVal(123, 456), 579, "TestOptVariantByRefByVal(123, 456)")
+    TestUtil.AssertEqualApprox(TestOptVariantByRefByVal(123), 123, "TestOptVariantByRefByVal(123)")
+    TestUtil.AssertEqualApprox(TestOptVariantByRefByVal(, 456), 456, "TestOptVariantByRefByVal(, 456)")
+    TestUtil.AssertEqualApprox(TestOptVariantByRefByVal(123, 456), 579, "TestOptVariantByRefByVal(123, 456)")
 
     ' optionals with double datatypes
     TestUtil.AssertEqual(TestOptDouble(), 0, "TestOptDouble()")
-    TestUtil.AssertEqual(TestOptDouble(123.4), 123.4, "TestOptDouble(123.4)")
-    TestUtil.AssertEqual(TestOptDouble(, 567.8), 567.8, "TestOptDouble(, 567.8)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptDouble(123.4, 567.8), "0.0")), 691.2, "TestOptDouble(123.4, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDouble(123.4), 123.4, 1E-5, "TestOptDouble(123.4)")
+    TestUtil.AssertEqualApprox(TestOptDouble(, 567.8), 567.8, 1E-5, "TestOptDouble(, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDouble(123.4, 567.8), 691.2, 1E-5, "TestOptDouble(123.4, 567.8)")
 
     ' optionals with double datatypes (ByRef and ByVal)
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(), 0, "TestOptDouble()")
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(123.4), 123.4, "TestOptDouble(123.4)")
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(, 567.8), 567.8, "TestOptDoubleByRefByVal(, 567.8)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptDoubleByRefByVal(123.4, 567.8), "0.0")), 691.2, "TestOptDoubleByRefByVal(123.4, 567.8)")
+    TestUtil.AssertEqual(TestOptDoubleByRefByVal(), 0, "TestOptDoubleByRefByVal()")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(123.4), 123.4, 1E-5, "TestOptDoubleByRefByVal(123.4)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(, 567.8), 567.8, 1E-5, "TestOptDoubleByRefByVal(, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(123.4, 567.8), 691.2, 1E-5, "TestOptDoubleByRefByVal(123.4, 567.8)")
 
     ' optionals with integer datatypes
     TestUtil.AssertEqual(TestOptInteger(), 0, "TestOptInteger()")
@@ -70,14 +70,14 @@ Sub verify_testOptionalsBasic()
     cB.Add (567.8)
     TestUtil.AssertEqual(TestOptObject(), 0, "TestOptObject()")
     TestUtil.AssertEqual(TestOptObject(cA), 579, "TestOptObject(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObject(, cB), "0.0")), 691.2, "TestOptObject(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObject(cA, cB), "0.0")), 1270.2, "TestOptObject(A, B)")
+    TestUtil.AssertEqualApprox(TestOptObject(, cB), 691.2, 1E-5, "TestOptObject(, B)")
+    TestUtil.AssertEqualApprox(TestOptObject(cA, cB), 1270.2, 1E-5, "TestOptObject(A, B)")
 
     ' optionals with object datatypes (ByRef and ByVal)
     TestUtil.AssertEqual(TestOptObjectByRefByVal(), 0, "TestOptObjectByRefByVal()")
     TestUtil.AssertEqual(TestOptObjectByRefByVal(cA), 579, "TestOptObjectByRefByVal(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObjectByRefByVal(, cB), "0.0")), 691.2, "TestOptObjectByRefByVal(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObjectByRefByVal(cA, cB), "0.0")), 1270.2, "TestOptObjectByRefByVal(A, B)")
+    TestUtil.AssertEqualApprox(TestOptObjectByRefByVal(, cB), 691.2, 1E-5, "TestOptObjectByRefByVal(, B)")
+    TestUtil.AssertEqualApprox(TestOptObjectByRefByVal(cA, cB), 1270.2, 1E-5, "TestOptObjectByRefByVal(A, B)")
 
     ' optionals with array datatypes
     Dim aA(0 To 1) As Integer
@@ -88,14 +88,14 @@ Sub verify_testOptionalsBasic()
     aB(1) = 567.8
     TestUtil.AssertEqual(TestOptArray(), 0, "TestOptArray()")
     TestUtil.AssertEqual(TestOptArray(aA), 579, "TestOptArray(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArray(, aB), "0.0")), 691.2, "TestOptArray(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArray(aA, aB), "0.0")), 1270.2, "TestOptArray(A, B)")
+    TestUtil.AssertEqualApprox(TestOptArray(, aB), 691.2, 1E-5, "TestOptArray(, B)")
+    TestUtil.AssertEqualApprox(TestOptArray(aA, aB), 1270.2, 1E-5, "TestOptArray(A, B)")
 
     ' optionals with array datatypes (ByRef and ByVal)
     TestUtil.AssertEqual(TestOptArrayByRefByVal(), 0, "TestOptArrayByRefByVal()")
     TestUtil.AssertEqual(TestOptArrayByRefByVal(aA), 579, "TestOptArrayByRefByVal(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArrayByRefByVal(, aB), "0.0")), 691.2, "TestOptArrayByRefByVal(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArrayByRefByVal(aA, aB), "0.0")), 1270.2, "TestOptArrayByRefByVal(A, B)")
+    TestUtil.AssertEqualApprox(TestOptArrayByRefByVal(, aB), 691.2, 1E-5, "TestOptArrayByRefByVal(, B)")
+    TestUtil.AssertEqualApprox(TestOptArrayByRefByVal(aA, aB), 1270.2, 1E-5, "TestOptArrayByRefByVal(A, B)")
 
     Exit Sub
 errorHandler:
diff --git a/basic/qa/basic_coverage/test_optional_paramters_compatible.bas b/basic/qa/basic_coverage/test_optional_paramters_compatible.bas
index d5ea9046d4fc..00aada0698ea 100644
--- a/basic/qa/basic_coverage/test_optional_paramters_compatible.bas
+++ b/basic/qa/basic_coverage/test_optional_paramters_compatible.bas
@@ -28,16 +28,16 @@ Sub verify_testOptionalsCompatible()
     TestUtil.AssertEqual(TestOptVariantByRefByVal(123, 456), 579, "TestOptVariantByRefByVal(123, 456)")
 
     ' optionals with double datatypes
-    TestUtil.AssertEqual(TestOptDouble(), 123.4, "TestOptDouble()")
-    TestUtil.AssertEqual(TestOptDouble(123.4), 246.8, "TestOptDouble(123.4)")
-    TestUtil.AssertEqual(TestOptDouble(, 567.8), 567.8, "TestOptDouble(, 567.8)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptDouble(123.4, 567.8), "0.0")), 691.2, "TestOptDouble(123.4, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDouble(), 123.4, 1E-5, "TestOptDouble()")
+    TestUtil.AssertEqualApprox(TestOptDouble(123.4), 246.8, 1E-5, "TestOptDouble(123.4)")
+    TestUtil.AssertEqualApprox(TestOptDouble(, 567.8), 567.8, 1E-5, "TestOptDouble(, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDouble(123.4, 567.8), 691.2, 1E-5, "TestOptDouble(123.4, 567.8)")
 
     ' optionals with double datatypes (ByRef and ByVal)
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(), 123.4, "TestOptDouble()")
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(123.4), 246.8, "TestOptDouble(123.4)")
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(, 567.8), 567.8, "TestOptDoubleByRefByVal(, 567.8)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptDoubleByRefByVal(123.4, 567.8), "0.0")), 691.2, "TestOptDoubleByRefByVal(123.4, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(), 123.4, 1E-5, "TestOptDoubleByRefByVal()")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(123.4), 246.8, 1E-5, "TestOptDoubleByRefByVal(123.4)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(, 567.8), 567.8, 1E-5, "TestOptDoubleByRefByVal(, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(123.4, 567.8), 691.2, 1E-5, "TestOptDoubleByRefByVal(123.4, 567.8)")
 
     ' optionals with integer datatypes
     TestUtil.AssertEqual(TestOptInteger(), 123, "TestOptInteger()")
@@ -72,14 +72,14 @@ Sub verify_testOptionalsCompatible()
     cB.Add (567.8)
     TestUtil.AssertEqual(TestOptObject(), 0, "TestOptObject()")
     TestUtil.AssertEqual(TestOptObject(cA), 579, "TestOptObject(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObject(, cB), "0.0")), 691.2, "TestOptObject(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObject(cA, cB), "0.0")), 1270.2, "TestOptObject(A, B)")
+    TestUtil.AssertEqualApprox(TestOptObject(, cB), 691.2, 1E-5, "TestOptObject(, B)")
+    TestUtil.AssertEqualApprox(TestOptObject(cA, cB), 1270.2, 1E-5, "TestOptObject(A, B)")
 
     ' optionals with object datatypes (ByRef and ByVal)
     TestUtil.AssertEqual(TestOptObjectByRefByVal(), 0, "TestOptObjectByRefByVal()")
     TestUtil.AssertEqual(TestOptObjectByRefByVal(cA), 579, "TestOptObjectByRefByVal(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObjectByRefByVal(, cB), "0.0")), 691.2, "TestOptObjectByRefByVal(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptObjectByRefByVal(cA, cB), "0.0")), 1270.2, "TestOptObjectByRefByVal(A, B)")
+    TestUtil.AssertEqualApprox(TestOptObjectByRefByVal(, cB), 691.2, 1E-5, "TestOptObjectByRefByVal(, B)")
+    TestUtil.AssertEqualApprox(TestOptObjectByRefByVal(cA, cB), 1270.2, 1E-5, "TestOptObjectByRefByVal(A, B)")
 
     ' optionals with array datatypes
     Dim aA(0 To 1) As Integer
@@ -90,14 +90,14 @@ Sub verify_testOptionalsCompatible()
     aB(1) = 567.8
     TestUtil.AssertEqual(TestOptArray(), 0, "TestOptArray()")
     TestUtil.AssertEqual(TestOptArray(aA), 579, "TestOptArray(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArray(, aB), "0.0")), 691.2, "TestOptArray(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArray(aA, aB), "0.0")), 1270.2, "TestOptArray(A, B)")
+    TestUtil.AssertEqualApprox(TestOptArray(, aB), 691.2, 1E-5, "TestOptArray(, B)")
+    TestUtil.AssertEqualApprox(TestOptArray(aA, aB), 1270.2, 1E-5, "TestOptArray(A, B)")
 
     ' optionals with array datatypes (ByRef and ByVal)
     TestUtil.AssertEqual(TestOptArrayByRefByVal(), 0, "TestOptArrayByRefByVal()")
     TestUtil.AssertEqual(TestOptArrayByRefByVal(aA), 579, "TestOptArrayByRefByVal(A)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArrayByRefByVal(, aB), "0.0")), 691.2, "TestOptArrayByRefByVal(, B)")
-    TestUtil.AssertEqual(CDbl(Format(TestOptArrayByRefByVal(aA, aB), "0.0")), 1270.2, "TestOptArrayByRefByVal(A, B)")
+    TestUtil.AssertEqualApprox(TestOptArrayByRefByVal(, aB), 691.2, 1E-5, "TestOptArrayByRefByVal(, B)")
+    TestUtil.AssertEqualApprox(TestOptArrayByRefByVal(aA, aB), 1270.2, 1E-5, "TestOptArrayByRefByVal(A, B)")
 
     Exit Sub
 errorHandler:
diff --git a/basic/qa/cppunit/_test_asserts.bas b/basic/qa/cppunit/_test_asserts.bas
index 0ecbf6828bcd..51442a0590a6 100644
--- a/basic/qa/cppunit/_test_asserts.bas
+++ b/basic/qa/cppunit/_test_asserts.bas
@@ -52,6 +52,15 @@ Sub AssertEqual(actual As Variant, expected As Variant, testName As String)
     End If
 End Sub
 
+Sub AssertEqualApprox(actual, expected, epsilon, testName As String)
+    If Abs(expected - actual) <= epsilon Then
+        passCount = passCount + 1
+    Else
+        result = result & Chr$(10) & " Failed: " & testName & " returned " & actual & ", expected " & expected & ", epsilon " & epsilon
+        failCount = failCount + 1
+    End If
+End Sub
+
 Sub ReportErrorHandler(testName As String, aErr, sError, nErl)
     Assert False, testName, "hit error handler - " & aErr & ": " & sError & " line : " & nErl
 End Sub
diff --git a/basic/qa/vba_tests/optional_paramters.vb b/basic/qa/vba_tests/optional_paramters.vb
index e8f712bdeb35..9001ab952de9 100644
--- a/basic/qa/vba_tests/optional_paramters.vb
+++ b/basic/qa/vba_tests/optional_paramters.vb
@@ -35,16 +35,16 @@ Sub verify_testOptionalsVba()
     TestUtil.AssertEqual(TestOptVariantByRefByVal(123, 456), 579, "TestOptVariantByRefByVal(123, 456)")
 
     ' optionals with double datatypes
-    TestUtil.AssertEqual(TestOptDouble(), 123.4, "TestOptDouble()")
-    TestUtil.AssertEqual(TestOptDouble(123.4), 246.8, "TestOptDouble(123.4)")
-    TestUtil.AssertEqual(TestOptDouble(, 567.8), 567.8, "TestOptDouble(, 567.8)")
-    TestUtil.AssertEqual(Format(TestOptDouble(123.4, 567.8), "0.0"), 691.2, "TestOptDouble(123.4, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDouble(), 123.4, 1E-5, "TestOptDouble()")
+    TestUtil.AssertEqualApprox(TestOptDouble(123.4), 246.8, 1E-5, "TestOptDouble(123.4)")
+    TestUtil.AssertEqualApprox(TestOptDouble(, 567.8), 567.8, 1E-5, "TestOptDouble(, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDouble(123.4, 567.8), 691.2, 1E-5, "TestOptDouble(123.4, 567.8)")
 
     ' optionals with double datatypes (ByRef and ByVal)
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(), 123.4, "TestOptDouble()")
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(123.4), 246.8, "TestOptDouble(123.4)")
-    TestUtil.AssertEqual(TestOptDoubleByRefByVal(, 567.8), 567.8, "TestOptDoubleByRefByVal(, 567.8)")
-    TestUtil.AssertEqual(Format(TestOptDoubleByRefByVal(123.4, 567.8), "0.0"), 691.2, "TestOptDoubleByRefByVal(123.4, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(), 123.4, 1E-5, "TestOptDoubleByRefByVal()")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(123.4), 246.8, 1E-5, "TestOptDoubleByRefByVal(123.4)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(, 567.8), 567.8, 1E-5, "TestOptDoubleByRefByVal(, 567.8)")
+    TestUtil.AssertEqualApprox(TestOptDoubleByRefByVal(123.4, 567.8), 691.2, 1E-5, "TestOptDoubleByRefByVal(123.4, 567.8)")
 
     ' optionals with integer datatypes
     TestUtil.AssertEqual(TestOptInteger(), 123, "TestOptInteger()")
@@ -79,14 +79,14 @@ Sub verify_testOptionalsVba()
     cB.Add (567.8)
     TestUtil.AssertEqual(TestOptObject(), 0, "TestOptObject()")
     TestUtil.AssertEqual(TestOptObject(cA), 579, "TestOptObject(A)")
-    TestUtil.AssertEqual(Format(TestOptObject(, cB), "0.0"), 691.2, "TestOptObject(, B)")
-    TestUtil.AssertEqual(Format(TestOptObject(cA, cB), "0.0"), 1270.2, "TestOptObject(A, B)")
+    TestUtil.AssertEqualApprox(TestOptObject(, cB), 691.2, 1E-5, "TestOptObject(, B)")
+    TestUtil.AssertEqualApprox(TestOptObject(cA, cB), 1270.2, 1E-5, "TestOptObject(A, B)")
 
     ' optionals with object datatypes (ByRef and ByVal)
     TestUtil.AssertEqual(TestOptObjectByRefByVal(), 0, "TestOptObjectByRefByVal()")
     TestUtil.AssertEqual(TestOptObjectByRefByVal(cA), 579, "TestOptObjectByRefByVal(A)")
-    TestUtil.AssertEqual(Format(TestOptObjectByRefByVal(, cB), "0.0"), 691.2, "TestOptObjectByRefByVal(, B)")
-    TestUtil.AssertEqual(Format(TestOptObjectByRefByVal(cA, cB), "0.0"), 1270.2, "TestOptObjectByRefByVal(A, B)")
+    TestUtil.AssertEqualApprox(TestOptObjectByRefByVal(, cB), 691.2, 1E-5, "TestOptObjectByRefByVal(, B)")
+    TestUtil.AssertEqualApprox(TestOptObjectByRefByVal(cA, cB), 1270.2, 1E-5, "TestOptObjectByRefByVal(A, B)")
 
     ' optionals with array datatypes
     Dim aA(0 To 1) As Integer
@@ -98,15 +98,15 @@ Sub verify_testOptionalsVba()
     ' TODO - New bug report? Scanner initializes variable not as an array
     ' TestUtil.AssertEqual(TestOptArray(), 0, "TestOptArray()")
     ' TestUtil.AssertEqual(TestOptArray(aA), 579, "TestOptArray(A)")
-    ' TestUtil.AssertEqual(Format(TestOptArray(, aB), "0.0"), 691.2, "TestOptArray(, B)")
-    TestUtil.AssertEqual(Format(TestOptArray(aA, aB), "0.0"), 1270.2, "TestOptArray(A, B)")
+    ' TestUtil.AssertEqualApprox(TestOptArray(, aB), 691.2, 1E-5, "TestOptArray(, B)")
+    TestUtil.AssertEqualApprox(TestOptArray(aA, aB), 1270.2, 1E-5, "TestOptArray(A, B)")
 
     ' optionals with array datatypes (ByRef and ByVal)
     ' TODO - New bug report? Scanner initializes variable not as an array
     ' TestUtil.AssertEqual(TestOptArrayByRefByVal(), 0, "TestOptArrayByRefByVal()")
     ' TestUtil.AssertEqual(TestOptArrayByRefByVal(aA), 579, "TestOptArrayByRefByVal(A)")
-    ' TestUtil.AssertEqual(Format(TestOptArrayByRefByVal(, aB), "0.0"), 691.2, "TestOptArrayByRefByVal(, B)")
-    TestUtil.AssertEqual(Format(TestOptArrayByRefByVal(aA, aB), "0.0"), 1270.2, "TestOptArrayByRefByVal(A, B)")
+    ' TestUtil.AssertEqualApprox(TestOptArrayByRefByVal(, aB), 691.2, 1E-5, "TestOptArrayByRefByVal(, B)")
+    TestUtil.AssertEqualApprox(TestOptArrayByRefByVal(aA, aB), 1270.2, 1E-5, "TestOptArrayByRefByVal(A, B)")
 
     Exit Sub
 errorHandler:
diff --git a/basic/source/sbx/sbxdbl.cxx b/basic/source/sbx/sbxdbl.cxx
index 9010dfaa3e74..a2fed927496d 100644
--- a/basic/source/sbx/sbxdbl.cxx
+++ b/basic/source/sbx/sbxdbl.cxx
@@ -198,7 +198,8 @@ start:
         case SbxLPSTR:
             if( !p->pOUString )
                 p->pOUString = new OUString;
-            ImpCvtNum( n, 14, *p->pOUString, bCoreString );
+            // tdf#107953 - show 17 significant digits
+            ImpCvtNum( n, 17, *p->pOUString, bCoreString );
             break;
         case SbxOBJECT:
         {
diff --git a/basic/source/sbx/sbxsng.cxx b/basic/source/sbx/sbxsng.cxx
index ad4301f9ca89..a8129b8cd48d 100644
--- a/basic/source/sbx/sbxsng.cxx
+++ b/basic/source/sbx/sbxsng.cxx
@@ -222,7 +222,8 @@ start:
         {
             if( !p->pOUString )
                 p->pOUString = new OUString;
-            ImpCvtNum( static_cast<double>(n), 6, *p->pOUString );
+            // tdf#107953 - show 9 significant digits
+            ImpCvtNum( static_cast<double>(n), 9, *p->pOUString );
             break;
         }
         case SbxOBJECT: