Resolves: tdf#144209 Handle General containing formats in RoundValueAsShown()

Calling SvNumberformat::GetThousandDivisorPrecision() for a

"AA "General

format resulted in 3000 as that was implemented for tdf#106253
without taking into account that ImpSvNumberformatInfo::nThousand
may be abused under some conditions, which here is having
FLAG_STANDARD_IN_FORMAT = 1000 as nThousand, multiplied by 3 gives
3000. Subtracted from the 0 precision gave -3000 decimals for
which of course the new rounding produced 0 where it previously
simply ignored the decimals and returned the original value.

Change-Id: I66afaf1e2d8b2654d9f7cc8cfb66389357fb742d
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/121447
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Jenkins

3 files changed, 37 insertions, 7 deletions

diff --git a/include/svl/zformat.hxx b/include/svl/zformat.hxx
index 3fdee06d99dc..cd04f96ac8a1 100644
--- a/include/svl/zformat.hxx
+++ b/include/svl/zformat.hxx
@@ -272,10 +272,15 @@ public:
         { return NumFor[nIx].Info().nCntPre; }
 
     /** Count of hidden integer digits with thousands divisor:
-     * formats like "0," to show only thousands
+        formats like "0," to show only thousands.
+
+        Works only with SvNumFormatType::NUMBER and SvNumFormatType::CURRENCY,
+        returns 0 otherwise.
+
+        Returns SvNumberFormatter::UNLIMITED_PRECISION for formats that contain
+        the General keyword.
      */
-    sal_uInt16 GetThousandDivisorPrecision( sal_uInt16 nIx = 0 ) const
-        { return NumFor[nIx].Info().nThousand * 3; }
+    sal_uInt16 GetThousandDivisorPrecision( sal_uInt16 nIx = 0 ) const;
 
     //! Read/write access on a special sal_uInt16 component, may only be used on the
     //! standard format 0, 10000, ... and only by the number formatter!
diff --git a/sc/source/core/data/documen4.cxx b/sc/source/core/data/documen4.cxx
index 951a02700ec6..8f8bd3d43e98 100644
--- a/sc/source/core/data/documen4.cxx
+++ b/sc/source/core/data/documen4.cxx
@@ -641,8 +641,12 @@ double ScDocument::RoundValueAsShown( double fVal, sal_uInt32 nFormat, const ScI
     SvNumFormatType nType = pFormat->GetMaskedType();
     if (nType != SvNumFormatType::DATE && nType != SvNumFormatType::TIME && nType != SvNumFormatType::DATETIME )
     {
-        short nPrecision;
-        if ((nFormat % SV_COUNTRY_LANGUAGE_OFFSET) != 0)
+        // MSVC doesn't recognize all paths init nPrecision and wails about
+        // "potentially uninitialized local variable 'nPrecision' used"
+        // so init to some random sensible value preserving all decimals.
+        short nPrecision = 20;
+        bool bStdPrecision = ((nFormat % SV_COUNTRY_LANGUAGE_OFFSET) == 0);
+        if (!bStdPrecision)
         {
             sal_uInt16 nIdx = pFormat->GetSubformatIndex( fVal );
             nPrecision = static_cast<short>(pFormat->GetFormatPrecision( nIdx ));
@@ -679,13 +683,18 @@ double ScDocument::RoundValueAsShown( double fVal, sal_uInt32 nFormat, const ScI
                 case SvNumFormatType::NUMBER:
                 case SvNumFormatType::CURRENCY:
                 {   // tdf#106253 Thousands divisors for format "0,"
-                    nPrecision -=  pFormat->GetThousandDivisorPrecision( nIdx );
+                    const sal_uInt16 nTD = pFormat->GetThousandDivisorPrecision( nIdx );
+                    if (nTD == SvNumberFormatter::UNLIMITED_PRECISION)
+                        // Format contains General keyword, handled below.
+                        bStdPrecision = true;
+                    else
+                        nPrecision -= nTD;
                     break;
                 }
                 default: break;
             }
         }
-        else
+        if (bStdPrecision)
         {
             nPrecision = static_cast<short>(GetDocOptions().GetStdPrecision());
             // #i115512# no rounding for automatic decimals
diff --git a/svl/source/numbers/zformat.cxx b/svl/source/numbers/zformat.cxx
index 6b68e79f1517..22a377a6cc69 100644
--- a/svl/source/numbers/zformat.cxx
+++ b/svl/source/numbers/zformat.cxx
@@ -5947,6 +5947,22 @@ OUString SvNumberformat::GetFormatStringForTimePrecision( int nPrecision ) const
     return sString.makeStringAndClear();
 }
 
+sal_uInt16 SvNumberformat::GetThousandDivisorPrecision( sal_uInt16 nIx ) const
+{
+    if (nIx >= 4)
+        return 0;
+
+    const ImpSvNumberformatInfo& rInfo = NumFor[nIx].Info();
+
+    if (rInfo.eScannedType != SvNumFormatType::NUMBER && rInfo.eScannedType != SvNumFormatType::CURRENCY)
+        return 0;
+
+    if (rInfo.nThousand == FLAG_STANDARD_IN_FORMAT)
+        return SvNumberFormatter::UNLIMITED_PRECISION;
+
+    return rInfo.nThousand * 3;
+}
+
 const CharClass& SvNumberformat::rChrCls() const
 {
     return rScan.GetChrCls();