Resolves: tdf#76310 Preserve whitespace TAB, CR, LF in formula expressions

Allowed whitespace in ODFF and OOXML are
U+0020 SPACE
U+0009 CHARACTER TABULATION
U+000A LINE FEED
U+000D CARRIAGE RETURN

Line feed and carriage return look a bit funny in the Function Wizard if
part of a function's argument but work. Once a formula is edited, CR are
converted to LF though, probably already in EditEngine, didn't
investigate.

Change-Id: I6278f6be48872e0710a3d74212db391dda249ed2
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/119635
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Jenkins

3 files changed, 57 insertions, 19 deletions

diff --git a/formula/source/core/api/FormulaCompiler.cxx b/formula/source/core/api/FormulaCompiler.cxx
index be5ce09d132f..f7174807f0f4 100644
--- a/formula/source/core/api/FormulaCompiler.cxx
+++ b/formula/source/core/api/FormulaCompiler.cxx
@@ -475,7 +475,8 @@ uno::Sequence< sheet::FormulaOpCodeMapEntry > FormulaCompiler::OpCodeMap::create
             { FormulaMapGroupSpecialOffset::DB_AREA           , ocDBArea }         ,
             /* TODO: { FormulaMapGroupSpecialOffset::TABLE_REF         , ocTableRef }       , */
             { FormulaMapGroupSpecialOffset::MACRO             , ocMacro }          ,
-            { FormulaMapGroupSpecialOffset::COL_ROW_NAME      , ocColRowName }
+            { FormulaMapGroupSpecialOffset::COL_ROW_NAME      , ocColRowName }     ,
+            { FormulaMapGroupSpecialOffset::WHITESPACE        , ocWhitespace }
         };
         const size_t nCount = SAL_N_ELEMENTS(aMap);
         // Preallocate vector elements.
@@ -1267,14 +1268,18 @@ bool FormulaCompiler::GetToken()
              nWasColRowName = 1;
         else
              nWasColRowName = 0;
+        OpCode eTmpOp;
         mpToken = maArrIterator.Next();
-        while( mpToken && mpToken->GetOpCode() == ocSpaces )
+        while (mpToken && ((eTmpOp = mpToken->GetOpCode()) == ocSpaces || eTmpOp == ocWhitespace))
         {
-            // For significant whitespace remember last ocSpaces token. Usually
-            // there's only one even for multiple spaces.
-            pSpacesToken = mpToken;
-            if ( nWasColRowName )
-                nWasColRowName++;
+            if (eTmpOp == ocSpaces)
+            {
+                // For significant whitespace remember last ocSpaces token.
+                // Usually there's only one even for multiple spaces.
+                pSpacesToken = mpToken;
+                if ( nWasColRowName )
+                    nWasColRowName++;
+            }
             if ( bAutoCorrect && !pStack )
                 CreateStringFromToken( aCorrectedFormula, mpToken.get() );
             mpToken = maArrIterator.Next();
@@ -2272,10 +2277,10 @@ const FormulaToken* FormulaCompiler::CreateStringFromToken( OUStringBuffer& rBuf
     if( bSpaces )
         rBuffer.append( ' ');
 
-    if( eOp == ocSpaces )
+    if (eOp == ocSpaces || eOp == ocWhitespace)
     {
         bool bWriteSpaces = true;
-        if (mxSymbols->isODFF())
+        if (eOp == ocSpaces && mxSymbols->isODFF())
         {
             const FormulaToken* p = maArrIterator.PeekPrevNoSpaces();
             bool bIntersectionOp = (p && p->GetOpCode() == ocColRowName);
@@ -2316,7 +2321,10 @@ const FormulaToken* FormulaCompiler::CreateStringFromToken( OUStringBuffer& rBuf
             sal_uInt8 n = t->GetByte();
             for ( sal_uInt8 j=0; j<n; ++j )
             {
-                rBuffer.append( ' ');
+                if (eOp == ocWhitespace)
+                    rBuffer.append( t->GetChar());
+                else
+                    rBuffer.append( ' ');
             }
         }
     }
@@ -2403,6 +2411,7 @@ const FormulaToken* FormulaCompiler::CreateStringFromToken( OUStringBuffer& rBuf
                                 case ocPush:
                                 case ocRange:
                                 case ocSpaces:
+                                case ocWhitespace:
                                     break;
                                 default:
                                     nLevel = 0;
diff --git a/formula/source/core/api/token.cxx b/formula/source/core/api/token.cxx
index 0af1f63f0e5e..c5b69acf2c90 100644
--- a/formula/source/core/api/token.cxx
+++ b/formula/source/core/api/token.cxx
@@ -244,6 +244,13 @@ void FormulaToken::SetSheet( sal_Int16 )
     assert( !"virtual dummy called" );
 }
 
+sal_Unicode FormulaToken::GetChar() const
+{
+    // This Get is worth an assert.
+    assert( !"virtual dummy called" );
+    return 0;
+}
+
 short* FormulaToken::GetJump() const
 {
     SAL_WARN( "formula.core", "FormulaToken::GetJump: virtual dummy called" );
@@ -348,6 +355,15 @@ bool FormulaToken::TextEqual( const FormulaToken& rToken ) const
 // real implementations of virtual functions
 
 
+sal_uInt8   FormulaSpaceToken::GetByte() const  { return nByte; }
+sal_Unicode FormulaSpaceToken::GetChar() const  { return cChar; }
+bool FormulaSpaceToken::operator==( const FormulaToken& r ) const
+{
+    return FormulaToken::operator==( r ) && nByte == r.GetByte() &&
+        cChar == r.GetChar();
+}
+
+
 sal_uInt8   FormulaByteToken::GetByte() const           { return nByte; }
 void        FormulaByteToken::SetByte( sal_uInt8 n )    { nByte = n; }
 ParamClass  FormulaByteToken::GetInForceArray() const    { return eInForceArray; }
@@ -425,6 +441,13 @@ bool FormulaTokenArray::AddFormulaToken(
                     AddStringXML( aStrVal );
                 else if ( eOpCode == ocExternal || eOpCode == ocMacro )
                     Add( new formula::FormulaExternalToken( eOpCode, aStrVal ) );
+                else if ( eOpCode == ocWhitespace )
+                {
+                    // Simply ignore empty string.
+                    // Convention is one character repeated.
+                    if (!aStrVal.isEmpty())
+                        Add( new formula::FormulaSpaceToken( static_cast<sal_uInt8>(aStrVal.getLength()), aStrVal[0]));
+                }
                 else
                     bError = true;      // unexpected string: don't know what to do with it
             }
@@ -1472,17 +1495,21 @@ FormulaTokenArray * FormulaTokenArray::RewriteMissing( const MissingConvention &
     return pNewArr;
 }
 
+namespace {
+inline bool isWhitespace( OpCode eOp ) { return eOp == ocSpaces || eOp == ocWhitespace; }
+}
+
 bool FormulaTokenArray::MayReferenceFollow()
 {
     if ( pCode && nLen > 0 )
     {
         // ignore trailing spaces
         sal_uInt16 i = nLen - 1;
-        while ( i > 0 && pCode[i]->GetOpCode() == SC_OPCODE_SPACES )
+        while (i > 0 && isWhitespace( pCode[i]->GetOpCode()))
         {
             --i;
         }
-        if ( i > 0 || pCode[i]->GetOpCode() != SC_OPCODE_SPACES )
+        if (i > 0 || !isWhitespace( pCode[i]->GetOpCode()))
         {
             OpCode eOp = pCode[i]->GetOpCode();
             if ( (SC_OPCODE_START_BIN_OP <= eOp && eOp < SC_OPCODE_STOP_BIN_OP ) ||
@@ -1756,7 +1783,7 @@ FormulaToken* FormulaTokenArrayPlainIterator::NextNoSpaces()
 {
     if( mpFTA->GetArray() )
     {
-        while( (mnIndex < mpFTA->GetLen()) && (mpFTA->GetArray()[ mnIndex ]->GetOpCode() == ocSpaces) )
+        while ((mnIndex < mpFTA->GetLen()) && isWhitespace( mpFTA->GetArray()[ mnIndex ]->GetOpCode()))
             ++mnIndex;
         if( mnIndex < mpFTA->GetLen() )
             return mpFTA->GetArray()[ mnIndex++ ];
@@ -1793,7 +1820,7 @@ FormulaToken* FormulaTokenArrayPlainIterator::PeekNextNoSpaces() const
     if( mpFTA->GetArray() && mnIndex < mpFTA->GetLen() )
     {
         sal_uInt16 j = mnIndex;
-        while ( j < mpFTA->GetLen() && mpFTA->GetArray()[j]->GetOpCode() == ocSpaces )
+        while (j < mpFTA->GetLen() && isWhitespace( mpFTA->GetArray()[j]->GetOpCode()))
             j++;
         if ( j < mpFTA->GetLen() )
             return mpFTA->GetArray()[ j ];
@@ -1809,9 +1836,9 @@ FormulaToken* FormulaTokenArrayPlainIterator::PeekPrevNoSpaces() const
     if( mpFTA->GetArray() && mnIndex > 1 )
     {
         sal_uInt16 j = mnIndex - 2;
-        while ( mpFTA->GetArray()[j]->GetOpCode() == ocSpaces && j > 0 )
+        while (isWhitespace( mpFTA->GetArray()[j]->GetOpCode()) && j > 0 )
             j--;
-        if ( j > 0 || mpFTA->GetArray()[j]->GetOpCode() != ocSpaces )
+        if (j > 0 || !isWhitespace( mpFTA->GetArray()[j]->GetOpCode()))
             return mpFTA->GetArray()[ j ];
         else
             return nullptr;
diff --git a/formula/source/ui/dlg/formula.cxx b/formula/source/ui/dlg/formula.cxx
index 81931d8d586b..36b59d5eb0ec 100644
--- a/formula/source/ui/dlg/formula.cxx
+++ b/formula/source/ui/dlg/formula.cxx
@@ -389,6 +389,9 @@ sal_Int32 FormulaDlg_Impl::GetFunctionPos(sal_Int32 nPos)
         sal_Int32 nOldTokPos = 1;
         sal_Int32 nPrevFuncPos = 1;
         short nBracketCount = 0;
+        const sal_Int32 nOpPush = m_aSpecialOpCodes[sheet::FormulaMapGroupSpecialOffset::PUSH].Token.OpCode;
+        const sal_Int32 nOpSpaces = m_aSpecialOpCodes[sheet::FormulaMapGroupSpecialOffset::SPACES].Token.OpCode;
+        const sal_Int32 nOpWhitespace = m_aSpecialOpCodes[sheet::FormulaMapGroupSpecialOffset::WHITESPACE].Token.OpCode;
         while ( pIter != pEnd )
         {
             const sal_Int32 eOp = pIter->OpCode;
@@ -401,8 +404,7 @@ sal_Int32 FormulaDlg_Impl::GetFunctionPos(sal_Int32 nPos)
                 m_xBtnMatrix->set_active(true);
             }
 
-            if (eOp == m_aSpecialOpCodes[sheet::FormulaMapGroupSpecialOffset::PUSH].Token.OpCode ||
-                eOp == m_aSpecialOpCodes[sheet::FormulaMapGroupSpecialOffset::SPACES].Token.OpCode)
+            if (eOp == nOpPush || eOp == nOpSpaces || eOp == nOpWhitespace)
             {
                 const sal_Int32 n1 = nTokPos < 0 ? -1 : aFormString.indexOf( sep, nTokPos);
                 const sal_Int32 n2 = nTokPos < 0 ? -1 : aFormString.indexOf( ')', nTokPos);
@@ -444,7 +446,7 @@ sal_Int32 FormulaDlg_Impl::GetFunctionPos(sal_Int32 nPos)
                     m_pFunctionOpCodesEnd,
                     [&eOp](const sheet::FormulaOpCodeMapEntry& aEntry) { return aEntry.Token.OpCode == eOp; });
 
-            if ( bIsFunction && m_aSpecialOpCodes[sheet::FormulaMapGroupSpecialOffset::SPACES].Token.OpCode != eOp )
+            if ( bIsFunction && nOpSpaces != eOp && nOpWhitespace != eOp )
             {
                 nPrevFuncPos = nFuncPos;
                 nFuncPos = nOldTokPos;