Resolves: tdf#76310 Preserve whitespace TAB, CR, LF in formula expressions

Allowed whitespace in ODFF and OOXML are
U+0020 SPACE
U+0009 CHARACTER TABULATION
U+000A LINE FEED
U+000D CARRIAGE RETURN

Line feed and carriage return look a bit funny in the Function Wizard if
part of a function's argument but work. Once a formula is edited, CR are
converted to LF though, probably already in EditEngine, didn't
investigate.

Change-Id: I6278f6be48872e0710a3d74212db391dda249ed2
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/119635
Reviewed-by: Eike Rathke <erack@redhat.com>
Tested-by: Jenkins

3 files changed, 43 insertions, 20 deletions

diff --git a/include/formula/compiler.hxx b/include/formula/compiler.hxx
index baf3e23f6161..fcf7326d3e0f 100644
--- a/include/formula/compiler.hxx
+++ b/include/formula/compiler.hxx
@@ -40,26 +40,27 @@
 #define SC_OPCODE_BAD                14
 #define SC_OPCODE_STRINGXML          15
 #define SC_OPCODE_SPACES             16
-#define SC_OPCODE_MAT_REF            17
-#define SC_OPCODE_DB_AREA            18     /* additional access operators */
-#define SC_OPCODE_TABLE_REF          19
-#define SC_OPCODE_MACRO              20
-#define SC_OPCODE_COL_ROW_NAME       21
-#define SC_OPCODE_COL_ROW_NAME_AUTO  22
-#define SC_OPCODE_PERCENT_SIGN       23     /* operator _follows_ value */
-#define SC_OPCODE_ARRAY_OPEN         24
-#define SC_OPCODE_ARRAY_CLOSE        25
-#define SC_OPCODE_ARRAY_ROW_SEP      26
-#define SC_OPCODE_ARRAY_COL_SEP      27     /* some convs use sep != col_sep */
-#define SC_OPCODE_TABLE_REF_OPEN     28
-#define SC_OPCODE_TABLE_REF_CLOSE    29
-#define SC_OPCODE_TABLE_REF_ITEM_ALL      30
-#define SC_OPCODE_TABLE_REF_ITEM_HEADERS  31
-#define SC_OPCODE_TABLE_REF_ITEM_DATA     32
-#define SC_OPCODE_TABLE_REF_ITEM_TOTALS   33
-#define SC_OPCODE_TABLE_REF_ITEM_THIS_ROW 34
-#define SC_OPCODE_STOP_DIV           35
-#define SC_OPCODE_SKIP               36     /* used to skip raw tokens during string compilation */
+#define SC_OPCODE_WHITESPACE         17
+#define SC_OPCODE_MAT_REF            18
+#define SC_OPCODE_DB_AREA            19     /* additional access operators */
+#define SC_OPCODE_TABLE_REF          20
+#define SC_OPCODE_MACRO              21
+#define SC_OPCODE_COL_ROW_NAME       22
+#define SC_OPCODE_COL_ROW_NAME_AUTO  23
+#define SC_OPCODE_PERCENT_SIGN       24     /* operator _follows_ value */
+#define SC_OPCODE_ARRAY_OPEN         25
+#define SC_OPCODE_ARRAY_CLOSE        26
+#define SC_OPCODE_ARRAY_ROW_SEP      27
+#define SC_OPCODE_ARRAY_COL_SEP      28     /* some convs use sep != col_sep */
+#define SC_OPCODE_TABLE_REF_OPEN     29
+#define SC_OPCODE_TABLE_REF_CLOSE    30
+#define SC_OPCODE_TABLE_REF_ITEM_ALL      31
+#define SC_OPCODE_TABLE_REF_ITEM_HEADERS  32
+#define SC_OPCODE_TABLE_REF_ITEM_DATA     33
+#define SC_OPCODE_TABLE_REF_ITEM_TOTALS   34
+#define SC_OPCODE_TABLE_REF_ITEM_THIS_ROW 35
+#define SC_OPCODE_STOP_DIV           36
+#define SC_OPCODE_SKIP               37     /* used to skip raw tokens during string compilation */
 
 /*** error constants #... ***/
 #define SC_OPCODE_START_ERRORS       40
diff --git a/include/formula/opcode.hxx b/include/formula/opcode.hxx
index 3123e8f3fa38..d92ae0b1d41d 100644
--- a/include/formula/opcode.hxx
+++ b/include/formula/opcode.hxx
@@ -53,6 +53,7 @@ enum OpCode : sal_uInt16
         ocBad               = SC_OPCODE_BAD,
         ocStringXML         = SC_OPCODE_STRINGXML,
         ocSpaces            = SC_OPCODE_SPACES,
+        ocWhitespace        = SC_OPCODE_WHITESPACE,
         ocMatRef            = SC_OPCODE_MAT_REF,
         ocTableRefItemAll     = SC_OPCODE_TABLE_REF_ITEM_ALL,
         ocTableRefItemHeaders = SC_OPCODE_TABLE_REF_ITEM_HEADERS,
@@ -545,6 +546,7 @@ inline std::string OpCodeEnumToString(OpCode eCode)
     case ocBad: return "Bad";
     case ocStringXML: return "StringXML";
     case ocSpaces: return "Spaces";
+    case ocWhitespace: return "Whitespace";
     case ocMatRef: return "MatRef";
     case ocTableRefItemAll: return "TableRefItemAll";
     case ocTableRefItemHeaders: return "TableRefItemHeaders";
diff --git a/include/formula/token.hxx b/include/formula/token.hxx
index 3fa00e89339f..77bf3eeb90ea 100644
--- a/include/formula/token.hxx
+++ b/include/formula/token.hxx
@@ -187,6 +187,7 @@ public:
     virtual void                SetIndex( sal_uInt16 n );
     virtual sal_Int16           GetSheet() const;
     virtual void                SetSheet( sal_Int16 n );
+    virtual sal_Unicode         GetChar() const;
     virtual short*              GetJump() const;
     virtual const OUString&     GetExternal() const;
     virtual FormulaToken*       GetFAPOrigToken() const;
@@ -225,6 +226,25 @@ inline void intrusive_ptr_release(const FormulaToken* p)
     p->DecRef();
 }
 
+class FORMULA_DLLPUBLIC FormulaSpaceToken : public FormulaToken
+{
+private:
+            sal_uInt8           nByte;
+            sal_Unicode         cChar;
+public:
+                                FormulaSpaceToken( sal_uInt8 n, sal_Unicode c ) :
+                                    FormulaToken( svByte, ocWhitespace ),
+                                    nByte( n ), cChar( c ) {}
+                                FormulaSpaceToken( const FormulaSpaceToken& r ) :
+                                    FormulaToken( r ),
+                                    nByte( r.nByte ), cChar( r.cChar ) {}
+
+    virtual FormulaToken*       Clone() const override { return new FormulaSpaceToken(*this); }
+    virtual sal_uInt8           GetByte() const override;
+    virtual sal_Unicode         GetChar() const override;
+    virtual bool                operator==( const FormulaToken& rToken ) const override;
+};
+
 class FORMULA_DLLPUBLIC FormulaByteToken : public FormulaToken
 {
 private: