tdf#111705: Add support of Arabic summation operator (U+1EEF0)

Needed to fix a punch of code to work with UTF-16 not just UCS-2 (in
other words, handle surrogate pairs).

Change-Id: Idd7aa823349a032f20f37abbb588856afbc548d6
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/156994
Tested-by: Jenkins
Reviewed-by: خالد حسني <khaled@libreoffice.org>

3 files changed, 32 insertions, 29 deletions

diff --git a/starmath/source/mathml/mathmlexport.cxx b/starmath/source/mathml/mathmlexport.cxx
index d12b11aafdd5..c498187222e9 100644
--- a/starmath/source/mathml/mathmlexport.cxx
+++ b/starmath/source/mathml/mathmlexport.cxx
@@ -74,12 +74,12 @@ using namespace ::xmloff::token;
 
 namespace
 {
-bool IsInPrivateUseArea(sal_Unicode cChar) { return 0xE000 <= cChar && cChar <= 0xF8FF; }
+bool IsInPrivateUseArea(sal_uInt32 cChar) { return 0xE000 <= cChar && cChar <= 0xF8FF; }
 
-sal_Unicode ConvertMathToMathML(sal_Unicode cChar)
+sal_uInt32 ConvertMathToMathML(std::u16string_view rText, sal_Int32 nIndex = 0)
 {
-    sal_Unicode cRes = cChar;
-    if (IsInPrivateUseArea(cChar))
+    auto cRes = o3tl::iterateCodePoints(rText, &nIndex);
+    if (IsInPrivateUseArea(cRes))
     {
         SAL_WARN("starmath", "Error: private use area characters should no longer be in use!");
         cRes = u'@'; // just some character that should easily be notice as odd in the context
@@ -749,12 +749,9 @@ void SmXMLExport::ExportMath(const SmNode* pNode)
         AddAttribute(XML_NAMESPACE_MATH, XML_MATHVARIANT, XML_NORMAL);
         pMath.reset(new SvXMLElementExport(*this, XML_NAMESPACE_MATH, XML_MI, true, false));
     }
-    sal_Unicode nArse = pTemp->GetText()[0];
-    sal_Unicode cTmp = ConvertMathToMathML(nArse);
-    if (cTmp != 0)
-        nArse = cTmp;
+    auto nArse = ConvertMathToMathML(pTemp->GetText());
     OSL_ENSURE(nArse != 0xffff, "Non existent symbol");
-    GetDocHandler()->characters(OUString(nArse));
+    GetDocHandler()->characters(OUString(&nArse, 1));
 }
 
 void SmXMLExport::ExportText(const SmNode* pNode)
@@ -1350,11 +1347,8 @@ void SmXMLExport::ExportNodes(const SmNode* pNode, int nLevel)
         case SmNodeType::GlyphSpecial:
         case SmNodeType::Math:
         {
-            sal_Unicode cTmp = 0;
             const SmTextNode* pTemp = static_cast<const SmTextNode*>(pNode);
-            if (!pTemp->GetText().isEmpty())
-                cTmp = ConvertMathToMathML(pTemp->GetText()[0]);
-            if (cTmp == 0)
+            if (pTemp->GetText().isEmpty())
             {
                 // no conversion to MathML implemented -> export it as text
                 // thus at least it will not vanish into nothing
diff --git a/starmath/source/mathml/mathmlimport.cxx b/starmath/source/mathml/mathmlimport.cxx
index e83d4e292016..b4d0b0ac8470 100644
--- a/starmath/source/mathml/mathmlimport.cxx
+++ b/starmath/source/mathml/mathmlimport.cxx
@@ -1006,8 +1006,8 @@ namespace
 class SmXMLFencedContext_Impl : public SmXMLRowContext_Impl
 {
 protected:
-    sal_Unicode cBegin;
-    sal_Unicode cEnd;
+    OUString cBegin;
+    OUString cEnd;
     bool bIsStretchy;
 
 public:
@@ -1034,10 +1034,10 @@ void SmXMLFencedContext_Impl::startFastElement(
         {
             //temp, starmath cannot handle multichar brackets (I think)
             case XML_OPEN:
-                cBegin = aIter.toString()[0];
+                cBegin = aIter.toString();
                 break;
             case XML_CLOSE:
-                cEnd = aIter.toString()[0];
+                cEnd = aIter.toString();
                 break;
             case XML_STRETCHY:
                 bIsStretchy = IsXMLToken(aIter, XML_TRUE);
@@ -1377,25 +1377,23 @@ public:
 
 void SmXMLOperatorContext_Impl::TCharacters(const OUString& rChars)
 {
-    aToken.setChar(rChars[0]);
+    aToken.setChar(rChars);
     SmToken bToken;
     if (bIsFenced)
     {
         if (isPrefix)
-            bToken
-                = starmathdatabase::Identify_Prefix_SmXMLOperatorContext_Impl(aToken.cMathChar[0]);
+            bToken = starmathdatabase::Identify_Prefix_SmXMLOperatorContext_Impl(aToken.cMathChar);
         else if (isInfix)
             bToken = SmToken(TMLINE, MS_VERTLINE, "mline", TG::NONE, 0);
         else if (isPostfix)
-            bToken
-                = starmathdatabase::Identify_Postfix_SmXMLOperatorContext_Impl(aToken.cMathChar[0]);
+            bToken = starmathdatabase::Identify_Postfix_SmXMLOperatorContext_Impl(aToken.cMathChar);
         else
             bToken = starmathdatabase::Identify_PrefixPostfix_SmXMLOperatorContext_Impl(
-                aToken.cMathChar[0]);
+                aToken.cMathChar);
     }
     else
-        bToken = starmathdatabase::Identify_SmXMLOperatorContext_Impl(aToken.cMathChar[0],
-                                                                      bIsStretchy);
+        bToken
+            = starmathdatabase::Identify_SmXMLOperatorContext_Impl(aToken.cMathChar, bIsStretchy);
     if (bToken.eType != TERROR)
         aToken = bToken;
 }
diff --git a/starmath/source/mathml/starmathdatabase.cxx b/starmath/source/mathml/starmathdatabase.cxx
index cc6614cee4f1..6eb6d209c8e1 100644
--- a/starmath/source/mathml/starmathdatabase.cxx
+++ b/starmath/source/mathml/starmathdatabase.cxx
@@ -20,8 +20,10 @@
 #include <starmathdatabase.hxx>
 #include <types.hxx>
 
-SmToken starmathdatabase::Identify_SmXMLOperatorContext_Impl(sal_Unicode cChar, bool bIsStretchy)
+SmToken starmathdatabase::Identify_SmXMLOperatorContext_Impl(std::u16string_view rText,
+                                                             bool bIsStretchy, sal_Int32 nIndex)
 {
+    auto cChar = o3tl::iterateCodePoints(rText, &nIndex);
     switch (cChar)
     {
         case MS_COPROD:
@@ -45,6 +47,8 @@ SmToken starmathdatabase::Identify_SmXMLOperatorContext_Impl(sal_Unicode cChar,
             return SmToken(TPROD, MS_PROD, "prod", TG::Oper, 5);
         case MS_SUM:
             return SmToken(TSUM, MS_SUM, "sum", TG::Oper, 5);
+        case MS_MAJ:
+            return SmToken(TSUM, MS_MAJ, "maj", TG::Oper, 5);
         case MS_FACT:
             return SmToken(TFACT, MS_FACT, "!", TG::UnOper, 5);
         case MS_NEG:
@@ -217,8 +221,10 @@ SmToken starmathdatabase::Identify_SmXMLOperatorContext_Impl(sal_Unicode cChar,
     }
 }
 
-SmToken starmathdatabase::Identify_Prefix_SmXMLOperatorContext_Impl(sal_Unicode cChar)
+SmToken starmathdatabase::Identify_Prefix_SmXMLOperatorContext_Impl(std::u16string_view rText,
+                                                                    sal_Int32 nIndex)
 {
+    auto cChar = o3tl::iterateCodePoints(rText, &nIndex);
     switch (cChar)
     {
         case MS_VERTLINE:
@@ -264,8 +270,10 @@ SmToken starmathdatabase::Identify_Prefix_SmXMLOperatorContext_Impl(sal_Unicode
     }
 }
 
-SmToken starmathdatabase::Identify_Postfix_SmXMLOperatorContext_Impl(sal_Unicode cChar)
+SmToken starmathdatabase::Identify_Postfix_SmXMLOperatorContext_Impl(std::u16string_view rText,
+                                                                     sal_Int32 nIndex)
 {
+    auto cChar = o3tl::iterateCodePoints(rText, &nIndex);
     switch (cChar)
     {
         case MS_VERTLINE:
@@ -311,8 +319,11 @@ SmToken starmathdatabase::Identify_Postfix_SmXMLOperatorContext_Impl(sal_Unicode
     }
 }
 
-SmToken starmathdatabase::Identify_PrefixPostfix_SmXMLOperatorContext_Impl(sal_Unicode cChar)
+SmToken
+starmathdatabase::Identify_PrefixPostfix_SmXMLOperatorContext_Impl(std::u16string_view rText,
+                                                                   sal_Int32 nIndex)
 {
+    auto cChar = o3tl::iterateCodePoints(rText, &nIndex);
     switch (cChar)
     {
         case MS_VERTLINE: