tdf#105170: Ignore diacritics on precomposed chars

When ignoring diacritics, the old code just ignore combining marks which
works when using decomposed forms (NFD) but does not work for
precomposed forms (NFC).

Instead, we now decompose, strip marks, then recompose, and use a nice
icu::Transliterator that does the hard work for us. As a bonus, we
should now handle surrogate pairs fine (most of the time).

The new code (in ignoreDiacritics_CTL::folding()) might not be as
efficient as the old code that used transliteration_Ignore::folding(),
but it is less ugly and more easier to resonate with, or so I hope.

Change-Id: If48c8be30527580cdd68f20b40a6533c5f258d83
Reviewed-on: https://gerrit.libreoffice.org/32826
Tested-by: Jenkins <ci@libreoffice.org>
Reviewed-by: Eike Rathke <erack@redhat.com>

2 files changed, 87 insertions, 11 deletions

diff --git a/i18npool/inc/transliteration_Ignore.hxx b/i18npool/inc/transliteration_Ignore.hxx
index e6573b6adfaa..3b67d64120e7 100644
--- a/i18npool/inc/transliteration_Ignore.hxx
+++ b/i18npool/inc/transliteration_Ignore.hxx
@@ -21,6 +21,7 @@
 
 #include <transliteration_commonclass.hxx>
 #include <i18nutil/oneToOneMapping.hxx>
+#include <unicode/translit.h>
 
 typedef sal_Unicode (*TransFunc)(const sal_Unicode);
 
@@ -91,9 +92,25 @@ TRANSLITERATION_IGNORE(Space_ja_JP)
 TRANSLITERATION_IGNORE(TraditionalKana_ja_JP)
 TRANSLITERATION_IGNORE(TraditionalKanji_ja_JP)
 TRANSLITERATION_IGNORE(ZiZu_ja_JP)
-TRANSLITERATION_IGNORE(Diacritics_CTL)
 TRANSLITERATION_IGNORE(Kashida_CTL)
 
+class ignoreDiacritics_CTL : public transliteration_Ignore
+{
+    icu::Transliterator* m_transliterator;
+
+public:
+    ignoreDiacritics_CTL();
+
+    OUString SAL_CALL
+    folding(const OUString& rInStr, sal_Int32 nStartPos, sal_Int32 nCount, css::uno::Sequence<sal_Int32>& rOffset)
+        throw(css::uno::RuntimeException, std::exception) override;
+
+    sal_Unicode SAL_CALL
+    transliterateChar2Char(sal_Unicode nInChar)
+        throw(css::uno::RuntimeException,
+        css::i18n::MultipleCharsOutputException, std::exception) override;
+};
+
 #undef TRANSLITERATION_IGNORE
 
 #define TRANSLITERATION_IGNORE( name ) \
diff --git a/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx b/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx
index 8d6292c38d0d..c910b94f8a96 100644
--- a/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx
+++ b/i18npool/source/transliteration/ignoreDiacritics_CTL.cxx
@@ -9,25 +9,84 @@
 
 #include <transliteration_Ignore.hxx>
 #include <unicode/uchar.h>
+#include <unicode/translit.h>
 
 namespace com { namespace sun { namespace star { namespace i18n {
 
-sal_Unicode
-ignoreDiacritics_CTL_translator (const sal_Unicode c)
-{
-    if(u_getIntPropertyValue(c, UCHAR_GENERAL_CATEGORY) == U_NON_SPACING_MARK)
-        return 0xffff; // Skip this character
-
-    return c;
-}
-
 ignoreDiacritics_CTL::ignoreDiacritics_CTL()
 {
-    func = ignoreDiacritics_CTL_translator;
+    func = nullptr;
     table = nullptr;
     map = nullptr;
     transliterationName = "ignoreDiacritics_CTL";
     implementationName = "com.sun.star.i18n.Transliteration.ignoreDiacritics_CTL";
+
+    UErrorCode nStatus = U_ZERO_ERROR;
+    m_transliterator = icu::Transliterator::createInstance("NFD; [:M:] Remove; NFC",
+            UTRANS_FORWARD, nStatus);
+    if (U_FAILURE(nStatus))
+        m_transliterator = nullptr;
+}
+
+sal_Unicode SAL_CALL
+ignoreDiacritics_CTL::transliterateChar2Char(sal_Unicode nInChar)
+    throw(css::uno::RuntimeException,
+        css::i18n::MultipleCharsOutputException, std::exception)
+{
+    if (!m_transliterator)
+        throw css::uno::RuntimeException();
+
+    UnicodeString aChar(nInChar);
+    m_transliterator->transliterate(aChar);
+
+    if (aChar.isEmpty())
+        return 0xffff; // Skip this character.
+
+    if (aChar.length() > 1)
+        return nInChar; // Don't know what to do here, return the otiginal.
+
+    return aChar[0];
+}
+
+OUString SAL_CALL
+ignoreDiacritics_CTL::folding(const OUString& rInStr, sal_Int32 nStartPos,
+    sal_Int32 nCount, css::uno::Sequence<sal_Int32>& rOffset)
+    throw(css::uno::RuntimeException, std::exception)
+{
+    if (!m_transliterator)
+        throw css::uno::RuntimeException();
+
+    OUString aOutStr;
+
+    sal_Int32 nPosition = nStartPos;
+    sal_Int32 nOffset = 0;
+    if (useOffset)
+        rOffset.realloc(nCount);
+
+    while (nPosition < nStartPos + nCount)
+    {
+        sal_Int32 nIndex = nPosition;
+        UChar32 nChar = rInStr.iterateCodePoints(&nIndex);
+        UnicodeString aChar(nChar);
+        m_transliterator->transliterate(aChar);
+
+        if (useOffset && nOffset + aChar.length() > rOffset.getLength())
+            rOffset.realloc(rOffset.getLength() + aChar.length());
+
+        for (int32_t i = 0; i < aChar.length(); i++)
+        {
+            aOutStr += OUStringLiteral1(aChar[i]);
+            if (useOffset)
+                rOffset[nOffset++] = nPosition;
+        }
+
+        nPosition = nIndex;
+    }
+
+    if (useOffset)
+        rOffset.realloc(aOutStr.getLength());
+
+    return aOutStr;
 }
 
 } } } }