Resolves: fdo49629 GotoEndOfWord fails with footnote at word end

a) remove special handling of 0x0002 in our custom icu rules.
Which brings us a step closer to getting rid of at least
some of them in favour of the defaults
b) expand the 0x02 in SwTxtNode::BuildConversionMap like we
do for fields so

Good side effect is our word count and character count now take into account
the actual footnote indicator text, as does our cursor travelling. Both of
which are more word-alike.

Change-Id: I3b0024ac4b10934bee7a9e83b0fce08a18556c7b

11 files changed, 43 insertions, 10 deletions

diff --git a/i18npool/qa/cppunit/test_breakiterator.cxx b/i18npool/qa/cppunit/test_breakiterator.cxx
index 3c2ba50b8712..3ed44fcb9ff5 100644
--- a/i18npool/qa/cppunit/test_breakiterator.cxx
+++ b/i18npool/qa/cppunit/test_breakiterator.cxx
@@ -125,6 +125,39 @@ void TestBreakIterator::testLineBreaking()
             CPPUNIT_ASSERT_MESSAGE("Expected a break at the the start of the word", aResult.breakIndex == aWord.getLength()+1);
         }
     }
+
+    //See https://bugs.freedesktop.org/show_bug.cgi?id=49629
+    for (int mode = i18n::WordType::ANY_WORD; mode <= i18n::WordType::WORD_COUNT; ++mode)
+    {
+        //make sure that in all cases isBeginWord and isEndWord matches getWordBoundary
+        //
+        //test "Word", then "Word\x01" then "Word\x02"
+        for (sal_Unicode i = 0; i < 3; ++i)
+        {
+            ::rtl::OUString aTest("Word");
+            if (i > 0)
+                aTest += rtl::OUString(i) + rtl::OUString("Word");
+            i18n::Boundary aBounds = m_xBreak->getWordBoundary(aTest, 0, aLocale, mode, true);
+            switch (mode)
+            {
+                case i18n::WordType::ANY_WORD:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+                case i18n::WordType::ANYWORD_IGNOREWHITESPACES:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+                case i18n::WordType::DICTIONARY_WORD:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+                case i18n::WordType::WORD_COUNT:
+                    CPPUNIT_ASSERT(aBounds.startPos == 0 && aBounds.endPos == 4);
+                    break;
+            }
+
+            CPPUNIT_ASSERT(m_xBreak->isBeginWord(aTest, aBounds.startPos, aLocale, mode));
+            CPPUNIT_ASSERT(m_xBreak->isEndWord(aTest, aBounds.endPos, aLocale, mode));
+        }
+    }
 }
 
 //See https://bugs.freedesktop.org/show_bug.cgi?id=49629
diff --git a/i18npool/source/breakiterator/data/dict_word.txt b/i18npool/source/breakiterator/data/dict_word.txt
index f69f038e7f7e..b1666f44daab 100644
--- a/i18npool/source/breakiterator/data/dict_word.txt
+++ b/i18npool/source/breakiterator/data/dict_word.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/dict_word_ca.txt b/i18npool/source/breakiterator/data/dict_word_ca.txt
index ca4a14abe724..6ad6a0bbb915 100644
--- a/i18npool/source/breakiterator/data/dict_word_ca.txt
+++ b/i18npool/source/breakiterator/data/dict_word_ca.txt
@@ -22,7 +22,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
                                    [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
 
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Katakana
                            - [:Script = Thai:]
                            - [:Script = Lao:]
diff --git a/i18npool/source/breakiterator/data/dict_word_fi.txt b/i18npool/source/breakiterator/data/dict_word_fi.txt
index 2d7617908576..7026c992f1f9 100644
--- a/i18npool/source/breakiterator/data/dict_word_fi.txt
+++ b/i18npool/source/breakiterator/data/dict_word_fi.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:] [:name = HYPHEN-MINUS:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:] [:name = HYPHEN-MINUS:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/dict_word_he.txt b/i18npool/source/breakiterator/data/dict_word_he.txt
index b98350e9d3fe..40197d92a431 100644
--- a/i18npool/source/breakiterator/data/dict_word_he.txt
+++ b/i18npool/source/breakiterator/data/dict_word_he.txt
@@ -22,7 +22,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
                                    [:name = HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK:]];
 
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Katakana
                            - [:Script = Thai:]
                            - [:Script = Lao:]
diff --git a/i18npool/source/breakiterator/data/dict_word_hu.txt b/i18npool/source/breakiterator/data/dict_word_hu.txt
index 49ae8b5bf407..33d014ec60ed 100644
--- a/i18npool/source/breakiterator/data/dict_word_hu.txt
+++ b/i18npool/source/breakiterator/data/dict_word_hu.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                 [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
                 [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:]
                 [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:]
diff --git a/i18npool/source/breakiterator/data/dict_word_nodash.txt b/i18npool/source/breakiterator/data/dict_word_nodash.txt
index 4c5c80823041..279cc50e5b66 100644
--- a/i18npool/source/breakiterator/data/dict_word_nodash.txt
+++ b/i18npool/source/breakiterator/data/dict_word_nodash.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/dict_word_prepostdash.txt b/i18npool/source/breakiterator/data/dict_word_prepostdash.txt
index 0a60cc5d4924..fb29b478af21 100644
--- a/i18npool/source/breakiterator/data/dict_word_prepostdash.txt
+++ b/i18npool/source/breakiterator/data/dict_word_prepostdash.txt
@@ -30,7 +30,7 @@ $Hangul = [:Script = HANGUL:];
 $PrePostDashHyphen = [ [:name = HYPHEN-MINUS:] ];
 
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
+$ALetter   = [[:Alphabetic:] [:name= COMMERCIAL AT:] [:name= HEBREW PUNCTUATION GERESH:]
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/edit_word.txt b/i18npool/source/breakiterator/data/edit_word.txt
index ff8321262269..92b344c19d41 100644
--- a/i18npool/source/breakiterator/data/edit_word.txt
+++ b/i18npool/source/breakiterator/data/edit_word.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
+$ALetter   = [[:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/edit_word_he.txt b/i18npool/source/breakiterator/data/edit_word_he.txt
index 0061f552e1bf..0b5908814e08 100644
--- a/i18npool/source/breakiterator/data/edit_word_he.txt
+++ b/i18npool/source/breakiterator/data/edit_word_he.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
+$ALetter   = [[:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
                            - $Ideographic
                            - $Katakana
                            - $Hangul
diff --git a/i18npool/source/breakiterator/data/edit_word_hu.txt b/i18npool/source/breakiterator/data/edit_word_hu.txt
index 5262726cbce5..4a08acab0029 100644
--- a/i18npool/source/breakiterator/data/edit_word_hu.txt
+++ b/i18npool/source/breakiterator/data/edit_word_hu.txt
@@ -24,7 +24,7 @@ $Katakana  = [[:Script = KATAKANA:] [:name = KATAKANA-HIRAGANA PROLONGED SOUND M
 $Ideographic = [:Ideographic:];
 $Hangul = [:Script = HANGUL:];
 
-$ALetter   = [\u0002 [:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
+$ALetter   = [[:Alphabetic:] [:name= NO-BREAK SPACE:] [:name= HEBREW PUNCTUATION GERESH:] 
                 [:name = PERCENT SIGN:] [:name = PER MILLE SIGN:] [:name = PER TEN THOUSAND SIGN:]
                 [:name = SECTION SIGN:] [:name = DEGREE SIGN:] [:name = EURO SIGN:]
                 [:name = HYPHEN-MINUS:] [:name = EN DASH:] [:name = EM DASH:]