Use po file for merge 1. step

Make executables to waiting for po as input
Add some function to po files
Rewrite makefiles to use po files for merge
Change KeyId generator not to use xml special
charachters
One filetype remain: properties

Change-Id: If2bd2ae7afc61518c2d838a3cae301ef62dd9761
Reviewed-on: https://gerrit.libreoffice.org/593
Reviewed-by: Michael Stahl <mstahl@redhat.com>
Tested-by: Michael Stahl <mstahl@redhat.com>

9 files changed, 238 insertions, 74 deletions

diff --git a/l10ntools/Executable_cfgex.mk b/l10ntools/Executable_cfgex.mk
index 7dc23275da14..9fea979482bc 100644
--- a/l10ntools/Executable_cfgex.mk
+++ b/l10ntools/Executable_cfgex.mk
@@ -33,6 +33,7 @@ $(eval $(call gb_Executable_set_include,cfgex,\
 
 $(eval $(call gb_Executable_use_libraries,cfgex,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_add_scanners,cfgex,\
diff --git a/l10ntools/Executable_helpex.mk b/l10ntools/Executable_helpex.mk
index 470bd4c89d2c..3569dffb5156 100644
--- a/l10ntools/Executable_helpex.mk
+++ b/l10ntools/Executable_helpex.mk
@@ -33,6 +33,7 @@ $(eval $(call gb_Executable_set_include,helpex,\
 
 $(eval $(call gb_Executable_use_libraries,helpex,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_use_external,helpex,expat_utf8))
diff --git a/l10ntools/Executable_transex3.mk b/l10ntools/Executable_transex3.mk
index bc9a85272d9f..2d9590641d83 100644
--- a/l10ntools/Executable_transex3.mk
+++ b/l10ntools/Executable_transex3.mk
@@ -33,6 +33,7 @@ $(eval $(call gb_Executable_set_include,transex3,\
 
 $(eval $(call gb_Executable_use_libraries,transex3,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_add_scanners,transex3,\
diff --git a/l10ntools/Executable_ulfex.mk b/l10ntools/Executable_ulfex.mk
index e82c9b8ee812..f3fd559ac4f7 100644
--- a/l10ntools/Executable_ulfex.mk
+++ b/l10ntools/Executable_ulfex.mk
@@ -33,6 +33,7 @@ $(eval $(call gb_Executable_set_include,ulfex,\
 
 $(eval $(call gb_Executable_use_libraries,ulfex,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_use_static_libraries,ulfex,\
diff --git a/l10ntools/Executable_xrmex.mk b/l10ntools/Executable_xrmex.mk
index 33255f4eb7e1..b86dbc4b80d2 100644
--- a/l10ntools/Executable_xrmex.mk
+++ b/l10ntools/Executable_xrmex.mk
@@ -33,6 +33,7 @@ $(eval $(call gb_Executable_set_include,xrmex,\
 
 $(eval $(call gb_Executable_use_libraries,xrmex,\
     sal \
+    i18nregexp \
 ))
 
 $(eval $(call gb_Executable_add_scanners,xrmex,\
diff --git a/l10ntools/inc/po.hxx b/l10ntools/inc/po.hxx
index b4b18c49d980..81bafe7b7f10 100644
--- a/l10ntools/inc/po.hxx
+++ b/l10ntools/inc/po.hxx
@@ -25,6 +25,7 @@ private:
     OString    m_sUnTransStr;
     OString    m_sTransStr;
     bool       m_bFuzzy;
+    bool       m_bNull;
     OString    m_sKeyId;
 
 public:
@@ -38,6 +39,8 @@ public:
     virtual OString     getUnTransStr() const   { return m_sUnTransStr; }
     virtual OString     getTransStr() const     { return m_sTransStr; }
     virtual bool        getFuzzy() const        { return m_bFuzzy; }
+    virtual bool        isNull() const          { return m_bNull; }
+    virtual OString     getKeyId() const        { return m_sKeyId; }
 
     virtual void        setWhiteSpace(const OString& rWhiteSpace);
     virtual void        setExtractCom(const OString& rExtractCom);
@@ -68,6 +71,8 @@ private:
     OString     m_sResourceType;
     TYPE        m_eType;
     OString     m_sHelpText;
+
+    void            SetMembers();
 public:
 
                     PoEntry();
@@ -84,6 +89,8 @@ public:
     OString         getUnTransStr() const;
     OString         getTransStr() const;
     bool            getFuzzy() const            { return m_aGenPo.getFuzzy(); }
+    bool            isNull() const              { return m_aGenPo.isNull(); }
+    OString         getKeyId() const            { return m_aGenPo.getKeyId(); }
     void            setUnTransStr(const OString& rUnTransStr);
     void            setTransStr(const OString& rTransStr);
     void            setFuzzy(const bool bFuzzy);
@@ -91,6 +98,8 @@ public:
     void            writeToFile(std::ofstream& rOFStream);
     void            readFromFile(std::ifstream& rIFStream);
 
+    static bool     IsInSameComp(const PoEntry& rPo1,const PoEntry& rPo2);
+
 };
 
 class PoHeader
@@ -105,18 +114,23 @@ private:
     OString    m_sPoRevisionDate;
     OString    m_sLastTranslator;
     OString    m_sLanguageTeam;
+    OString    m_sLanguage;
     OString    m_sMimeVersion;
-    OString    m_sPluralForms;
     OString    m_sContentType;
-    OString    m_sCharset;
     OString    m_sEncoding;
+    OString    m_sPluralForms;
     OString    m_sXGenerator;
     OString    m_sXAcceleratorMarker;
 
+    void            SetMembers();
 public:
+                    PoHeader();
                     PoHeader( const OString& rExtSrc );
                     ~PoHeader();
+
+    OString         getLanguage() const { return m_sLanguage; }
     void            writeToFile(std::ofstream& rOFStream);
+    void            readFromFile(std::ifstream& rIFStream);
 };
 
 #endif // _PO_INCLUDED
diff --git a/l10ntools/source/merge.cxx b/l10ntools/source/merge.cxx
index 5da52c57d640..a1c4570056e6 100644
--- a/l10ntools/source/merge.cxx
+++ b/l10ntools/source/merge.cxx
@@ -34,6 +34,7 @@
 #include <vector>
 
 #include "export.hxx"
+#include "po.hxx"
 
 namespace
 {
@@ -142,42 +143,95 @@ MergeDataFile::MergeDataFile(
     bool bCaseSensitive)
 {
     std::ifstream aInputStream(rFileName.getStr());
-    const ::rtl::OString sHACK(RTL_CONSTASCII_STRINGPARAM("HACK"));
-    const ::rtl::OString sFileNormalized(lcl_NormalizeFilename(rFile));
-    const bool isFileEmpty = !sFileNormalized.isEmpty();
-
     if (!aInputStream.is_open())
     {
-        printf("Warning : Can't open %s\n", rFileName.getStr());
+        printf("Warning : Can't open po path container file");
         return;
     }
-    while (!aInputStream.eof())
+    std::string sPoFileName;
+    aInputStream >> sPoFileName;
+    bool bFirstLang = true;
+    while(!aInputStream.eof())
     {
-        std::string buf;
-        std::getline(aInputStream, buf);
-        rtl::OString sLine(buf.data(), buf.length());
-        sal_Int32 n = 0;
-        // Skip all wrong filenames
-        const ::rtl::OString filename = lcl_NormalizeFilename(sLine.getToken(1, '\t', n)); // token 1
-        if(isFileEmpty || sFileNormalized.equals("") || (!isFileEmpty && filename.equals(sFileNormalized)) )
+        std::ifstream aPoFile(sPoFileName.c_str());
+        const OString sHACK("HACK");
+        const OString sFileName(lcl_NormalizeFilename(rFile));
+
+        if (!aPoFile.is_open())
         {
-            const rtl::OString sTYP = sLine.getToken( 1, '\t', n ); // token 3
-            const rtl::OString sGID = sLine.getToken( 0, '\t', n ); // token 4
-            const rtl::OString sLID = sLine.getToken( 0, '\t', n ); // token 5
-            rtl::OString sPFO = sLine.getToken( 1, '\t', n ); // token 7
-            sPFO = sHACK;
-            rtl::OString nLANG = sLine.getToken( 1, '\t', n ); // token 9
-            nLANG = nLANG.trim();
-            const rtl::OString sTEXT = sLine.getToken( 0, '\t', n ); // token 10
-            const rtl::OString sQHTEXT = sLine.getToken( 1, '\t', n ); // token 12
-            const rtl::OString sTITLE = sLine.getToken( 0, '\t', n ); // token 13
-
-            if (!nLANG.equalsIgnoreAsciiCaseL(RTL_CONSTASCII_STRINGPARAM("en-US")))
+            printf("Warning : Can't open %s\n", sPoFileName.c_str());
+            return;
+        }
+
+        PoHeader aPoHeader;
+        aPoHeader.readFromFile(aPoFile);
+        const OString nLANG = aPoHeader.getLanguage();
+        aLanguageSet.insert(nLANG);
+        PoEntry aNextPo;
+        do
+        {
+            aNextPo.readFromFile(aPoFile);
+        } while( !aNextPo.isNull() && aNextPo.getSourceFile() != sFileName );
+        while( !aNextPo.isNull() && aNextPo.getSourceFile() == sFileName )
+        {
+            PoEntry aActPo(aNextPo);
+
+            bool bInSameComp = false;
+            OString sText;
+            OString sQHText;
+            OString sTitle;
+            OString sExText;
+            OString sExQHText;
+            OString sExTitle;
+            OString sQTZText;
+            OString sQTZQHText;
+            OString sQTZTitle;
+            do
+            {
+                if( bInSameComp )
+                    aActPo = PoEntry(aNextPo);
+                OString sTemp = aActPo.getTransStr();
+                if( aActPo.getFuzzy() || sTemp.isEmpty() )
+                    sTemp = aActPo.getUnTransStr();
+                switch( aActPo.getType() )
+                {
+                    case PoEntry::TTEXT:
+                        sText = sTemp;
+                        sExText = aActPo.getUnTransStr();
+                        sQTZText = aActPo.getKeyId();
+                        break;
+                    case PoEntry::TQUICKHELPTEXT:
+                        sQHText = sTemp;
+                        sExQHText = aActPo.getUnTransStr();
+                        sQTZQHText = aActPo.getKeyId();
+                        break;
+                    case PoEntry::TTITLE:
+                        sTitle = sTemp;
+                        sExTitle = aActPo.getUnTransStr();
+                        sQTZTitle = aActPo.getKeyId();
+                        break;
+                }
+                aNextPo.readFromFile(aPoFile);
+            } while(!aNextPo.isNull() &&
+                    (bInSameComp = PoEntry::IsInSameComp(aActPo,aNextPo)));
+
+            InsertEntry( aActPo.getResourceType(), aActPo.getGroupId(),
+                         aActPo.getLocalId(), sHACK, nLANG, sText,
+                         sQHText, sTitle, sFileName, bCaseSensitive );
+            if( bFirstLang )
             {
-                aLanguageSet.insert(nLANG);
-                InsertEntry( sTYP, sGID, sLID, sPFO, nLANG, sTEXT, sQHTEXT, sTITLE, filename, bCaseSensitive );
+                aLanguageSet.insert("qtz");
+                InsertEntry( aActPo.getResourceType(), aActPo.getGroupId(),
+                             aActPo.getLocalId(), sHACK, "qtz",
+                             sQTZText + "‖" + sExText,
+                             sQTZQHText + "‖" + sExQHText,
+                             sQTZTitle + "‖" + sExTitle,
+                             sFileName, bCaseSensitive );
             }
         }
+        aPoFile.close();
+        aInputStream >> sPoFileName;
+        bFirstLang = false;
     }
     aInputStream.close();
 }
diff --git a/l10ntools/source/po.cxx b/l10ntools/source/po.cxx
index 1a7c5aead7df..3be13fcc7361 100644
--- a/l10ntools/source/po.cxx
+++ b/l10ntools/source/po.cxx
@@ -32,11 +32,12 @@ OString ImplGenKeyId(const OString& rGenerator)
     boost::crc_32_type aCRC32;
     aCRC32.process_bytes(rGenerator.getStr(), rGenerator.getLength());
     sal_uInt32 nCRC = aCRC32.checksum();
+    //Use all readable ASCII charachter exclude xml special tags: ",',&,<,>
+    const OString sSymbols = "!#$%()*+,-./0123456789:;=?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~";
     char sKeyId[5];
-    for(int nIndex = 0; nIndex < 4; ++nIndex)
+    for( short nKeyInd = 0; nKeyInd < 4; ++nKeyInd )
     {
-        //Get a char from the [33,126] interval of ASCII
-        sKeyId[nIndex] = static_cast<char>((nCRC & 255) % 93 + 33);
+        sKeyId[nKeyInd] = sSymbols[(nCRC & 255) % 89];
         nCRC >>= 8;
     }
     sKeyId[4] = '\0';
@@ -109,7 +110,7 @@ OString ImplGenNormString(const OString& rString)
 //Decide whether a string starts with an other string
 bool ImplStartsWith(const OString& rString,const OString& rStart)
 {
-    return rString.copy(0,rStart.getLength())==rStart;
+    return rString.match(rStart);
 }
 
 //Default constructor
@@ -121,6 +122,7 @@ GenPoEntry::GenPoEntry()
     , m_sUnTransStr( OString() )
     , m_sTransStr( OString() )
     , m_bFuzzy( false )
+    , m_bNull( false )
     , m_sKeyId( OString() )
 {
 }
@@ -197,6 +199,12 @@ void GenPoEntry::writeToFile(std::ofstream& rOFStream)
 //Read from file
 void GenPoEntry::readFromFile(std::ifstream& rIFStream)
 {
+    *this = GenPoEntry();
+    if( rIFStream.eof() )
+    {
+        m_bNull = true;
+        return;
+    }
     m_sWhiteSpace = "\n";
     OString* pLastMsg = 0;
     std::string sTemp;
@@ -412,6 +420,34 @@ PoEntry::~PoEntry()
 {
 }
 
+//Set members on the basis of m_aGenPo
+void PoEntry::SetMembers()
+{
+    if( !m_aGenPo.isNull() )
+    {
+        m_sSourceFile = m_aGenPo.getReference();
+        OString sContext =  m_aGenPo.getContext();
+        m_sGroupId = sContext.getToken(0,'\n');
+
+        if (sContext.indexOf('\n')==sContext.lastIndexOf('\n'))
+            m_sResourceType = sContext.getToken(1,'\n').getToken(0,'.');
+        else
+        {
+            m_sLocalId = sContext.getToken(1,'\n');
+            m_sResourceType = sContext.getToken(2,'\n').getToken(0,'.');
+        }
+        if (sContext.endsWith(".text"))
+            m_eType = TTEXT;
+        else if (sContext.endsWith(".quickhelptext"))
+            m_eType = TQUICKHELPTEXT;
+        else if (sContext.endsWith(".title"))
+            m_eType = TTITLE;
+        else{
+            throw;}
+        m_sHelpText = m_aGenPo.getExtractCom();
+    }
+}
+
 //Get translation string in sdf/merge format
 OString PoEntry::getUnTransStr() const
 {
@@ -478,29 +514,18 @@ void PoEntry::writeToFile(std::ofstream& rOFStream)
 //Read from file
 void PoEntry::readFromFile(std::ifstream& rIFStream)
 {
+    *this = PoEntry();
     m_aGenPo.readFromFile(rIFStream);
-    m_sSourceFile = m_aGenPo.getReference();
-
-    OString sContext =  m_aGenPo.getContext();
-    m_sGroupId = sContext.getToken(0,'\n');
-
-    if (sContext.indexOf('\n')==sContext.lastIndexOf('\n'))
-        m_sResourceType = sContext.getToken(1,'\n').getToken(0,'.');
-    else
-    {
-        m_sLocalId = sContext.getToken(1,'\n');
-        m_sResourceType = sContext.getToken(2,'\n').getToken(0,'.');
-    }
-    if (sContext.endsWith(".text"))
-        m_eType = TTEXT;
-    else if (sContext.endsWith(".quickhelptext"))
-        m_eType = TQUICKHELPTEXT;
-    else if (sContext.endsWith(".title"))
-        m_eType = TTITLE;
-    else
-        throw;
+    SetMembers();
+}
 
-    m_sHelpText = m_aGenPo.getExtractCom();
+//Check whether po-s belong to the same localization component
+bool PoEntry::IsInSameComp(const PoEntry& rPo1,const PoEntry& rPo2)
+{
+    return ( rPo1.m_sSourceFile == rPo2.m_sSourceFile &&
+             rPo1.m_sGroupId == rPo2.m_sGroupId &&
+             rPo1.m_sLocalId == rPo2.m_sLocalId &&
+             rPo1.m_sResourceType == rPo2.m_sResourceType );
 }
 
 //Class PoHeader
@@ -515,7 +540,35 @@ OString ImplGetTime()
     return pBuff;
 }
 
-//Constructor
+//Get relevant part of actual token
+OString ImplGetElement(const OString& rText, const sal_Int32 nToken)
+{
+    OString sToken = rText.getToken(nToken,'\n');
+    sal_Int32 nFirstIndex = sToken.indexOf(':') + 2;
+    return sToken.copy( nFirstIndex,sToken.getLength()-nFirstIndex );
+}
+
+//Default Constructor
+PoHeader::PoHeader()
+    : m_aGenPo( GenPoEntry() )
+    , m_sExtractionSource( OString() )
+    , m_sProjectIdVersion( OString() )
+    , m_sReportMsgidBugsTo( OString() )
+    , m_sPotCreationDate( OString() )
+    , m_sPoRevisionDate( OString() )
+    , m_sLastTranslator( OString() )
+    , m_sLanguageTeam( OString() )
+    , m_sLanguage( OString() )
+    , m_sMimeVersion( OString() )
+    , m_sContentType( OString() )
+    , m_sEncoding( OString() )
+    , m_sPluralForms( OString() )
+    , m_sXGenerator( OString() )
+    , m_sXAcceleratorMarker( OString() )
+{
+}
+
+//Template Constructor
 PoHeader::PoHeader( const OString& rExtSrc )
     : m_aGenPo( GenPoEntry() )
     , m_sExtractionSource( rExtSrc )
@@ -527,36 +580,74 @@ PoHeader::PoHeader( const OString& rExtSrc )
     , m_sPoRevisionDate( "YEAR-MO-DA HO:MI+ZONE" )
     , m_sLastTranslator( "FULL NAME <EMAIL@ADDRESS>" )
     , m_sLanguageTeam( "LANGUAGE <LL@li.org>" )
+    , m_sLanguage( OString() )
     , m_sMimeVersion( "1.0" )
-    , m_sContentType( "text/plain" )
-    , m_sCharset( "UTF-8" )
+    , m_sContentType( "text/plain; charset=UTF-8" )
     , m_sEncoding( "8bit" )
+    , m_sPluralForms( OString() )
     , m_sXGenerator( "LibreOffice" )
     , m_sXAcceleratorMarker( "~" )
 {
-    m_aGenPo.setExtractCom("extracted from " + rExtSrc);
-    m_aGenPo.setTransStr(
-            "Project-Id-Version: " + m_sProjectIdVersion + "\n" +
-            "Report-Msgid-Bugs-To: " + m_sReportMsgidBugsTo + "\n" +
-            "POT-Creation-Date: " + m_sPotCreationDate + "\n" +
-            "PO-Revision-Date: " + m_sPoRevisionDate + "\n" +
-            "Last-Translator: " + m_sLastTranslator + "\n" +
-            "Language-Team: " + m_sLanguageTeam + "\n" +
-            "MIME-Version: " + m_sMimeVersion + "\n" +
-            "Content-Type: " + m_sContentType + "; " +
-            "charset=" + m_sCharset + "\n" +
-            "Content-Transfer-Encoding: " + m_sEncoding + "\n" +
-            "X-Genarator: " + m_sXGenerator + "\n" +
-            "X-Accelerator_Marker: " + m_sXAcceleratorMarker + "\n");
 }
 
 PoHeader::~PoHeader()
 {
 }
 
+//Set members on the basis of m_aGenPo
+void PoHeader::SetMembers()
+{
+    if( !m_aGenPo.isNull() )
+    {
+        m_sExtractionSource = m_aGenPo.getExtractCom();
+        OString sTemp = m_aGenPo.getTransStr();
+
+        sal_Int32 nToken = 0;
+        m_sProjectIdVersion = ImplGetElement(sTemp,nToken++);
+        m_sReportMsgidBugsTo = ImplGetElement(sTemp,nToken++);
+        m_sPotCreationDate = ImplGetElement(sTemp,nToken++);
+        m_sPoRevisionDate = ImplGetElement(sTemp,nToken++);
+        m_sLastTranslator = ImplGetElement(sTemp,nToken++);
+        m_sLanguageTeam = ImplGetElement(sTemp,nToken++);
+        if( sTemp.getToken(nToken,'\n').match("Language:") )
+            m_sLanguage = ImplGetElement(sTemp,nToken++);
+        m_sMimeVersion = ImplGetElement(sTemp,nToken++);
+        m_sContentType = ImplGetElement(sTemp,nToken++);
+        m_sEncoding = ImplGetElement(sTemp,nToken++);
+        if( sTemp.getToken(nToken,'\n').match("Plural-Forms:") )
+            m_sPluralForms = ImplGetElement(sTemp,nToken++);
+        m_sXGenerator = ImplGetElement(sTemp,nToken++);
+        m_sXAcceleratorMarker = ImplGetElement(sTemp,nToken);
+    }
+}
+
 void PoHeader::writeToFile(std::ofstream& rOFStream)
 {
+    m_aGenPo.setExtractCom("extracted from " + m_sExtractionSource);
+    m_aGenPo.setTransStr(
+        "Project-Id-Version: " + m_sProjectIdVersion + "\n" +
+        "Report-Msgid-Bugs-To: " + m_sReportMsgidBugsTo + "\n" +
+        "POT-Creation-Date: " + m_sPotCreationDate + "\n" +
+        "PO-Revision-Date: " + m_sPoRevisionDate + "\n" +
+        "Last-Translator: " + m_sLastTranslator + "\n" +
+        "Language-Team: " + m_sLanguageTeam + "\n" +
+        ( m_sLanguage.isEmpty() ? "" : "Language: " + m_sLanguage + "\n" ) +
+        "MIME-Version: " + m_sMimeVersion + "\n" +
+        "Content-Type: " + m_sContentType + "\n" +
+        "Content-Transfer-Encoding: " + m_sEncoding + "\n" +
+        ( m_sPluralForms.isEmpty() ? "" :
+            "Plural-Forms: " + m_sPluralForms + "\n" ) +
+        "X-Genarator: " + m_sXGenerator + "\n" +
+        "X-Accelerator_Marker: " + m_sXAcceleratorMarker + "\n");
     m_aGenPo.writeToFile(rOFStream);
 }
 
+void PoHeader::readFromFile(std::ifstream& rIFStream)
+{
+    *this = PoHeader();
+    m_aGenPo.readFromFile(rIFStream);
+    SetMembers();
+}
+
+
 /* vim:set shiftwidth=4 softtabstop=4 expandtab: */
diff --git a/l10ntools/source/renewpo.cxx b/l10ntools/source/renewpo.cxx
index 7e567f07d52e..3e4e98f34c02 100644
--- a/l10ntools/source/renewpo.cxx
+++ b/l10ntools/source/renewpo.cxx
@@ -124,15 +124,15 @@ void HandleLanguage(struct dirent* pLangEntry, const OString& rPath,
         {
             if (!sActUnTrans.getToken(vTypes[nIndex],'\t').isEmpty())
             {
-                /**Because of xrmex lexer there are duplicated id's,
-                   only use this if the lexer have already fixed*/
+                /**Because of xrmex there are duplicated id's,
+                   only use this if xrmex have already fixed
                 if (sActUnTrans.getToken(PoEntry::GROUPID,'\t')==
                     sActUnTrans.getToken(PoEntry::LOCALID,'\t') &&
                     sActUnTrans.getToken(PoEntry::SOURCEFILE,'\t').
                                 endsWith(".xrm"))
                 {
                     sActUnTrans = DelLocalId(sActUnTrans);
-                }
+                }*/
                 PoEntry aPE(sActUnTrans, vTypes[nIndex]);
                 aPE.setTransStr(sActTrans.getToken(vTypes[nIndex],'\t'));
                 aPE.setFuzzy(sActTrans.isEmpty() ? 0 :