MWS_SRX644: migrate branch mws_srx644 -> HEAD

11 files changed, 318 insertions, 100 deletions

diff --git a/transex3/inc/export.hxx b/transex3/inc/export.hxx
index 7e66756e4a89..a38a3729bf9a 100644
--- a/transex3/inc/export.hxx
+++ b/transex3/inc/export.hxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: export.hxx,v $
  *
- *  $Revision: 1.4 $
+ *  $Revision: 1.5 $
  *
- *  last change: $Author: nf $ $Date: 2002-11-18 11:49:03 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:43 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -195,11 +195,15 @@
 #define HINDI_ISO               "hi-IN"
 #define HINDI_INDEX                 30
 
+#define ESTONIAN                    77
+#define ESTONIAN_ISO                "et"
+#define ESTONIAN_INDEX                  31
+
 // special language for l10n framework
 #define EXTERN                  99
-#define EXTERN_INDEX                31
+#define EXTERN_INDEX                32
 
-#define LANGUAGES                   32
+#define LANGUAGES                   33
 
 #define LANGUAGE_ALLOWED( n )  (( n != 0xFFFF ) && ( Export::LanguageAllowed( Export::LangId[ n ] )) &&                                         \
                                (( Export::LangId[ n ] == 01 ) || ( Export::LangId[ n ] == 03 ) || ( Export::LangId[ n ] == 07 ) ||  \
@@ -209,8 +213,9 @@
                                 ( Export::LangId[ n ] == 48 ) || ( Export::LangId[ n ] == 49 ) || ( Export::LangId[ n ] == 55 ) ||  \
                                 ( Export::LangId[ n ] == 81 ) || ( Export::LangId[ n ] == 82 ) || ( Export::LangId[ n ] == 86 ) ||  \
                                 ( Export::LangId[ n ] == 88 ) || ( Export::LangId[ n ] == 90 ) || ( Export::LangId[ n ] == 96 ) ||  \
+                                ( Export::LangId[ n ] == 42 ) || ( Export::LangId[ n ] == 43 ) || ( Export::LangId[ n ] == 97 ) ||  \
                                 ( Export::LangId[ n ] == 66 ) || ( Export::LangId[ n ] == 91 ) || ( Export::LangId[ n ] == 00 ) ||  \
-                                ( Export::LangId[ n ] == 99 )))
+                                ( Export::LangId[ n ] == 99 ) || ( Export::LangId[ n ] == 77 )))
 
 #define GERMAN_LIST_LINE_INDEX  LANGUAGES
 #define LIST_REFID              (GERMAN_LIST_LINE_INDEX+1)
diff --git a/transex3/inc/tagtest.hxx b/transex3/inc/tagtest.hxx
index ac8caf31b2fa..92ebad370c4f 100644
--- a/transex3/inc/tagtest.hxx
+++ b/transex3/inc/tagtest.hxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: tagtest.hxx,v $
  *
- *  $Revision: 1.3 $
+ *  $Revision: 1.4 $
  *
- *  last change: $Author: gh $ $Date: 2003-03-14 13:13:21 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:43 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
diff --git a/transex3/prj/d.lst b/transex3/prj/d.lst
index 4ccb3ba31697..f88cbe04eaa8 100644
--- a/transex3/prj/d.lst
+++ b/transex3/prj/d.lst
@@ -10,6 +10,8 @@ mkdir: %_DEST%\inc%_EXT%\transex3
 ..\%__SRC%\bin\localize %_DEST%\bin%_EXT%\localize
 ..\%__SRC%\bin\xrmex.exe %_DEST%\bin%_EXT%\xrmex.exe
 ..\%__SRC%\bin\xrmex %_DEST%\bin%_EXT%\xrmex
+..\%__SRC%\bin\gsicheck.exe %_DEST%\bin%_EXT%\gsicheck.exe
+..\%__SRC%\bin\gsicheck %_DEST%\bin%_EXT%\gsicheck
 
 ..\%__SRC%\lib\lib*static*.dylib %_DEST%\lib%_EXT%\lib*static*.dylib
 ..\%__SRC%\misc\*staticdatamembers.cxx %_DEST%\inc%_EXT%\*staticdatamembers.cxx
diff --git a/transex3/source/cfglex.l b/transex3/source/cfglex.l
index e16dc480bc67..b5066371deb3 100644
--- a/transex3/source/cfglex.l
+++ b/transex3/source/cfglex.l
@@ -1,6 +1,6 @@
 %{
 /*
- * lexer for parsing cfg source files 
+ * lexer for parsing cfg source files
  *
  */
 
@@ -81,7 +81,7 @@ int bText=0;
 		c2 = c3;
 		c3 = input();
 		pChar[0] = c3;
-		WorkOnTokenSet( COMMEND, pChar );                          
+		WorkOnTokenSet( COMMEND, pChar );
 	}
 }
 
@@ -156,7 +156,7 @@ main( int argc, char* argv[])
 
 	pOutput = GetOutputFile( argc, argv );
 	if ( !pOutput ) {
-		fprintf( stdout, "Syntax: CFGEX[-p Prj][-r PrjRoot]-i FileIn [-o FileOut][-m DataBase][-e][-b][-u][-d DoneFile][-g[:dtd]][-NOUTF8][-L l1,l2,...][-ISO99 IsoCode]\n" );
+		fprintf( stdout, "Syntax: CFGEX[-p Prj][-r PrjRoot]-i FileIn [-o FileOut][-m DataBase][-e][-b][-u][-f][-d DoneFile][-g[:dtd]][-NOUTF8][-L l1,l2,...][-ISO99 IsoCode]\n" );
 		fprintf( stdout, " Prj:      Project\n" );
 		fprintf( stdout, " PrjRoot:  Path to project root (..\\.. etc.)\n" );
 		fprintf( stdout, " FileIn:   Source files (*.src)\n" );
@@ -165,6 +165,7 @@ main( int argc, char* argv[])
 		fprintf( stdout, " -e: Disable writing errorlog\n" );
 		fprintf( stdout, " -b: Break when Token \"HelpText\" found in source\n" );
 		fprintf( stdout, " -u: [english] and [german] are allowed, Id is Taken from DataBase \n" );
+		fprintf( stdout, " -f: force extraction and merge even if only one language is existent\n" );
 		fprintf( stdout, " -g[:dtd]: enables generation of properties (dtds if :dtd is set) - in this case FileOut is the output path\n" );
 		fprintf( stdout, " -d: enables generation of *.don if work is done\n" );
 		fprintf( stdout, " -NOUTF8: disable UTF8 as language independent encoding\n" );
diff --git a/transex3/source/cfgmerge.cxx b/transex3/source/cfgmerge.cxx
index 0ae727da0d6e..7df3147044b4 100644
--- a/transex3/source/cfgmerge.cxx
+++ b/transex3/source/cfgmerge.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: cfgmerge.cxx,v $
  *
- *  $Revision: 1.23 $
+ *  $Revision: 1.24 $
  *
- *  last change: $Author: er $ $Date: 2002-12-12 17:00:09 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:45 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -82,11 +82,13 @@ extern "C" { YYWarning( char * ); }
 #define STATE_UTF8      0x0008
 #define STATE_LANGUAGES 0X0009
 #define STATE_ISOCODE99 0x000A
+#define STATE_FORCE     0x000B
 
 // set of global variables
 BOOL bEnableExport;
 BOOL bMergeMode;
 BOOL bErrorLog;
+BOOL bForce;
 BOOL bUTF8;
 ByteString sPrj;
 ByteString sPrjRoot;
@@ -107,6 +109,7 @@ extern char *GetOutputFile( int argc, char* argv[])
     bEnableExport = FALSE;
     bMergeMode = FALSE;
     bErrorLog = TRUE;
+    bForce = FALSE;
     bUTF8 = TRUE;
     sPrj = "";
     sPrjRoot = "";
@@ -146,6 +149,10 @@ extern char *GetOutputFile( int argc, char* argv[])
             nState = STATE_UTF8;
             bUTF8 = FALSE;
         }
+        else if ( ByteString( argv[ i ] ).ToUpperAscii() == "-F" ) {
+            nState = STATE_FORCE;
+            bForce = TRUE;
+        }
         else if ( ByteString( argv[ i ] ).ToUpperAscii() == "-L" ) {
             nState = STATE_LANGUAGES;
         }
@@ -478,6 +485,12 @@ int CfgParser::ExecuteAnalyzedToken( int nToken, char *pToken )
                         bLocalize = TRUE;
                     break;
                     case CFG_TEXT_START: {
+                        if ( sCurrentResTyp != sTokenName ) {
+                            WorkOnRessourceEnd();
+                            for ( ULONG i = 0; i < LANGUAGES; i++ )
+                                if ( LANGUAGE_ALLOWED( i ))
+                                    pStackData->sText[ i ] = "";
+                        }
                         sCurrentResTyp = sTokenName;
 
                         ByteString sTemp = sToken.Copy( sToken.Search( "xml:lang=" ));
@@ -639,9 +652,14 @@ void CfgExport::WorkOnRessourceEnd()
 /*****************************************************************************/
 {
     if ( pOutputStream && bLocalize ) {
-        if ( pStackData->sText[ GERMAN_INDEX ].Len() &&
+        if (( pStackData->sText[ GERMAN_INDEX ].Len() &&
             ( pStackData->sText[ ENGLISH_US_INDEX ].Len() ||
-                pStackData->sText[ ENGLISH_INDEX ].Len())
+                pStackData->sText[ ENGLISH_INDEX ].Len())) ||
+            ( bForce &&
+                ( pStackData->sText[ GERMAN_INDEX ].Len() ||
+                    pStackData->sText[ ENGLISH_INDEX ].Len() ||
+                    pStackData->sText[ ENGLISH_US_INDEX ].Len())
+                )
             )
         {
             pStackData->FillInFallbacks();
@@ -795,7 +813,7 @@ void CfgMerge::Output( const ByteString& rOutput )
 void CfgMerge::WorkOnRessourceEnd()
 /*****************************************************************************/
 {
-    if ( pMergeDataFile && pResData && bLocalize && bGerman && bEnglish ) {
+    if ( pMergeDataFile && pResData && bLocalize && (( bGerman && bEnglish ) || bForce )) {
         PFormEntrys *pEntrys = pMergeDataFile->GetPFormEntrys( pResData );
         if ( pEntrys ) {
             for ( ULONG nIndex = 0; nIndex < LANGUAGES; nIndex++ ) {
diff --git a/transex3/source/export.cxx b/transex3/source/export.cxx
index b34e2b57f7fd..3212fa46ec57 100644
--- a/transex3/source/export.cxx
+++ b/transex3/source/export.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: export.cxx,v $
  *
- *  $Revision: 1.30 $
+ *  $Revision: 1.31 $
  *
- *  last change: $Author: er $ $Date: 2002-12-12 17:00:09 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:45 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -1223,6 +1223,8 @@ USHORT Export::GetLangIndex( const ByteString &rLang )
         return CZECH_INDEX;
     else if ( sLang == "SLOVAK" )
         return SLOVAK_INDEX;
+    else if ( sLang == "ESTONIAN" )
+        return ESTONIAN_INDEX;
     else if ( sLang == "ENGLISH" )
         return ENGLISH_INDEX;
     else if ( sLang == "DANISH" )
@@ -1255,6 +1257,8 @@ USHORT Export::GetLangIndex( const ByteString &rLang )
         return CATALAN_INDEX;
     else if ( sLang == "THAI" )
         return THAI_INDEX;
+    else if ( sLang == "HINDI" )
+        return HINDI_INDEX;
     else if ( sLang == "EXTERN" )
         return EXTERN_INDEX;
 
diff --git a/transex3/source/export2.cxx b/transex3/source/export2.cxx
index 703d22d38fda..c45196d05983 100644
--- a/transex3/source/export2.cxx
+++ b/transex3/source/export2.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: export2.cxx,v $
  *
- *  $Revision: 1.21 $
+ *  $Revision: 1.22 $
  *
- *  last change: $Author: nf $ $Date: 2002-11-18 11:49:56 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:46 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -147,6 +147,7 @@ USHORT Export::LangId[ LANGUAGES ] =
     CATALAN,
     THAI,
     HINDI,
+    ESTONIAN,
     EXTERN
 };
 
@@ -236,6 +237,8 @@ USHORT Export::GetLangByIsoLang( const ByteString &rIsoLang )
         return THAI;
     else if ( sLang == ByteString( HINDI_ISO ).ToUpperAscii())
         return HINDI;
+    else if ( sLang == ByteString( ESTONIAN_ISO ).ToUpperAscii())
+        return ESTONIAN;
     else if ( sLang == ByteString( sIsoCode99 ).ToUpperAscii())
         return EXTERN;
 
@@ -277,6 +280,7 @@ ByteString Export::GetIsoLangByIndex( USHORT nIndex )
         case CATALAN_INDEX: return CATALAN_ISO;
         case THAI_INDEX: return THAI_ISO;
         case HINDI_INDEX: return HINDI_ISO;
+        case ESTONIAN_INDEX: return ESTONIAN_ISO;
         case EXTERN_INDEX: return sIsoCode99;
     }
     return "";
@@ -404,6 +408,7 @@ const ByteString Export::LangName[ LANGUAGES ] =
     "catalan",
     "thai",
     "hindi",
+    "estonian",
     "extern"
 };
 
diff --git a/transex3/source/gsicheck.cxx b/transex3/source/gsicheck.cxx
index 29e3514e955d..7cc86302422b 100644
--- a/transex3/source/gsicheck.cxx
+++ b/transex3/source/gsicheck.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: gsicheck.cxx,v $
  *
- *  $Revision: 1.7 $
+ *  $Revision: 1.8 $
  *
- *  last change: $Author: gh $ $Date: 2003-02-05 16:12:56 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:46 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -69,21 +69,36 @@
 //
 // class GSILine
 //
+enum LineFormat { FORMAT_GSI1, FORMAT_SDF, FORMAT_UNKNOWN };
 
 class GSILine : public ByteString
 {
 private:
-    ByteString aUniqId;
+    LineFormat aFormat;
     ULONG nLineNumber;
+
+    ByteString aUniqId;
+    ByteString aLineType;
+    USHORT nLangId;
+    ByteString aStatus;
+    ByteString aText;
+
     BOOL bOK;
 
+
 public:
     GSILine( const ByteString &rLine, ULONG nLine );
 
-    ByteString GetUniqId() { return aUniqId; }
-    ULONG GetLineNumber() { return nLineNumber; }
+    LineFormat  const GetLineFormat()     { return aFormat; }
+    ULONG       const GetLineNumber()     { return nLineNumber; }
+
+    ByteString  const GetUniqId()         { return aUniqId; }
+    ByteString  const GetLineType()       { return aLineType; }
+    USHORT      const GetLanguageId()     { return nLangId; }
+    ByteString  const GetLineStatus()     { return aStatus; }
+    ByteString  const GetText()           { return aText; }
 
-    BOOL IsOK() { return bOK; }
+    BOOL const IsOK() { return bOK; }
     void NotOK() { bOK = FALSE; }
 };
 
@@ -97,21 +112,45 @@ class GSIBlock : public GSIBlock_Impl
 {
 private:
     GSILine *pSourceLine;
+    GSILine *pReferenceLine;
     void PrintList( ParserMessageList &rList, ByteString aPrefix, GSILine *pLine );
     BOOL bPrintContext;
     BOOL bInternal;
+    BOOL bReference;
+
+    BOOL TestUTF8( GSILine* pTestee );
 
 public:
-    GSIBlock( BOOL PbPrintContext, BOOL bInt ) : pSourceLine( NULL ), bPrintContext( PbPrintContext ), bInternal( bInt ) {};
+    GSIBlock( BOOL PbPrintContext, BOOL bInt, BOOL bRef ) : pSourceLine( NULL ), pReferenceLine( NULL ), bPrintContext( PbPrintContext ), bInternal( bInt ), bReference( bRef ) {};
     ~GSIBlock();
     void PrintError( ByteString aMsg, ByteString aPrefix, ByteString aContext, ULONG nLine, ByteString aUniqueId = ByteString() );
-    void InsertLine( const ByteString &rLine, ULONG nLine , const USHORT nSourceLang);
+    void InsertLine( GSILine* pLine, const USHORT nSourceLang);
+    void SetReferenceLine( GSILine* pLine );
     BOOL CheckSyntax( ULONG nLine );
 
     void WriteError( SvStream &aErrOut );
     void WriteCorrect( SvStream &aOkOut );
 };
 
+
+
+/*****************************************************************************/
+void PrintError( ByteString aMsg, ByteString aPrefix,
+    ByteString aContext, BOOL bPrintContext, ULONG nLine, ByteString aUniqueId = ByteString() )
+/*****************************************************************************/
+{
+    fprintf( stdout, "Error: %s, Line %lu", aPrefix.GetBuffer(), nLine );
+    if ( aUniqueId.Len() )
+        fprintf( stdout, ", UniqueID %s", aUniqueId.GetBuffer() );
+    fprintf( stdout, ": %s", aMsg.GetBuffer() );
+
+    if ( bPrintContext )
+        fprintf( stdout, "  \"%s\"", aContext.GetBuffer() );
+    fprintf( stdout, "\n" );
+}
+
+
+
 //
 // class GSILine
 //
@@ -123,10 +162,36 @@ GSILine::GSILine( const ByteString &rLine, ULONG nLine )
                 , nLineNumber( nLine )
                 , bOK( TRUE )
 {
-    ByteString sTmp( rLine );
-    sTmp.SearchAndReplaceAll( "($$)", "\t" );
+       if ( rLine.GetTokenCount( '\t' ) == 1 )
+    {
+        ByteString sTmp( rLine );
+        sTmp.SearchAndReplaceAll( "($$)", "\t" );
+        if ( sTmp.GetTokenCount( '\t' ) == 5 )
+        {
+            aFormat = FORMAT_GSI1;
+            aUniqId = sTmp.GetToken( 0, '\t' );
+            aLineType = sTmp.GetToken( 1, '\t' );
+            nLangId = sTmp.GetToken( 2, '\t' ).ToInt32();
+            aStatus = sTmp.GetToken( 3, '\t' );     // ext int ...
+            aText = sTmp.GetToken( 4, '\t' );
+        }
+        else
+            aFormat = FORMAT_UNKNOWN;
+    }
+    else if ( rLine.GetTokenCount( '\t' ) == 15 )
+    {
+        aFormat = FORMAT_SDF;
+        aUniqId = rLine.GetToken( 4, '\t' ).Append("/").Append( rLine.GetToken( 3, '\t' ) ).Append("/").Append( rLine.GetToken( 5, '\t' ) );
+        aLineType = "";
+        nLangId = rLine.GetToken( 9, '\t' ).ToInt32();
+        aStatus = "";
+        aText = rLine.GetToken( 10, '\t' );
+    }
+    else
+        aFormat = FORMAT_UNKNOWN;
 
-    aUniqId = sTmp.GetToken( 0, '\t' );
+    if ( FORMAT_UNKNOWN == GetLineFormat() )
+        NotOK();
 }
 
 //
@@ -137,59 +202,55 @@ GSIBlock::~GSIBlock()
 /*****************************************************************************/
 {
     delete pSourceLine;
+    delete pReferenceLine;
 
     for ( ULONG i = 0; i < Count(); i++ )
         delete ( GetObject( i ));
 }
 
 /*****************************************************************************/
-void GSIBlock::InsertLine( const ByteString &rLine, ULONG nLine , const USHORT nSourceLang)
+void GSIBlock::InsertLine( GSILine* pLine, const USHORT nSourceLang)
 /*****************************************************************************/
 {
-    GSILine *pLine = new GSILine( rLine, nLine );
-
-    ByteString sTmp( rLine );
-    sTmp.SearchAndReplaceAll( "($$)", "\t" );
-
-    if ( sTmp.GetTokenCount( '\t' ) < 5 )
+    if ( pLine->GetLanguageId() == nSourceLang )
     {
-        PrintError( "Unable to determin language and/or state", "Line format", rLine.Copy( 0,100 ), nLine );
-        pLine->NotOK();
-    }
-
-    USHORT nLangId = sTmp.GetToken( 2, '\t' ).ToInt32();
-    if ( nLangId == nSourceLang )
+        if ( pSourceLine && bInternal )
+            PrintError( "Source Language entry double. Ignoring.", "File format", "", pLine->GetLineNumber(), pLine->GetUniqId() );
         pSourceLine = pLine;
+    }
     else {
         ULONG nPos = 0;
 
-        BOOL bLineIsInternal = sTmp.GetToken( 3, '\t' ).EqualsIgnoreCaseAscii( "int" );
+        BOOL bLineIsInternal = pLine->GetLineStatus().EqualsIgnoreCaseAscii( "int" );
         if ( (  bLineIsInternal &&  bInternal )
-           ||( !bLineIsInternal && !bInternal ) )
+           ||( !bLineIsInternal && !bInternal )
+           ||( pLine->GetLineFormat() == FORMAT_SDF ))  // in sdf files check it all
         {
-            while (( nPos < Count()) &&
-                    ( GetObject( nPos )->GetLineNumber() < pLine->GetLineNumber()))
+            while ( nPos < Count() )
+            {
+                if ( GetObject( nPos )->GetLanguageId() == pLine->GetLanguageId() )
+                    PrintError( "Translation Language entry double. Checking both.", "File format", "", pLine->GetLineNumber(), pLine->GetUniqId() );
                 nPos++;
+            }
 
-            Insert( pLine, nPos );
+            Insert( pLine, LIST_APPEND );
         }
     }
 }
 
 /*****************************************************************************/
+void GSIBlock::SetReferenceLine( GSILine* pLine )
+/*****************************************************************************/
+{
+    pReferenceLine = pLine;
+}
+
+/*****************************************************************************/
 void GSIBlock::PrintError( ByteString aMsg, ByteString aPrefix,
     ByteString aContext, ULONG nLine, ByteString aUniqueId )
 /*****************************************************************************/
 {
-    fprintf( stdout, "Error: %s, Line %lu", aPrefix.GetBuffer(),
-        nLine );
-    if ( aUniqueId.Len() )
-        fprintf( stdout, ", UniqueID %s", aUniqueId.GetBuffer() );
-    fprintf( stdout, ": %s", aMsg.GetBuffer() );
-
-    if ( bPrintContext )
-        fprintf( stdout, "  \"%s\"", aContext.GetBuffer() );
-    fprintf( stdout, "\n" );
+    ::PrintError( aMsg, aPrefix, aContext, bPrintContext, nLine, aUniqueId );
 }
 
 /*****************************************************************************/
@@ -205,7 +266,7 @@ void GSIBlock::PrintList( ParserMessageList &rList, ByteString aPrefix,
         if ( bPrintContext )
         {
             if ( pMsg->GetTagBegin() == STRING_NOTFOUND )
-                aContext = pLine->Copy( 0, 300 );
+                aContext = pLine->GetText().Copy( 0, 300 );
             else
                 aContext = pLine->Copy( pMsg->GetTagBegin()-150, 300 );
             aContext.EraseTrailingChars(' ');
@@ -217,6 +278,20 @@ void GSIBlock::PrintList( ParserMessageList &rList, ByteString aPrefix,
 }
 
 /*****************************************************************************/
+BOOL GSIBlock::TestUTF8( GSILine* pTestee )
+/*****************************************************************************/
+{
+    String aUTF8Tester( pTestee->GetText(), RTL_TEXTENCODING_UTF8 );
+    if ( !ByteString( aUTF8Tester, RTL_TEXTENCODING_UTF8 ).Equals( pTestee->GetText() ) )
+    {
+        PrintError( "UTF8 Encoding seems to be broken", "File format", "", pTestee->GetLineNumber(), pTestee->GetUniqId() );
+        pTestee->NotOK();
+        return FALSE;
+    }
+    return TRUE;
+}
+
+/*****************************************************************************/
 BOOL GSIBlock::CheckSyntax( ULONG nLine )
 /*****************************************************************************/
 {
@@ -224,7 +299,7 @@ BOOL GSIBlock::CheckSyntax( ULONG nLine )
 
     if ( !pSourceLine )
     {
-        PrintError( "No source languages reference defined!", "File format", "", nLine );
+        PrintError( "No source language entry defined!", "File format", "", nLine );
         aTester.ReferenceOK( "" );
     }
     else
@@ -236,6 +311,33 @@ BOOL GSIBlock::CheckSyntax( ULONG nLine )
             pSourceLine->NotOK();
         }
     }
+    if ( bReference )
+    {
+        if ( !pReferenceLine )
+        {
+            GSILine *pSource;
+            if ( pSourceLine )
+                pSource = pSourceLine;
+            else
+                pSource = GetObject( 0 );   // get some other line
+            if ( pSource )
+                PrintError( "No reference line found. Entry is new in source file", "File format", "", pSource->GetLineNumber(), pSource->GetUniqId() );
+            else
+                PrintError( "No reference line found. Entry is new in source file", "File format", "", nLine );
+        }
+        else
+        {
+            if ( pSourceLine && !pSourceLine->Equals( *pReferenceLine ) )
+            {
+                xub_StrLen nPos = pSourceLine->Match( *pReferenceLine );
+                PrintError( "Source Language has changed.", "File format", pReferenceLine->Copy( nPos - 5, 15).Append( "\" --> \"" ). Append( pSourceLine->Copy( nPos - 5, 15) ), pSourceLine->GetLineNumber(), pSourceLine->GetUniqId() );
+                pSourceLine->NotOK();
+            }
+        }
+    }
+
+    if ( bInternal && pSourceLine )
+        TestUTF8( pSourceLine );
 
     ULONG i;
     for ( i = 0; i < Count(); i++ )
@@ -245,9 +347,10 @@ BOOL GSIBlock::CheckSyntax( ULONG nLine )
             GetObject( i )->NotOK();
             if ( aTester.HasTesteeErrors() )
                 PrintList( aTester.GetTesteeErrors(), "Translation", GetObject( i ) );
-            if ( aTester.HasCompareWarnings() )
+            if ( pSourceLine && aTester.HasCompareWarnings() )
                 PrintList( aTester.GetCompareWarnings(), "Translation Tag Missmatch", GetObject( i ) );
         }
+        TestUTF8( GetObject( i ) );
     }
 
     return TRUE;
@@ -304,42 +407,25 @@ void Help()
 /*****************************************************************************/
 {
     fprintf( stdout, "\n" );
-    fprintf( stdout, "gsicheck Version 1.5.3 (c)1999 - 2001 by SUN Microsystems\n" );
-    fprintf( stdout, "================================================\n" );
+    fprintf( stdout, "gsicheck Version 1.6.0 (c)1999 - 2001 by SUN Microsystems\n" );
+    fprintf( stdout, "=========================================================\n" );
     fprintf( stdout, "\n" );
-    fprintf( stdout, "gsicheck checks the syntax of tags in GSI-Files (Gutschmitt-Interface)\n" );
+    fprintf( stdout, "gsicheck checks the syntax of tags in GSI-Files and SDF-Files\n" );
+    fprintf( stdout, "         checks for inconsistencies and malicious UTF8 encoding\n" );
     fprintf( stdout, "\n" );
-    fprintf( stdout, "Syntax: gsicheck [ -c ] [ -we ] [ -wc ] [ -i ] [ -l ISO-code ] filename\n" );
+    fprintf( stdout, "Syntax: gsicheck [ -c ] [ -we ] [ -wc ] [ -i ] [ -l LanguageID ]\n" );
+    fprintf( stdout, "                 [ -r ReferenceFile ] filename\n" );
     fprintf( stdout, "\n" );
     fprintf( stdout, "-c    Add context to error message (Print the line containing the error)\n" );
     fprintf( stdout, "-we   Write GSI-File containing all errors\n" );
     fprintf( stdout, "-wc   Write GSI-File containing all correct parts\n" );
     fprintf( stdout, "-i    Check records marked 'int' rather than marked 'ext' or similar\n" );
-    fprintf( stdout, "-l    Numerical 2 digits ISO-code of the source language. Default = 49 \n" );
+    fprintf( stdout, "-l    Numerical 2 digits Identifier of the source language. Default = 49\n" );
+    fprintf( stdout, "-r    Reference filename to check that source language has not been changed\n" );
        fprintf( stdout, "\n" );
 }
 
 /*****************************************************************************/
-ByteString GetLineType( const ByteString &rLine )
-/*****************************************************************************/
-{
-    ByteString sTmp( rLine );
-    sTmp.SearchAndReplaceAll( "($$)", "\t" );
-
-    return sTmp.GetToken( 1, '\t' );
-}
-
-/*****************************************************************************/
-ByteString GetUniqId( const ByteString &rLine )
-/*****************************************************************************/
-{
-    ByteString sTmp( rLine );
-    sTmp.SearchAndReplaceAll( "($$)", "\t" );
-
-    return sTmp.GetToken( 0, '\t' );
-}
-
-/*****************************************************************************/
 #if defined( UNX ) || defined( MAC )
 int main( int argc, char *argv[] )
 #else
@@ -355,6 +441,8 @@ int _cdecl main( int argc, char *argv[] )
     BOOL bWriteCorrect = FALSE;
     USHORT nSourceLang = 49;     // German is default
     ByteString aFilename;
+    ByteString aReferenceFilename;
+    BOOL bReferenceFile = FALSE;
     for ( USHORT i = 1 ; i < argc ; i++ )
     {
         if ( *argv[ i ] == '-' )
@@ -392,6 +480,21 @@ int _cdecl main( int argc, char *argv[] )
                         }
                     }
                     break;
+                case 'r':
+                    {
+                        if ( (i+1) < argc )
+                        {
+                            aReferenceFilename = argv[ i+1 ];
+                            bReferenceFile = TRUE;
+                            i++;
+                        }
+                        else
+                        {
+                            fprintf( stderr, "\nERROR: Switch %s requires parameter!\n\n", argv[ i ] );
+                            bError = TRUE;
+                        }
+                    }
+                    break;
                 default:
                     fprintf( stderr, "\nERROR: Unknown Switch %s!\n\n", argv[ i ] );
                     bError = TRUE;
@@ -428,6 +531,22 @@ int _cdecl main( int argc, char *argv[] )
         exit ( 3 );
     }
 
+    SvFileStream aReferenceGSI;
+    if ( bReferenceFile )
+    {
+        DirEntry aReferenceSource = DirEntry( String( aReferenceFilename, RTL_TEXTENCODING_ASCII_US ));
+        if ( !aReferenceSource.Exists()) {
+            fprintf( stderr, "\nERROR: GSI-File %s not found!\n\n", aFilename.GetBuffer() );
+            exit ( 2 );
+        }
+
+        aReferenceGSI.Open( String( aReferenceFilename, RTL_TEXTENCODING_ASCII_US ), STREAM_STD_READ );
+        if ( !aReferenceGSI.IsOpen()) {
+            fprintf( stderr, "\nERROR: Could not open GSI-File %s!\n\n", aFilename.GetBuffer() );
+            exit ( 3 );
+        }
+    }
+
     SvFileStream aOkOut;
     String aBaseName = aSource.GetBase();
     if ( bWriteCorrect )
@@ -458,27 +577,40 @@ int _cdecl main( int argc, char *argv[] )
     }
 
 
-    ByteString sGSILine;
+    ByteString sReferenceLine;
+    GSILine* pReferenceLine = NULL;
+    ByteString aOldReferenceId("No Valid ID");   // just set to something which can never be an ID
+    ULONG nReferenceLine = 0;
 
-    ByteString aOldId;
+    ByteString sGSILine;
+    GSILine* pGSILine = NULL;
+    ByteString aOldId("No Valid ID");   // just set to something which can never be an ID
     GSIBlock *pBlock = NULL;
     ULONG nLine = 0;
 
-    while ( !aGSI.IsEof())
+    while ( !aGSI.IsEof() )
     {
         aGSI.ReadLine( sGSILine );
         nLine++;
+        pGSILine = new GSILine( sGSILine, nLine );
+        BOOL bDelete = TRUE;
 
-        if ( sGSILine.Len())
+
+        if ( pGSILine->Len() )
         {
-            if ( GetLineType( sGSILine ).CompareIgnoreCaseToAscii("res-comment") == COMPARE_EQUAL )
+            if ( FORMAT_UNKNOWN == pGSILine->GetLineFormat() )
+            {
+                PrintError( "Format of line is unknown. Ignoring!", "Line format", pGSILine->Copy( 0,40 ), bPrintContext, pGSILine->GetLineNumber() );
+                pGSILine->NotOK();
+            }
+            else if ( pGSILine->GetLineType().EqualsIgnoreCaseAscii("res-comment") )
             {   // ignore comment lines, but write them to Correct Items File
                 if ( bWriteCorrect )
-                       aOkOut.WriteLine( sGSILine );
+                       aOkOut.WriteLine( *pGSILine );
             }
             else
             {
-                ByteString aId = GetUniqId( sGSILine );
+                ByteString aId = pGSILine->GetUniqId();
                 if ( aId != aOldId )
                 {
                     if ( pBlock )
@@ -492,14 +624,62 @@ int _cdecl main( int argc, char *argv[] )
 
                         delete pBlock;
                     }
-                    pBlock = new GSIBlock( bPrintContext, bInternal );
+                    pBlock = new GSIBlock( bPrintContext, bInternal, bReferenceFile );
 
                     aOldId = aId;
+
+
+                    // find corrosponding line in reference file
+                    if ( bReferenceFile )
+                    {
+                        BOOL bContinueSearching = TRUE;
+                        while ( ( !aReferenceGSI.IsEof() || pReferenceLine ) && bContinueSearching )
+                        {
+                            if ( !pReferenceLine )
+                            {
+                                aReferenceGSI.ReadLine( sReferenceLine );
+                                nReferenceLine++;
+                                pReferenceLine = new GSILine( sReferenceLine, nReferenceLine );
+                            }
+                            if ( pReferenceLine->GetLineFormat() != FORMAT_UNKNOWN )
+                            {
+                                if ( pReferenceLine->GetUniqId() == aId && pReferenceLine->GetLanguageId() == nSourceLang )
+                                {
+                                    pBlock->SetReferenceLine( pReferenceLine );
+                                    pReferenceLine = NULL;
+                                }
+                                else if ( pReferenceLine->GetUniqId() > aId )
+                                {
+//                                    if ( pGSILine->GetLanguageId() == nSourceLang )
+//                                      PrintError( "No reference line found. Entry is new in source file", "File format", "", bPrintContext, pGSILine->GetLineNumber(), aId );
+                                    bContinueSearching = FALSE;
+                                }
+                                else
+                                {
+                                    if ( pReferenceLine->GetUniqId() < aId  && pReferenceLine->GetLanguageId() == nSourceLang )
+                                        PrintError( "No Entry in source file found. Entry has been removed from source file", "File format", "", bPrintContext, pGSILine->GetLineNumber(), pReferenceLine->GetUniqId() );
+                                    delete pReferenceLine;
+                                    pReferenceLine = NULL;
+                                }
+                            }
+                            else
+                            {
+                                delete pReferenceLine;
+                                pReferenceLine = NULL;
+                            }
+
+                        }
+                    }
+
                 }
 
-                pBlock->InsertLine( sGSILine, nLine , nSourceLang);
+                pBlock->InsertLine( pGSILine, nSourceLang );
+                bDelete = FALSE;
             }
         }
+        if ( bDelete )
+            delete pGSILine;
+
     }
     if ( pBlock )
     {
diff --git a/transex3/source/localize.cxx b/transex3/source/localize.cxx
index 4755854e2311..0097ccc79f67 100644
--- a/transex3/source/localize.cxx
+++ b/transex3/source/localize.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: localize.cxx,v $
  *
- *  $Revision: 1.27 $
+ *  $Revision: 1.28 $
  *
- *  last change: $Author: ihi $ $Date: 2002-11-08 10:54:56 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:47 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -80,6 +80,7 @@ const char *ExeTable[][5] = {
     { "xgf", "xmlex", "-UTF8 -e -t:xgf", "negative", "iso" },
     { "xcd", "cfgex", "-UTF8 -e", "negative", "iso" },
     { "xcu", "cfgex", "-UTF8 -e", "negative", "iso" },
+    { "xcs", "cfgex", "-UTF8 -e -f", "negative", "iso" },
     { "xrm", "xrmex", "-UTF8 -e", "negative", "iso" },
     { "NULL", "NULL", "NULL", "NULL", "NULL" }
 };
diff --git a/transex3/source/merge.cxx b/transex3/source/merge.cxx
index a60cd1f7c3bc..d240121c7ec3 100644
--- a/transex3/source/merge.cxx
+++ b/transex3/source/merge.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: merge.cxx,v $
  *
- *  $Revision: 1.10 $
+ *  $Revision: 1.11 $
  *
- *  last change: $Author: nf $ $Date: 2002-08-15 11:13:17 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:48 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses
@@ -420,6 +420,8 @@ USHORT MergeDataFile::GetLangIndex( USHORT nId )
         case HEBREW: return HEBREW_INDEX;
         case CATALAN: return CATALAN_INDEX;
         case THAI: return THAI_INDEX;
+        case HINDI: return HINDI_INDEX;
+        case ESTONIAN: return ESTONIAN_INDEX;
         case EXTERN: return EXTERN_INDEX;
     }
     return 0xFFFF;
diff --git a/transex3/source/tagtest.cxx b/transex3/source/tagtest.cxx
index 75b5fc8cc210..4562ee075637 100644
--- a/transex3/source/tagtest.cxx
+++ b/transex3/source/tagtest.cxx
@@ -2,9 +2,9 @@
  *
  *  $RCSfile: tagtest.cxx,v $
  *
- *  $Revision: 1.5 $
+ *  $Revision: 1.6 $
  *
- *  last change: $Author: gh $ $Date: 2003-02-05 14:11:45 $
+ *  last change: $Author: hr $ $Date: 2003-03-27 17:10:50 $
  *
  *  The Contents of this file are made available subject to the terms of
  *  either of the following licenses