CWS-TOOLING: integrate CWS l10ncleanup04

2009-05-28 13:46:54 +0200 ihi  r272407 : remove forgotten sdf's
2009-05-25 22:32:35 +0200 ihi  r272273 : no WITH_LANG fix
2009-05-25 20:45:58 +0200 ihi  r272272 : remove some comments
2009-05-25 19:55:21 +0200 ihi  r272271 : svx dialog -> cui l10n move
2009-05-25 18:30:58 +0200 ihi  r272268 : build fix
2009-05-18 16:32:02 +0200 ihi  r272033 : bash fix
2009-05-18 16:31:32 +0200 ihi  r272032 : bash fix
2009-05-14 16:23:39 +0200 ihi  r271901 : #i79750# Translation moved to l10n module
2009-05-11 23:36:05 +0200 ihi  r271793 : #i79750# Translation moved into own module

5 files changed, 641 insertions, 206 deletions

diff --git a/transex3/inc/export.hxx b/transex3/inc/export.hxx
index 89c1cf548947..3d7eee8e1eaf 100644
--- a/transex3/inc/export.hxx
+++ b/transex3/inc/export.hxx
@@ -510,7 +510,9 @@ private:
     SvFileStream aErrLog;
     ByteStringSet aLanguageSet;
     MergeDataHashMap aMap;
-    std::vector<ByteString> aLanguages;
+    ByteStringHashMap aLanguageMap;
+    std::vector<ByteString> aLanguageList;
+    ByteStringHashMap aFilenames;
 
 
 public:
diff --git a/transex3/prj/d.lst b/transex3/prj/d.lst
index 7bd8e7d3e72d..54d1ab156205 100644
--- a/transex3/prj/d.lst
+++ b/transex3/prj/d.lst
@@ -28,6 +28,7 @@ mkdir: %_DEST%\inc%_EXT%\transex3
 
 ..\scripts\localize.pl %_DEST%\bin%_EXT%\localize.pl
 ..\scripts\localize %_DEST%\bin%_EXT%\localize
+..\scripts\fast_merge.pl %_DEST%\bin%_EXT%\fast_merge.pl
 
 ..\inc\export.hxx %_DEST%\inc%_EXT%\transex3\export.hxx
 ..\inc\transex3\directory.hxx %_DEST%\inc%_EXT%\transex3\directory.hxx
diff --git a/transex3/scripts/fast_merge.pl b/transex3/scripts/fast_merge.pl
new file mode 100644
index 000000000000..266c1c0801e8
--- /dev/null
+++ b/transex3/scripts/fast_merge.pl
@@ -0,0 +1,348 @@
+:
+eval 'exec perl -wS $0 ${1+"$@"}'
+    if 0;
+#*************************************************************************
+#
+#   OpenOffice.org - a multi-platform office productivity suite
+#
+#   $RCSfile: fast_merge.pl,v $
+#
+#   $Revision: 1.1.2.2 $
+#
+#   last change: $Author: ihi $ $Date: 2007/07/20 10:37:53 $
+#
+#   The Contents of this file are made available subject to
+#   the terms of GNU Lesser General Public License Version 2.1.
+#
+#
+#     GNU Lesser General Public License Version 2.1
+#     =============================================
+#     Copyright 2005 by Sun Microsystems, Inc.
+#     901 San Antonio Road, Palo Alto, CA 94303, USA
+#
+#     This library is free software; you can redistribute it and/or
+#     modify it under the terms of the GNU Lesser General Public
+#     License version 2.1, as published by the Free Software Foundation.
+#
+#     This library is distributed in the hope that it will be useful,
+#     but WITHOUT ANY WARRANTY; without even the implied warranty of
+#     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+#     Lesser General Public License for more details.
+#
+#     You should have received a copy of the GNU Lesser General Public
+#     License along with this library; if not, write to the Free Software
+#     Foundation, Inc., 59 Temple Place, Suite 330, Boston,
+#     MA  02111-1307  USA
+#
+#*************************************************************************
+
+use strict;
+use Class::Struct;
+use Getopt::Long;
+use File::Temp;
+use File::Path;
+
+my @files;
+my @file_names;
+my $module_name = '';
+my @current;
+my @buffer;
+my $last_file;
+my $last_path;
+my $last_localize_file;
+my $first_run = "1";
+my $sdf_filename;
+my $merge_dir;
+my $WIN;
+my $state = "none";
+
+if ( defined $ENV{USE_SHELL} && $ENV{USE_SHELL} eq '4nt' ) { $WIN = 'TRUE'; }
+else { $WIN = ''; }
+
+$SIG{INT}  = 'inthandler';
+$SIG{QUIT} = 'quithandler';
+
+struct ( sdf_obj =>
+{
+    module      => '$',
+    file        => '$',
+    dir         => '$',
+    FILEHANDLE  => '$',
+    line        => '$',
+    endoffile   => '$'
+}
+);
+
+parse_options();
+my $lock_file   = $merge_dir."/lock.mk";
+$lock_file =~ s/\//\\/g , if ( $WIN ) ;
+acquire_lock();
+read_sdf_file_names();
+init();
+my $reference;
+my $path ;
+my $localize_file;
+while( hasLines() )
+{
+    @current = ();
+    foreach ( @files )
+    {
+        push @current , $_;
+    }
+
+    $reference = getNextIdentifier( );
+
+    @current = ();
+    foreach ( @files )
+    {
+        if( $_->module eq $reference->module && $_->dir eq $reference->dir )
+        {
+            push @current , $_ ;
+        }
+    }
+    write_lines();
+}
+if( $#current+1 ne 0 )
+{
+    ( $path , $localize_file ) = make_paths();
+    add_to_buffer();
+    write_buffer( $path , $localize_file );
+}
+release_lock();
+exit( 0 );
+
+##########################################################################################
+sub acquire_lock
+{
+    if( -e $lock_file ){
+        $state = "blocked";
+        print "WARNING: Lock file '$lock_file' 'found, waiting ....\n";
+        my $cnt = 0;
+        sleep 10 , while( -e $lock_file && $cnt++ < 180 );
+        exit( 0 );
+    }else
+    {
+        $state = "locked";
+        print "Writing lock file '$lock_file'\n";
+        open FILE, ">$lock_file" or die "Can't create lock file '$lock_file'";
+        print FILE "L10N_LOCK=YES" ;
+        close ( FILE );
+    }
+}
+sub release_lock
+{
+    print "Deleting lock file '$lock_file'\n";
+    unlink $lock_file, if( -e $lock_file );
+    $state = "none";
+}
+sub inthandler
+{
+    release_lock() , if( $state eq "locked" );
+    exit( -1 );
+}
+sub quithandler
+{
+    release_lock() , if( $state eq "locked" );
+    exit( 0 );
+}
+
+sub init
+{
+    foreach my $file ( @file_names )
+    {
+        my $obj = new sdf_obj;
+        open my $FILEHANDLE , "<$file" or die "Can't open file '$file'";
+        $obj->FILEHANDLE ( $FILEHANDLE ) ;
+        getNextSdfObj( $obj );
+        push @files, $obj ;
+        print "Open file '$file'\n";
+    }
+}
+
+# get the next module/file
+sub getNextIdentifier
+{
+    my @sorted = sort {
+        return $a->module.$a->dir cmp $b->module.$b->dir;
+    } @current ;
+    return shift @sorted;
+}
+
+# update the obj with the next line
+sub getNextSdfObj
+{
+    my $obj             = shift;
+    my $line = readline ( $obj->FILEHANDLE );
+    if ( $line eq undef )
+    {
+        $obj->endoffile( "true" );
+    }
+    else
+    {
+        $line =~ /^(([^\t]*)\t([^\t]*).*)/o ;
+        if( defined $1 && defined $2 && defined $3 )
+        {
+            $obj->line  ( $1 );
+            $obj->module( $2 );
+            $obj->file  ( $3 );
+            $obj->dir   ( getDir( $3 ) );
+        }
+        else
+        {
+            $obj->line  ( "" );
+            $obj->module( "" );
+            $obj->file  ( "" );
+            $obj->dir   ( "" );
+        }
+    }
+    return $obj;
+}
+sub getNextSdfObjModule
+{
+    my $obj             = shift;
+    while( !$obj->endoffile )
+    {
+        my $line = readline ( $obj->FILEHANDLE );
+        if ( $line eq undef )
+        {
+            $obj->endoffile( "true" );
+        }
+        else
+        {
+            $line =~ /^(([^\t]*)\t([^\t]*).*)/o ;
+            if( defined $1 && defined $2 && defined $3 )
+            {
+                $obj->line  ( $1 );
+                $obj->module( $2 );
+                $obj->file  ( $3 );
+                $obj->dir   ( getDir( $3 ) );
+            }
+            else
+            {
+                $obj->line  ( "" );
+                $obj->module( "" );
+                $obj->file  ( "" );
+                $obj->dir   ( "" );
+            }
+            return $obj , if( $obj->module eq $module_name )
+        }
+    }
+    #return $obj;
+}
+sub getDir
+{
+    my $path     = shift ;
+    $path        =~ s/\//\\/g;
+    my @tmp_path = split /\\/ , $path;
+    pop @tmp_path;
+    $path        = join  '\\' , @tmp_path;
+    return $path;
+}
+
+sub hasLines
+{
+    my $hasLines  = "";
+    my @tmpfiles;
+    foreach ( @files )
+    {
+        push @tmpfiles , $_, if( !$_->endoffile );
+    }
+    @files = @tmpfiles;
+    return $#files+1;
+}
+
+sub make_paths
+{
+    my $localizeFile = $merge_dir."\\".$current[ 0 ]->module."\\".$current[ 0 ]->file;
+    my $path = getDir( $localizeFile );
+    if ( !$WIN ) { $path =~ s/\\/\//g; }
+
+    $localizeFile = $path."\\localize.sdf";
+    if ( !$WIN ) { $localizeFile =~ s/\\/\//g; }
+
+    return ( $path , $localizeFile );
+}
+sub write_lines
+{
+    if( $first_run ){
+        add_to_buffer();
+        my( $path , $localize_file ) = make_paths();
+        $last_path = $path;
+        $last_localize_file = $localize_file;
+        mkpath $path;
+        write_buffer( $path , $localize_file );
+        $first_run = '';
+    }
+    else
+    {
+        return , if ( $#current+1 eq 0 );
+        my( $path , $localize_file ) = make_paths();
+        if( $path eq $last_path )
+        {
+            add_to_buffer();
+        }
+        else
+        {
+            mkpath $path;
+            write_buffer( $last_path , $last_localize_file );
+            add_to_buffer();
+            $last_path = $path;
+            $last_localize_file = $localize_file;
+        }
+    }
+}
+sub add_to_buffer
+{
+    my $plainline;
+    my $afile;
+    my $amodule;
+    foreach my $elem ( @current )
+    {
+        do {
+        $amodule=$elem->module;
+        $afile=$elem->file;
+        $plainline=$elem->line;
+        push @buffer, $plainline;
+        getNextSdfObj( $elem );
+        } while ( !$elem->endoffile && $amodule eq $elem->module && $afile eq $elem->file );
+    }
+}
+sub write_buffer
+{
+    my $path            = shift;
+    my $localize_file   = shift;
+    my $cnt             = $#buffer+1;
+    print "Write to $path $cnt lines\n";
+    open FILE , ">>$localize_file" or die "Can't open file '$localize_file'\n";
+    foreach ( @buffer )
+    {
+        print FILE $_."\n";
+    }
+    @buffer = ();
+}
+sub parse_options
+{
+    my $success = GetOptions( 'sdf_files=s' => \$sdf_filename , 'merge_dir=s' => \$merge_dir ); #, 'module=s' => \$module_name );
+    if( ! ( $sdf_filename && $merge_dir && $success ) )
+    {
+        usage();
+        exit( -1 );
+    }
+}
+
+sub usage
+{
+    print "Usage: fast_merge -sdf_files <file containing sdf file names> -merge_dir <directory>\n" ;
+}
+
+sub read_sdf_file_names
+{
+    open FILE , "<$sdf_filename" or die "Can't open file '$sdf_filename'\n";
+    while (  <FILE> )
+    {
+        push @file_names , split " " , $_ ;
+    }
+    close ( FILE );
+}
+
+
diff --git a/transex3/scripts/localize.pl b/transex3/scripts/localize.pl
index b9e478ba66bd..8d09b567b77b 100755
--- a/transex3/scripts/localize.pl
+++ b/transex3/scripts/localize.pl
@@ -13,7 +13,7 @@ eval 'exec perl -wS $0 ${1+"$@"}'
 #
 # $RCSfile: localize.pl,v $
 #
-# $Revision: 1.18 $
+# $Revision: 1.18.6.2 $
 #
 # This file is part of OpenOffice.org.
 #
@@ -39,10 +39,12 @@ use Getopt::Long;
 use IO::Handle;
 use File::Find;
 use File::Temp;
+use File::Path;
 use File::Copy;
 use File::Glob qw(:glob csh_glob);
 use Cwd;
 
+my $CVS_BINARY = "/usr/bin/cvs";
 # ver 1.1
 #
 #### module lookup
@@ -61,16 +63,24 @@ BEGIN {
 use lib (@lib_dirs);
 
 #### globals ####
-my $sdffile = '';
-my $no_sort = '';
-my $outputfile = '';
-my $mode = '';
-my $bVerbose="0";
-my $srcpath = '';
+my $sdffile                 = '';
+my $no_sort                 = '';
+my $create_dirs             = '';
+my $multi_localize_files    = '';
+my $module_to_merge         = '';
+my $sort_sdf_before         = '';
+my $outputfile              = '';
+my $no_gsicheck             = '';
+my $mode                    = '';
+my $bVerbose                = "0";
+my $srcpath                 = '';
 my $WIN;
 my $languages;
 #my %sl_modules;     # Contains all modules where en-US and de is source language
 my $use_default_date = '0';
+my %is_ooo_module;
+my %is_so_module;
+my $DELIMITER;
 
          #         (                           leftpart                                                     )            (           rightpart                    )
          #            prj      file      dummy     type       gid       lid      helpid    pform     width      lang       text    helptext  qhelptext   title    timestamp
@@ -82,21 +92,28 @@ my @sdfparticles;
 
 #### main ####
 parse_options();
+check_modules_scm();
 
 if ( defined $ENV{USE_SHELL} && $ENV{USE_SHELL} eq '4nt' ) {
    $WIN = 'TRUE';
+   $DELIMITER = "\\";
 }
  else {
    $WIN = '';
+   $DELIMITER = "/";
 }
 
 #%sl_modules = fetch_sourcelanguage_dirlist();
 
 
 if   ( $mode eq "merge"    )    {
-    merge_gsicheck();
+    if ( ! $no_gsicheck ){
+        merge_gsicheck();
+    }
     splitfile( $sdffile );
-    unlink $sdffile;             # remove temp file!
+    if ( ! $no_gsicheck ){
+        unlink $sdffile;             # remove temp file!
+    }
 }
 elsif( $mode eq "extract"  )    {
     collectfiles( $outputfile );
@@ -126,6 +143,12 @@ sub splitfile{
     open MYFILE , "< $sdffile"
     or die "Can't open '$sdffile'\n";
 
+#    my %lang_hash;
+    my %string_hash_ooo;
+    my %string_hash_so;
+    my %so_modules;
+    $so_modules{ "extras_full" } = "TRUE";
+
     while( <MYFILE>){
          if( /$sdf_regex/ ){
             my $line           = defined $_ ? $_ : '';
@@ -137,183 +160,187 @@ sub splitfile{
             my $lang           = defined $12 ? $12 : '';
             my $plattform      = defined $10 ? $10 : '';
             my $helpid         = defined $9 ? $9 : '';
-
             next if( $prj eq "binfilter" );     # Don't merge strings into binfilter module
             chomp( $line );
-            $currentFile  = $srcpath . '\\' . $prj . '\\' . $file;
-            if ( $WIN ) { $currentFile  =~ s/\//\\/g; }
-            else        { $currentFile  =~ s/\\/\//g; }
-
-            $cur_sdffile = $currentFile;
-            #if( $cur_sdffile =~ /\.$file_types[\s]*$/ ){
-                if( $WIN ) { $cur_sdffile =~ s/\\[^\\]*\.$file_types[\s]*$/\\localize.sdf/; }
-                else       { $cur_sdffile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/; }
-            #}
-
-            # Set default date
-            if( $line =~ /(.*)\t[^\t\$]*$/ ){
-                $line = $1."\t".$default_date;
-            }
-
-            if( $start ){
-                $start='';
-                $lastFile = $currentFile; # ?
-                $last_sdffile = $cur_sdffile;
-            }
 
-            if( $lang eq "en-US" ){}
-            elsif( $cur_sdffile eq $last_sdffile )
+            if( is_openoffice_module( $prj ) )
             {
-                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
+                $string_hash_ooo { $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
             }
             else
             {
-                writesdf( $lastFile , \%block );
-                $lastFile = $currentFile; #?
-                $last_sdffile = $cur_sdffile;
-                %block = ();
-                #if( ! $lang eq "en-US"  ) {
-                $block{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
-                #}
-
+                $string_hash_so{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = $line;
             }
-        } #else { print STDOUT "splitfile REGEX kaputt\n";}
-
+        }
     }
-    writesdf( $lastFile , \%block );
-    %block = ();
     close( MYFILE );
 
+    if( !defined $ENV{SRC_ROOT} ){
+        print "Error, no SRC_ROOT in env found.\n";
+        exit( -1 );
+    }
+    my $src_root = $ENV{SRC_ROOT};
+    #print $WIN eq "TRUE" ? $src_root."\\l10n_so\n" : $src_root."/l10n_so\n";
+    my $so_l10n_path  = $WIN eq "TRUE" ? $src_root."\\l10n_so\\source" : $src_root."/l10n_so/source";
+    my $ooo_l10n_path = $WIN eq "TRUE" ? $src_root."\\l10n\\source"    : $src_root."/l10n/source";
+
+    #print "$so_l10n_path\n";
+    #print "$ooo_l10n_path\n";
+
+    write_sdf( \%string_hash_so , $so_l10n_path );
+    write_sdf( \%string_hash_ooo , $ooo_l10n_path );
+
+}
+sub check_modules_scm
+{
+    #my @ooo_modules;
+    #my @so_modules;
+    my $src_path        = $ENV{ SRC_ROOT } ;
+    my $last_dir        = getcwd();
+    chdir $src_path ;
+    my @modules         = <*/.svn/entries>;
+
+    foreach my $module ( @modules )
+    {
+        #print "$module \n";
+        if( open ( FILE , "<$module" ) )
+        {
+            while( <FILE> )
+            {
+
+                my @path = split ( "/" , $module ) ;
+
+                if( /svn.services.openoffice.org/ )
+                {
+                    my $mod = $path[ 0 ];
+                    #push @ooo_modules , $mod;
+                    $is_ooo_module{ $mod } = "true";
+                    # print "$module -> ooo ";
+                }
+                elsif ( /jumbo2.germany.sun.com/ )
+                {
+                    my $mod = $path[ 0 ];
+                    #push @so_modules , $mod;
+                    # print "$module -> so ";
+                    #$so_lookup_hash{ $mod } = "true";
+                }
+                #else
+                #{
+                #    print "ERROR: Is $module a SO or OOo module? Can not parese the $module/.svn/entries file ... please check mwsfinnish/merge/splitsdf.pl line 280\n";
+                # exit -1;
+                #}
+            }
+        }
+    }
+    chdir $last_dir ;
+    #print "OOO\n";
+    #print @ooo_modules;
+    #print "\nSO\n";
+    #print @so_modules;
 }
-#########################################################
 
-#sub fetch_sourcelanguage_dirlist
+
+#sub parse
 #{
-#
-#    my $working_path = getcwd();
-#    my %sl_dirlist;
-#
-#    chdir $srcpath;
-#    my @all_dirs = csh_glob( "*" );
-#
-#    foreach my $file ( @all_dirs )
-#    {
-#        if( -d $file )
-#        {
-#            my $module = $file;
-#            $file .= "/prj/l10n";
-#            $file =~ s/\//\\/ , if( $WIN ) ;
-#
-#            if( -f $file )                                        # Test file <module>/prj/l10n
-#            {
-#                $sl_dirlist{ $module } = 1;
-#                if( $bVerbose eq "1" ) { print STDOUT "$module: de and en-US source language detected\n"; }
-#            }
-#        }
+#    my $command = "$CVS_BINARY -d:pserver:anoncvs\@anoncvs.services.openoffice.org:/cvs co -c";
+#    my $output  = `$command`;
+#    my $rc = $? << 8;
+#    if ( $output eq "" || $rc < 0 ){
+#        print STDERR "ERROR: Can not fetch cvs alias list, please login to the cvs server and press at the password prompt just return\ncvs -d:pserver:anoncvs\@anoncvs.services.openoffice.org:/cvs login\n";
+#        exit ( -1 );
 #    }
+#    my @list = split /\n/ , $output ;
+#    foreach my $string( @list )
+#    {
 #
-#    chdir $working_path;
+#        #        print "Found '$1'\n" , if( $string =~ /^(\w*)/ && $1 ne "" );
 #
-#    return %sl_dirlist;
+#        $is_ooo_module{ $1 } = "TRUE", if( $string =~ /^(\w*)/ && $1 ne "" );
+#    }
+#    #    foreach my $key( keys( %is_ooo_module ) )
+#    #{
+#        #    print "$key\n";
+#        #}
 #}
+sub is_openoffice_module
+{
+    my $module              = shift;
+    return "TRUE", if defined $is_ooo_module{ $module };
+    return "";
+}
 
-#sub has_two_sourcelanguages
-#{
-#    my $module          = shift;
-#    return defined $sl_modules{ $module } ;
-#}
-sub writesdf{
-
-    my $lastFile        = shift;
-    my $blockhash_ref   = shift;
-    my $localizeFile    = $lastFile;
-    my %index=();
-
-    if( $localizeFile =~ /\.$file_types[\s]*$/ ){
-        if( $WIN ) { $localizeFile =~ s/\\[^\\]*\.$file_types[\s]*$/\\localize.sdf/; }
-        else       { $localizeFile =~ s/\/[^\/]*\.$file_types[\s]*$/\/localize.sdf/; }
-        }else {
-            print STDERR "Strange filetype found '$localizeFile'\n";
-            return;
-        }
-        if( $bVerbose ){ print STDOUT "$localizeFile\n"; }
-        if( open DESTFILE , "< $localizeFile" ){
+sub write_sdf
+{
+    my $string_hash         = shift;
+    my $l10n_file           = shift;
 
-        #or die "Can't open/create '\$localizeFile'";
+    foreach my $lang( keys( %{ $string_hash } ) )
+    {
+        my @sdf_file;
 
-        #### Build hash
-        while(<DESTFILE>){
-         if( /$sdf_regex/ ){
-            my $line           = defined $_ ? $_ : '';
-            my $prj            = defined $3 ? $3 : '';
-            my $file           = defined $4 ? $4 : '';
-            my $type           = defined $6 ? $6 : '';
-            my $gid            = defined $7 ? $7 : '';
-            my $lid            = defined $8 ? $8 : '';
-            my $lang           = defined $12 ? $12 : '';
-            my $plattform      = defined $10 ? $10 : '';
-            my $helpid         = defined $9 ? $9 : '';
-
-            chomp( $line );
-            $index{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } =  $line ;
+        # mkdir!!!!
+        my $current_l10n_file = $WIN eq "TRUE" ? $l10n_file."\\$lang\\localize.sdf" : $l10n_file."/$lang/localize.sdf";
+        print "Writing '$current_l10n_file'\n";
+        if( open DESTFILE , "< $current_l10n_file" ){
 
-         } #else { print STDOUT "writesdf REGEX kaputt $_\n";}
+            while(<DESTFILE>){
+                if( /$sdf_regex/ ){
+                    my $line           = defined $_ ? $_ : '';
+                    my $prj            = defined $3 ? $3 : '';
+                    my $file           = defined $4 ? $4 : '';
+                    my $type           = defined $6 ? $6 : '';
+                    my $gid            = defined $7 ? $7 : '';
+                    my $lid            = defined $8 ? $8 : '';
+                    my $lang           = defined $12 ? $12 : '';
+                    my $plattform      = defined $10 ? $10 : '';
+                    my $helpid         = defined $9 ? $9 : '';
 
+                    chomp( $line );
+                    if ( defined $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } )
+                    {
+                        # Changed String!
+                        push @sdf_file , $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } ;
+                        $string_hash->{ $lang }{ "$prj\t$file\t$type\t$gid\t$lid\t$helpid\t$plattform\t$lang" } = undef;
+                    }
+                    else
+                    {
+                        # No new string
+                        push @sdf_file , $line;
+                    }
+                }
+            }
         }
         close( DESTFILE );
-    }
-    #### Copy new strings
-    my @mykeys = keys( %{ $blockhash_ref } );
-    my $isDirty = "FALSE";
-    foreach my $key( @mykeys ){
-        if( ! defined $index{ $key } ){
-            # Add new entry
-            $index{ $key }  = $blockhash_ref->{ $key} ;
-            $isDirty = "TRUE";
-        }elsif( $index{ $key } ne $blockhash_ref->{ $key } ){
-            # Overwrite old entry
-            $index{ $key } = $blockhash_ref->{ $key };
-            $isDirty = "TRUE";
-        }else {
+        #Now just append the enw strings
+        #FIXME!!! Implement insertion in the correct order
+        foreach my $key ( keys ( %{ $string_hash->{ $lang } } ) )
+        {
+            push @sdf_file , $string_hash->{ $lang }{ $key } , if ( defined $string_hash->{ $lang }{ $key } );
+            #print "WARNING: Not defined = ".$string_hash->{ $lang }{ $key }."\n", if( ! defined  $string_hash->{ $lang }{ $key } );
         }
-    }
-
-    #### Write file
 
-    if( !$bVerbose ){ print STDOUT "."; }
-    if( $isDirty eq "TRUE" ){
-        if( open DESTFILE , "+> $localizeFile" ){
+        # Write the new file
+        my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
+        if( open DESTFILE , "+> $tmpfile " ){
             print DESTFILE get_license_header();
-            @mykeys = sort keys( %index );
-            foreach my $key( @mykeys ){
-                print DESTFILE ( $index{ $key } , "\n" );
-            }
-            close DESTFILE;
-         }else {
-            print STDOUT "WARNING: File $localizeFile is not writable , try to merge ...\n";
-            my ( $TMPFILE , $tmpfile ) = File::Temp::tempfile();
-            if( open DESTFILE , "+> $tmpfile " ){
-                @mykeys = keys( %index );
-                foreach my $key( @mykeys ){
-                    print DESTFILE ( $index{ $key } , "\n" );
-                }
-                close DESTFILE;
-                if( move( $localizeFile , $localizeFile.".backup" ) ){
-                    if( copy( $tmpfile , $localizeFile ) ){
-                        unlink $localizeFile.".backup";
-                    } else { print STDERR "Can't open/create '$localizeFile', original file is renamed to  $localizeFile.backup\n"; }
-                } else { print STDERR "Can't open/create '$localizeFile'\n"; }
-            }else{
-                print STDERR "WARNING: Can't open/create '$localizeFile'\n";
+            foreach my $string( @sdf_file ){
+                print DESTFILE "$string\n";
             }
-            unlink $tmpfile;
-        }
-    }
-#    if( $no_sort eq '' ){
-#        sort_outfile( $localizeFile );
-#    }
+            close ( DESTFILE );
+            if( move( $current_l10n_file , $current_l10n_file.".backup" ) ){
+                if( copy( $tmpfile , $current_l10n_file ) ){
+                    unlink $l10n_file.".backup";
+                 } else { print STDERR "Can't open/create '$l10n_file', original file is renamed to $l10n_file.backup\n"; }
+            } else { print STDERR "Can't open/create '$l10n_file'\n"; }
+         }else{
+            print STDERR "WARNING: Can't open/create '$l10n_file'\n";
+         }
+         unlink $tmpfile;
+     }
 }
 
+#########################################################
+
 sub get_license_header{
     return
 "#\n".
@@ -398,6 +425,41 @@ sub wanted
     }
 }
 
+sub add_paths
+{
+    my $langhash_ref            = shift;
+    my $root_dir = $ENV{ SRC_ROOT };
+    my $ooo_l10n_dir = "$root_dir"."$DELIMITER"."l10n"."$DELIMITER"."source";
+    my $so_l10n_dir  = "$root_dir"."$DELIMITER"."l10n_so"."$DELIMITER"."source";
+
+    if( -e $ooo_l10n_dir )
+    {
+        foreach my $lang ( keys( %{ $langhash_ref } ) )
+        {
+            my $loc_file = "$ooo_l10n_dir"."$DELIMITER"."$lang"."$DELIMITER"."localize.sdf";
+            if( -e $loc_file )
+            {
+                push @sdfparticles , "$ooo_l10n_dir"."$DELIMITER"."$lang"."$DELIMITER"."localize.sdf";
+            }
+            else { print "WARNING: $loc_file not found ....\n"; }
+        }
+    }
+    else { die "ERROR: Can not find directory $ooo_l10n_dir!!!" }
+    if( -e $so_l10n_dir )
+    {
+        foreach my $lang ( keys( %{ $langhash_ref } ) )
+        {
+            my $loc_file = "$so_l10n_dir"."$DELIMITER"."$lang"."$DELIMITER"."localize.sdf";
+            if( -e $loc_file )
+            {
+                push @sdfparticles , "$ooo_l10n_dir"."$DELIMITER"."$lang"."$DELIMITER"."localize.sdf";
+            }
+            else { #print "WARNING: $loc_file not found ....\n";
+            }
+        }
+
+    }
+}
 sub collectfiles{
     print STDOUT "### Localize\n";
     my $localizehash_ref;
@@ -408,14 +470,14 @@ sub collectfiles{
     STDOUT->autoflush( 1 );
 
     ### Search sdf particles
-    print STDOUT "### Searching sdf particles\n";
+    #print STDOUT "### Searching sdf particles\n";
     my $working_path = getcwd();
-    chdir $srcpath;
-    find ( { wanted => \&wanted , follow => 1 }, getcwd() );
-    chdir $working_path;
-
-    my $nFound  = $#sdfparticles +1;
-    print "\n    $nFound files found !\n";
+    #chdir $srcpath;
+    #find ( { wanted => \&wanted , follow => 1 }, getcwd() );
+    #chdir $working_path;
+    add_paths( $langhash_ref );
+    #my $nFound  = $#sdfparticles +1;
+    #print "\n    $nFound files found !\n";
 
     my ( $LOCALIZEPARTICLE , $localizeSDF ) = File::Temp::tempfile();
     close( $LOCALIZEPARTICLE );
@@ -425,7 +487,7 @@ sub collectfiles{
     my ( $LOCALIZE_LOG , $my_localize_log ) = File::Temp::tempfile();
     close( $LOCALIZE_LOG );
 
-    ## Get the localize de,en-US extract
+    ## Get the localize en-US extract
     if( $bAll || $bUseLocalize ){
         print "### Fetching source language strings\n";
         my $command = "";
@@ -480,7 +542,8 @@ sub collectfiles{
 
     }
     ## Get sdf particles
-   open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
+#*****************
+    open ALLPARTICLES_MERGED , "+>> $particleSDF_merged"
     or die "Can't open $particleSDF_merged";
 
     ## Fill fackback hash
@@ -542,7 +605,7 @@ sub collectfiles{
         }
     }
     close ALLPARTICLES_MERGED;
-
+#***************
 
     # Hash of array
     my %output;
@@ -1040,7 +1103,8 @@ sub parse_options{
     my $merge;
     my $extract;
     my $success = GetOptions('f=s' => \$sdffile , 'l=s' => \$languages , 's=s' => \$srcpath ,  'h' => \$help , 'v' => \$bVerbose ,
-                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date );
+                             'm' => \$merge , 'e' => \$extract , 'x' => \$no_sort , 'd' => \$use_default_date , 'c' => \$create_dirs ,
+                             'n' => \$no_gsicheck );
     $outputfile = $sdffile;
 
     #print STDOUT "DBG: lang = $languages\n";
@@ -1068,13 +1132,16 @@ sub parse_options{
     if( $extract ){ $mode = "extract"; }
     else          { $mode = "merge";   }
 }
+#my $multi_localize_files    = ''; h
+#my $module_to_merge         = ''; i
+#my $sort_sdf_before         = ''; g
 
 #########################################################
 sub usage{
 
     print STDERR "Usage: localize.pl\n";
     print STDERR "Split or collect SDF files\n";
-    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ]\n";
+    print STDERR "           merge: -m -f <sdffile>    -l l1[=f1][,l2[=f2]][...] [ -s <sourceroot> ] [ -c ]\n";
     print STDERR "         extract: -e -f <outputfile> -l <lang> [ -s <sourceroot> ] [-d]\n";
     print STDERR "Options:\n";
     print STDERR "    -h              help\n";
@@ -1085,6 +1152,11 @@ sub usage{
     print STDERR "    -s <sourceroot> Path to the modules, if no \$SRC_ROOT is set\n";
     print STDERR "    -l ( all | <isocode> | <isocode>=fallback ) comma seperated languages\n";
     print STDERR "    -d              Use default date in extracted sdf file\n";
+    print STDERR "    -c              Create needed directories\n";
+    print STDERR "    -g              Sort sdf file before mergeing\n";
+    print STDERR "    -h              File with localize.sdf's\n!";
+    print STDERR "    -n              No gsicheck\n";
+    print STDERR "    -i              Module to merge\n";
     print STDERR "    -v              Verbose\n";
     print STDERR "\nExample:\n";
     print STDERR "\nlocalize -e -l en-US,pt-BR=en-US -f my.sdf\n( Extract en-US and pt-BR with en-US fallback )\n";
diff --git a/transex3/source/merge.cxx b/transex3/source/merge.cxx
index 3546d7e3accd..ee0f6f459655 100644
--- a/transex3/source/merge.cxx
+++ b/transex3/source/merge.cxx
@@ -7,7 +7,7 @@
  * OpenOffice.org - a multi-platform office productivity suite
  *
  * $RCSfile: merge.cxx,v $
- * $Revision: 1.29 $
+ * $Revision: 1.27.36.3 $
  *
  * This file is part of OpenOffice.org.
  *
@@ -210,38 +210,39 @@ MergeDataFile::MergeDataFile( const ByteString &rFileName, const ByteString& sFi
     ByteString sTEXT;
     ByteString sQHTEXT;
     ByteString sTITLE;
+    ByteString sHACK("HACK");
 
     const ByteString sEmpty("");
 
     if( !aInputStream.IsOpen() ) {
-        printf("ERROR : Can't open %s\n", rFileName.GetBuffer());
-        exit( -1 );
+        printf("Warning : Can't open %s\n", rFileName.GetBuffer());
+        //exit( -1 );
+        return;
     }
     while ( !aInputStream.IsEof()) {
+        xub_StrLen nToks;
         aInputStream.ReadLine( sLine );
         sLine = sLine.Convert( RTL_TEXTENCODING_MS_1252, aCharSet );
 
-        if ( sLine.GetTokenCount( '\t' ) == 15  ) {
+        nToks = sLine.GetTokenCount( '\t' );
+        if ( nToks == 15 ) {
             // Skip all wrong filenames
             ByteString filename = sLine.GetToken( 1 , '\t' );
             filename = filename.Copy( filename.SearchCharBackward( "\\" )+1 , filename.Len() );
 
             if( sFile.Equals( sEmpty ) || ( !sFile.Equals( sEmpty ) && filename.Equals( sFile )  ) )
             {
-                sTYP = sLine.GetToken( 3, '\t' );
-                sGID = sLine.GetToken( 4, '\t' );
-                sLID = sLine.GetToken( 5, '\t' );
-                sPFO = sLine.GetToken( 7, '\t' );
-                sPFO = ByteString("HACK");
-                nLANG = sLine.GetToken( 9, '\t' );
-
-                sTEXT = sLine.GetToken( 10, '\t' );
-    //            printf("%s\n",sTEXT.GetBuffer());
-    //            Quote( sTEXT );
-    //            printf("%s\n",sTEXT.GetBuffer());
-
-                sQHTEXT = sLine.GetToken( 12, '\t' );
-                sTITLE = sLine.GetToken( 13, '\t' );
+              xub_StrLen rIdx = 0;
+              sTYP = sLine.GetToken( 3, '\t', rIdx );
+              sGID = sLine.GetToken( 0, '\t', rIdx ); // 4
+              sLID = sLine.GetToken( 0, '\t', rIdx ); // 5
+              sPFO = sLine.GetToken( 1, '\t', rIdx ); // 7
+              sPFO = sHACK;
+              nLANG = sLine.GetToken( 1, '\t', rIdx ); // 9
+              sTEXT = sLine.GetToken( 0, '\t', rIdx ); // 10
+
+              sQHTEXT = sLine.GetToken( 1, '\t', rIdx ); // 12
+              sTITLE = sLine.GetToken( 0, '\t', rIdx );  // 13
 
                 nLANG.EraseLeadingAndTrailingChars();
 
@@ -250,20 +251,22 @@ MergeDataFile::MergeDataFile( const ByteString &rFileName, const ByteString& sFi
 #else
                 if (  !nLANG.EqualsIgnoreCaseAscii("en-US")  ){
 #endif
-                    InsertEntry( sTYP, sGID, sLID, sPFO, nLANG, sTEXT, sQHTEXT, sTITLE , filename , bCaseSensitive );
-                    if( nLANG.Len() > 0 ){
-                        bool bFound = false;
-                        for( unsigned int x = 0; x < aLanguages.size(); x++ ){
-                            if( aLanguages[ x ].Equals( nLANG ) )
-                                bFound = true;
-                        }
+                  ByteStringHashMap::const_iterator lit;
+                  lit = aLanguageMap.find (nLANG);
+                  ByteString aLANG;
+                  if (lit == aLanguageMap.end()) {
+                    aLANG = nLANG;
+                    aLanguageMap.insert( ByteStringHashMap::value_type( aLANG, aLANG ) );
                         // Remember read languages for -l all switch
-                        if( !bFound )   aLanguages.push_back( nLANG );
-                    }
+                    aLanguageList.push_back( nLANG );
+                  } else
+                    aLANG = lit->first;
+
+                  InsertEntry( sTYP, sGID, sLID, sPFO, aLANG, sTEXT, sQHTEXT, sTITLE , filename , bCaseSensitive );
                 }
             }
         }
-        else if ( sLine.GetTokenCount( '\t' ) == 10 ){
+        else if ( nToks == 10 ) {
             printf("ERROR: File format is obsolete and no longer supported!\n");
         }
     }
@@ -286,7 +289,7 @@ ByteString MergeDataFile::Dump(){
     ByteString sRet( "MergeDataFile\n" );
 
       //sRet.Append( Export::DumpMap( "aLanguageSet" , aLanguageSet ) );
-    //sRet.Append( Export::DumpMap( "aLanguages" , aLanguages ) );
+    //sRet.Append( Export::DumpMap( "aLanguageList" , aLanguageList ) );
     printf("MergeDataFile\n");
     MergeDataHashMap::const_iterator idbg;
     for( idbg = aMap.begin() ; idbg != aMap.end(); ++idbg ){
@@ -318,7 +321,7 @@ void MergeDataFile::WriteError( const ByteString &rLine )
         fprintf( stderr, "%s\n", rLine.GetBuffer());
 }
 std::vector<ByteString> MergeDataFile::GetLanguages(){
-    return aLanguages;
+    return aLanguageList;
 }
 
 /*****************************************************************************/
@@ -379,23 +382,32 @@ void MergeDataFile::InsertEntry(
                     const ByteString &rLID, const ByteString &rPFO,
                     const ByteString &nLANG, const ByteString &rTEXT,
                     const ByteString &rQHTEXT, const ByteString &rTITLE ,
-                    const ByteString &rFilename , bool bCaseSensitive
+                    const ByteString &rInFilename , bool bCaseSensitive
                     )
 /*****************************************************************************/
 {
     MergeData *pData;
     BOOL bFound = FALSE;
 
-    // search for MergeData
+    // uniquify the filename to save memory.
+    ByteStringHashMap::const_iterator fit = aFilenames.find (rInFilename);
+    ByteString aFilename;
+    if (fit == aFilenames.end()) {
+        aFilename = rInFilename;
+        aFilenames.insert (ByteStringHashMap::value_type (aFilename, aFilename));
+    } else
+        aFilename = fit->first;
 
-    ByteString sKey = CreateKey( rTYP , rGID , rLID , rFilename , bCaseSensitive );
-    ByteString sKey2;
+    // search for MergeData
 
-    if( aMap.find( sKey ) != aMap.end() ){
-        pData = aMap[ sKey ];
+    ByteString sKey = CreateKey( rTYP , rGID , rLID , aFilename , bCaseSensitive );
+    MergeDataHashMap::const_iterator mit;
+    mit = aMap.find( sKey );
+    if( mit != aMap.end() ){
+        pData = mit->second;
     }else{
-        pData = new MergeData( rTYP, rGID, rLID , rFilename );
-        aMap.insert( MergeDataHashMap::value_type( CreateKey( rTYP , rGID , rLID , rFilename , bCaseSensitive ) , pData ) );
+        pData = new MergeData( rTYP, rGID, rLID, aFilename );
+        aMap.insert( MergeDataHashMap::value_type( sKey, pData ) );
     }
 
     bFound = FALSE;