tdf#130911: convert some token generation scripts from Perl to Python.

Change-Id: Id655b6a0cee7bdfe4804941f20fe358af8f3185e
Reviewed-on: https://gerrit.libreoffice.org/c/core/+/89477
Tested-by: Jenkins
Reviewed-by: Mike Kaganski <mike.kaganski@collabora.com>
Reviewed-by: Michael Stahl <michael.stahl@cib.de>

3 files changed, 83 insertions, 71 deletions

diff --git a/solenv/bin/generate-tokens.pl b/solenv/bin/generate-tokens.pl
deleted file mode 100644
index d5f8f9fa7d1f..000000000000
--- a/solenv/bin/generate-tokens.pl
+++ /dev/null
@@ -1,69 +0,0 @@
-#
-# This file is part of the LibreOffice project.
-#
-# This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this
-# file, You can obtain one at http://mozilla.org/MPL/2.0/.
-#
-# This file incorporates work covered by the following license notice:
-#
-#   Licensed to the Apache Software Foundation (ASF) under one or more
-#   contributor license agreements. See the NOTICE file distributed
-#   with this work for additional information regarding copyright
-#   ownership. The ASF licenses this file to you under the Apache
-#   License, Version 2.0 (the "License"); you may not use this file
-#   except in compliance with the License. You may obtain a copy of
-#   the License at http://www.apache.org/licenses/LICENSE-2.0 .
-#
-
-$ARGV0 = shift @ARGV;
-$ARGV1 = shift @ARGV;
-$ARGV2 = shift @ARGV;
-$ARGV3 = shift @ARGV;
-
-open ( IDFILE, ">$ARGV1" ) or die "Error: cannot open output file: $!";
-open ( NAMEFILE, ">$ARGV2" ) or die "Error: cannot open output file: $!";
-open ( GPERFFILE, ">$ARGV3" ) or die "Error: cannot open output file: $!";
-
-print( GPERFFILE "%language=C++\n" );
-print( GPERFFILE "%global-table\n" );
-print( GPERFFILE "%null-strings\n" );
-print( GPERFFILE "%struct-type\n" );
-print( GPERFFILE "struct xmltoken {\n" );
-print( GPERFFILE "    const sal_Char *name;\n" );
-print( GPERFFILE "    sal_Int32 nToken;\n" );
-print( GPERFFILE "};\n" );
-print( GPERFFILE "%%\n" );
-
-open( INFILE, $ARGV0 ) or die "Error: cannot open input file: $!";
-
-$i = 0;
-while ( <INFILE> )
-{
-    # trim newline
-    chomp( $_ );
-    # trim leading/trailing whitespace
-    $_ =~ s/^\s*//g;
-    $_ =~ s/\s*$//g;
-    # check for valid characters
-    $_ =~ /^[a-zA-Z0-9-_]+$/ or die "Error: invalid character in token '$_'";
-    $id = "XML_$_";
-    # we have two ids with similar names("cut-offs" and "cut_offs")
-    if ($id eq "XML_cut_offs") {
-        $id = "cut_offs2";
-    }
-    $id =~ s/-/_/g;
-    $tokens{$_} = $id;
-    print( IDFILE "const sal_Int32 $id = $i;\n" );
-    print( NAMEFILE "\"$_\",\n" );
-    print( GPERFFILE "$_,$id\n" );
-    ++$i;
-}
-close ( INFILE );
-
-print( IDFILE "const sal_Int32 XML_TOKEN_COUNT = $i;\n" );
-print( GPERFFILE "%%\n" );
-
-close( IDFILE );
-close( NAMEFILE );
-close( GPERFFILE );
diff --git a/solenv/bin/generate-tokens.py b/solenv/bin/generate-tokens.py
new file mode 100644
index 000000000000..3fe929092304
--- /dev/null
+++ b/solenv/bin/generate-tokens.py
@@ -0,0 +1,80 @@
+# This file is part of the LibreOffice project.
+#
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+#
+# This file incorporates work covered by the following license notice:
+#
+#   Licensed to the Apache Software Foundation (ASF) under one or more
+#   contributor license agreements. See the NOTICE file distributed
+#   with this work for additional information regarding copyright
+#   ownership. The ASF licenses this file to you under the Apache
+#   License, Version 2.0 (the "License"); you may not use this file
+#   except in compliance with the License. You may obtain a copy of
+#   the License at http://www.apache.org/licenses/LICENSE-2.0 .
+#
+
+import sys, re
+
+infile_name = sys.argv[1]
+idfile_out_name = sys.argv[2]
+namefile_out_name = sys.argv[3]
+gperffile_out_name = sys.argv[4]
+
+idfile = open(idfile_out_name, 'w')
+namefile = open(namefile_out_name, 'w')
+gperffile = open(gperffile_out_name, 'w')
+
+gperffile.write("""%language=C++
+%global-table
+%null-strings
+%struct-type
+struct xmltoken {
+    const sal_Char *name;
+    sal_Int32 nToken;
+};
+%%
+""")
+
+token_count = 0;
+tokens = {}
+
+with open(infile_name) as infile:
+    for line in infile:
+        line = line.strip()
+        # check for valid characters
+        if not re.match(r'[a-zA-Z0-9-_]+$', line):
+            sys.exit("Error: invalid character in token '{}'".format(line));
+        cur_id = "XML_" + line;
+        # we have two ids with similar names("cut-offs" and "cut_offs")
+        if cur_id == "XML_cut_offs":
+            cur_id = "cut_offs2";
+        cur_id = cur_id.replace('-', '_')
+        tokens[line] = cur_id
+        idfile.write("const sal_Int32 {} = {};\n".format(cur_id, token_count))
+        namefile.write("\"{}\",\n".format(line));
+        gperffile.write("{},{}\n".format(line, cur_id));
+        token_count += 1
+
+idfile.write("const sal_Int32 XML_TOKEN_COUNT = {};\n".format(token_count))
+gperffile.write("%%\n")
+
+idfile.close()
+namefile.close()
+gperffile.close()
+
+def fix_linefeeds(fname):
+    # Gperf requires LF newlines, not CRLF, even on Windows.
+    # Making this work on both Python 2 and 3 is difficult.
+    # When Python 2 is dropped, delete this and add
+    # newline = '\n' to the open() calls above.
+    with open(fname, 'rb') as ifile:
+        d = ifile.read()
+    d = d.replace(b'\r', b'')
+    with open(fname, 'wb') as ofile:
+        ofile.write(d)
+
+fix_linefeeds(idfile_out_name)
+fix_linefeeds(namefile_out_name)
+fix_linefeeds(gperffile_out_name)
diff --git a/solenv/gbuild/CustomTarget.mk b/solenv/gbuild/CustomTarget.mk
index 27ce89fba06f..ed1a9addc103 100644
--- a/solenv/gbuild/CustomTarget.mk
+++ b/solenv/gbuild/CustomTarget.mk
@@ -100,8 +100,9 @@ $(if $(7),$(call gb_CustomTarget_get_workdir,$(1))/$(7)names.inc) : \
 	touch $$@
 
 $(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx : \
+		$(call gb_ExternalExecutable_get_dependencies,python) \
 		$(if $(7),$(SRCDIR)/$(3)/$(7).txt) \
-		$(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.pl) \
+		$(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.py) \
 		$(SRCDIR)/$(3)/$(4).txt \
 		$(SRCDIR)/$(3)/$(4).hxx.head \
 		$(SRCDIR)/$(3)/$(4).hxx.tail
@@ -110,7 +111,7 @@ $(call gb_CustomTarget_get_workdir,$(1))/$(2)/token/$(4).hxx : \
 	mkdir -p $(call gb_CustomTarget_get_workdir,$(1))/misc \
 	    	$(call gb_CustomTarget_get_workdir,$(1)) \
 		$(call gb_CustomTarget_get_workdir,$(1))/$(2)/token
-	perl $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.pl) \
+	$(call gb_ExternalExecutable_get_command,python) $(if $(8),$(SRCDIR)/$(3)/$(8),$(SRCDIR)/solenv/bin/generate-tokens.py) \
 	    	$(SRCDIR)/$(3)/$(4).txt \
 		$(call gb_CustomTarget_get_workdir,$(1))/misc/$(5)ids.inc \
 		$(call gb_CustomTarget_get_workdir,$(1))/$(5)names.inc \