Speed up find-german-comments

Instead of recreating a text_cat sub-process for every string, start a
single text-cat process (with the -s flag, "per line") and reuse that.

Testing on svl/source shows a speed-up of 7.5 x, down from 45.6s to 6.1s!

To avoid blocking on the pipes set autoflush in text-cat, and use
read_line instead of read on the find-german-comments side.

Change-Id: Ic6c00fc3c1e7cbcbbfff65512bf54ad11f900697
Reviewed-on: https://gerrit.libreoffice.org/9226
Reviewed-by: Michael Meeks <michael.meeks@collabora.com>
Tested-by: Michael Meeks <michael.meeks@collabora.com>

2 files changed, 16 insertions, 9 deletions

diff --git a/bin/find-german-comments b/bin/find-german-comments
index 49106119f3b7..7c5f718bd1f1 100755
--- a/bin/find-german-comments
+++ b/bin/find-german-comments
@@ -36,6 +36,7 @@ class Parser:
     """
     def __init__(self):
         self.strip = string.punctuation + " \n"
+        self.text_cat = self.start_text_cat();
         op = optparse.OptionParser()
         op.set_usage("%prog [options] <rootdir>\n\n" +
             "Searches for german comments in cxx/hxx source files inside a given root\n" +
@@ -117,19 +118,23 @@ class Parser:
                 yield (count, re.sub(".*/\*(.*)\*/.*", r"\1", i).strip(self.strip))
             count += 1
 
-    def get_lang(self, s):
-        """ the output is 'german' or 'english' or 'german or english'. when
-        unsure, just don't warn, there are strings where you just can't
-        teremine the results reliably, like '#110680#' """
+    def start_text_cat(self):
         cwd = os.getcwd()
         # change to our directory
         os.chdir(os.path.split(os.path.abspath(sys.argv[0]))[0])
-        sock = subprocess.Popen(["text_cat/text_cat", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
-        sock.stdin.write(s)
-        sock.stdin.close()
-        lang = sock.stdout.read().strip()
-        sock.stdout.close()
+        sock = subprocess.Popen(["text_cat/text_cat", "-s", "-d", "text_cat/LM"], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
         os.chdir(cwd)
+        return sock
+
+    def get_lang(self, s):
+        """ the output is 'german' or 'english' or 'german or english'. when
+        unsure, just don't warn, there are strings where you just can't
+        teremine the results reliably, like '#110680#' """
+
+        self.text_cat.stdin.write(s)
+        self.text_cat.stdin.write("\n")
+        self.text_cat.stdin.flush()
+        lang = self.text_cat.stdout.readline().strip()
         return lang
 
     def is_german(self, s):
diff --git a/bin/text_cat/text_cat b/bin/text_cat/text_cat
index 6c6b0d1d1483..c907f50b96bb 100755
--- a/bin/text_cat/text_cat
+++ b/bin/text_cat/text_cat
@@ -19,6 +19,8 @@ $opt_f ||= 0;
 $opt_t ||= 400;
 $opt_u ||= 1.05;
 
+$| = 1; # auto-flush stdout
+
 sub help {
     print <<HELP
 Text Categorization. Typically used to determine the language of a