Add basic multi-threading for each mimetype

Change-Id: Ife0766ddd259bb7d86a9c7bdcf3e9c2849208cf0
Reviewed-on: https://gerrit.libreoffice.org/7123
Reviewed-by: Markus Mohrhard <markus.mohrhard@googlemail.com>
Tested-by: Markus Mohrhard <markus.mohrhard@googlemail.com>

1 files changed, 50 insertions, 8 deletions

diff --git a/bin/get-bugzilla-attachments-by-mimetype b/bin/get-bugzilla-attachments-by-mimetype
index 5f061c625cd0..3480713f7f9c 100755
--- a/bin/get-bugzilla-attachments-by-mimetype
+++ b/bin/get-bugzilla-attachments-by-mimetype
@@ -27,6 +27,7 @@ import re
 import os, os.path
 import stat
 import sys
+import threading, Queue
 try:
     from urllib.request import urlopen
 except:
@@ -450,14 +451,55 @@ common_noncore_mimetypes = {
     'application/pdf': 'pdf',
 }
 
-for (prefix, uri) in rss_bugzillas.items():
-    for (mimetype,extension) in mimetypes.items():
-        # It seems that bugzilla has problems returing that many results
-        # (10000 results is probably a limit set somewhere) so we always
-        # end processing the complete list.
-        if mimetype == 'text/html' and prefix == 'moz':
-                continue
-        get_through_rss_query(uri, mimetype, prefix, extension)
+class manage_threads(threading.Thread):
+    def run(self):
+        #print(threading.current_thread().get_ident())
+        while 1:
+            # Try to receive a job from queue
+            try:
+                # Get job from queue
+                # Use job parameters to call our query
+                # Then let the queue know we are done with this job
+                job = jobs.get(True,5)
+                get_through_rss_query(job[0], job[1], job[2], job[3]) # [0] = uri; [1] = mimetype; [2] = prefix; [3] = extension
+                jobs.task_done()
+            except KeyboardInterrupt:
+                raise # Ctrl+C should work
+            except:
+                break
+
+def generate_multi_threading():
+    for (prefix, uri) in rss_bugzillas.items():
+
+        # Initialize threads
+        for i in xrange(max_threads):
+            manage_threads().start()
+
+        # Create a job for every mimetype for a bugzilla
+        for (mimetype,extension) in mimetypes.items():
+
+
+            # It seems that bugzilla has problems returing that many results
+            # (10000 results is probably a limit set somewhere) so we always
+            # end processing the complete list.
+            if mimetype == 'text/html' and prefix == 'moz':
+                    continue
+
+            try:
+                jobs.put([uri, mimetype, prefix, extension], block=True, timeout=3)
+                print("successfully placed a job in the queue searching for " + mimetype + "in bugtracker " + prefix)
+            except KeyboardInterrupt:
+                raise # Ctrl+C should work
+            except:
+                print("Queue full")
+
+        # Continue when all mimetypes are done for a bugzilla
+        jobs.join()
+
+max_threads = 20 # Number of threads to create, (1 = without multi-threading)
+jobs = Queue.Queue(40)
+
+generate_multi_threading()
 
 for (mimetype,extension) in mimetypes.items():
     get_through_rpc_query(redhatrpc, redhatbug, mimetype, "rhbz", extension)