Initial working commit.

2 files changed, 296 insertions, 0 deletions

diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..8b6d485
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/log
+ingore_list.txt
diff --git a/bibisect_build.py b/bibisect_build.py
new file mode 100755
index 0000000..056051e
--- /dev/null
+++ b/bibisect_build.py
@@ -0,0 +1,294 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+"""
+LibreOffice bibisect repo auto build
+Created on Thu Dec 28 09:29:01 2017
+Updated: 2020-02-17
+@author: Kevin Suo <suokunlong@126.com>
+
+Usage:
+    1. Clone libreoffice source codes.
+    2. Define the variables within this python code below.
+    3. Change "start_commit" and "to_commit" to the range you want to build.
+    4. By default, I use "step = 3" below, which means we build one bibisect
+       commit for every 3 source commits. You can change this to meet your own needs.
+    5. Run.
+
+"""
+
+import os
+import time
+from datetime import datetime
+from subprocess import Popen,PIPE, STDOUT
+
+#---------------------------------------------
+# Please define these variables before run:
+#---------------------------------------------
+
+# location of the source code
+source = "/mnt/data/lo/source/ref-master-bibisect"
+
+# location of the dir which will hold the bibisect git repo.
+target = "/mnt/data/lo/bibisect-repos/bibisect-master"
+
+# whether we should push each bibisect commit to a bare repo.
+push_to_bare = True
+# location of the git bare repo to be pushed to.
+target_bare = "/mnt/data/git/lo/bibisect-linux64-master-200117-current"
+
+# dir in which the log files to be saved
+log_dir = "/mnt/data/lo/bibisect_tool/log"
+
+
+# The range of source commits to be built.
+# You do not need to change "start_commit" in each run of this script.
+# Commits whcih are already included in the "target" will be automatically ignored.
+
+# 2020-02-08 22:53:48 +0100  Removed executable permission on file
+start_commit = "dd0dda17c9984ce29b674f7ba903cd1d3fe73ab5"
+
+# 2020-02-17 02:55:36 +0100  Fix UITest_options: changed property name
+to_commit = "592bfa4d21e22ca5aa0ce228388558c52941b81a"
+
+# The file holding a list of source commits to be ignored in this process,
+# the format is: each commit hash one line.
+# These are usually source commits which have no impact on our bibisect result,
+# such as update git submodules, unit tests / UI tests etc..
+# It is a waste of time to build these commits.
+ingore_list_file = "/mnt/data/lo/bibisect_tool/ingore_list.txt"
+
+# For the first run, we will initiate a git repo, 
+# so you need to put your user name and email here:
+git_name = "Kevin Suo"
+git_email = "suokunlong@126.com"
+
+# By default, we do the build for every 3 source commits, to save disk sapce of
+# the bibisect repo. You can change this value.
+step = 3
+
+# Stop editing.
+
+def run_process(cmd, log=None, wait=True):
+    """Run the process as specified in cmd.
+    Return list [returnCode, stdout]
+    """
+    if log:
+        log.write(f"Running Process: {cmd}\n")
+    print(f"{cmd}")
+    
+    p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, encoding='utf8')
+    if wait:
+        while True:
+            output = p.stdout.readline()
+            rc = p.poll()
+            if output == '' and rc is not None:
+                break
+            if output:
+                output = output.strip()
+                if log:
+                    log.write(output + "\n")
+                else:
+                    pass
+                    # print(output)
+            time.sleep(0.01)
+            if log:
+                log.flush()
+        if rc != 0:
+            raise Exception(f"Return code for this process is {rc}")
+    else:
+        rc = 0
+    
+    return rc
+
+def generate_range_list(range_text):
+    """Process the range text to a list of commits.
+    Return: [commit_date, commit_time, commit_zone, commit_hash, commit_msg]
+    """
+    range_list = range_text.split(b"\n")
+    commits = []
+    for range_item in range_list:
+        commit_date, commit_time, commit_zone, commit_hash, commit_msg = \
+            range_item.split(b" ", maxsplit=4)
+        commit_date, commit_time, commit_zone, commit_hash, commit_msg = \
+            commit_date.decode(), commit_time.decode(), commit_zone.decode(), \
+            commit_hash.decode(), commit_msg.decode()
+        commits.append([commit_date, commit_time, commit_zone, commit_hash, commit_msg])
+    
+    return commits
+
+def commit_in_target(hash_str, target):
+    """
+    """
+    cmd = f"git -C {target} log --pretty='%s'"
+    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, encoding='utf8')
+    stdout, stderr = p.communicate()
+    lines = stdout.strip().splitlines()
+    
+    target_source_hashes = []
+    for line in lines:
+        source_info = line.split(sep=" ", maxsplit=4)
+        try:
+        	target_source_hashes.append(source_info[3])
+        except IndexError:
+            pass
+        
+    if hash_str in target_source_hashes:
+        return True
+    else:
+        return False
+
+def generate_ingnore_list(ingore_list_file):
+    """Return a list of source commits to be ingored before build
+    """
+    with open(ingore_list_file) as f:
+        content = f.read().strip().split("\n")
+    
+    ingore_list = [i[:12] for i in content]
+    
+    return ingore_list
+
+def exclude_ignore(commits):
+    ret = []
+    ignore_list = generate_ingnore_list(ingore_list_file)
+    for i in range(0,len(commits)):
+        if commits[i][3][:12] not in ignore_list:
+            ret.append(commits[i])
+        else:
+            pass
+        
+    return ret
+
+        
+if __name__ == "__main__":
+
+    os.chdir(source)
+    print(f"Changed to dir {source}")
+    
+    # Get the commits to be built.
+    cmd = f"git log --reverse {start_commit}..{to_commit} --format='%ci %H %s' --boundary"
+    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
+    stdout, stderr = p.communicate()
+    rc = p.poll()
+    if rc != 0:
+        raise Exception(stderr)
+    range_text = stdout.strip()
+    commits = generate_range_list(range_text)
+    
+    if not commits:
+        exit(0)
+        
+    # Create target git repo if not exists.
+    if not os.path.exists(os.path.join(target, ".git")):
+        print("Iniating target git repo...")
+        if not os.path.exists(target):
+            os.mkdir(target)
+        cmd = f"git init {repr(target)}"
+        run_process(cmd)
+        cmd = f"git -C {repr(target)} config user.name {repr(git_name)}"
+        run_process(cmd)
+        cmd = f"git -C {repr(target)} config user.email {repr(git_email)}"
+        run_process(cmd)
+        
+    commits_to_be_built = exclude_ignore(commits)
+    commits_to_be_built = commits_to_be_built[::step]
+    # always include the last commit
+    if commits_to_be_built[-1] != commits[-1]:
+        commits_to_be_built.append(commits[-1])
+    
+    i = 0
+    fail_count = 0
+    for commit in commits_to_be_built:
+        i += 1
+        commit_date, commit_time, commit_zone, commit_hash, commit_msg = commit
+        
+        # Rescan the ignore list for each commit to be built
+        # new ignores can be added to the list at any time.
+        ignore_list = generate_ingnore_list(ingore_list_file)
+        if commit[:12] in ignore_list:
+            print(f"{commit} newly detected in ignore list, pass.")
+            continue
+        
+        if commit_in_target(commit_hash, target):
+            print(f"{commit_hash} already in target.")
+            continue
+        else:
+            print(f"\nBuilding {commit_hash}, {str(i)} of {str(len(commits_to_be_built))}...")
+            
+        the_datetime = datetime.now().strftime("%Y-%m-%d-%H%M%S")
+        log_file = os.path.join(log_dir, "lo-build-" + the_datetime + "-" + commit_hash + ".log")
+        try:
+            os.makedirs(log_dir)
+        except:
+            pass
+        log = open(log_file, "w")
+        msg = commit_date + " " + commit_time+ " " + commit_zone+ " " + commit_hash
+        msg_body = commit_msg.replace("\"", "'")
+        log.write("\n-----\n" + msg + "\n" + msg_body + "\n-----\n")
+        log.write(the_datetime + "\n")
+        print(f"{the_datetime}: {commit_date} - {commit_msg}")
+        
+        # Checkout
+        rc = run_process(f"./g checkout {commit_hash}", log=log)
+        if rc != 0:
+            raise Exception(f"Error when checking out {commit_hash}")
+        
+        # Build
+        try:
+            run_process("./autogen.sh && make build-nocheck", log=log)
+        except KeyboardInterrupt:
+            exit(0)
+        except:
+            print(f"Build error in commit {commit_hash}")
+            print("Second try:")
+            run_process("make clean", log=log)
+            try:
+                rc = run_process("./autogen.sh && make build-nocheck", log=log)
+            except KeyboardInterrupt:
+                exit(0)
+            except:
+                fail_count += 1
+                if fail_count >= 5:
+                    print("5 fail builds, aborted.")
+                    exit(0)
+                else:
+                    print("Build failed, ignored.")
+                    continue
+        
+        # build successfull, reset fail_count.
+        fail_count = 0
+        
+        # Checkout master on the bibisect repo
+        cmd = f"git -C {repr(target)} checkout master"
+        try:
+            run_process(cmd, log=log)
+        except:
+            pass
+        
+        # Sync instdir to target repo
+        cmd = f"rsync -vrpEogtl --delete {source}/instdir {target}/"
+        run_process(cmd, log=log)
+        
+        # Copy log file to target.
+        log.close()
+        cmd = f"cp -f {repr(log_file)} {repr(target)}/build.log"
+        run_process(cmd)
+        
+        # Commit the new build
+        cmd = f"git -C {repr(target)} add . && git -C {repr(target)} commit -m {repr(msg)} -m {repr(msg_body)}"
+        run_process(cmd)
+        
+        # Add tag
+        tag_name = f"source-hash-{commit_hash}"
+        cmd = f"git -C {repr(target)} tag {tag_name}"
+        try:
+            run_process(cmd)
+        except:
+            cmd = f"git -C {repr(target)} tag -d {tag_name}"; run_process(cmd)
+            cmd = f"git -C {repr(target)} tag {tag_name}"; run_process(cmd)
+        
+        # push to bare repo
+        if push_to_bare:
+            cmd = f"git -C {repr(target)} push {repr(target_bare)} master --tags"
+            run_process(cmd)
+    
+    print("All Done!")