aboutsummaryrefslogtreecommitdiff
path: root/bibisect_build.py
blob: 2d69a05322fbac1972dfae082cf417a6d746c0ce (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
LibreOffice bibisect repo auto build
Created on Thu Dec 28 09:29:01 2017
Updated: 2020-02-17
@author: Kevin Suo <suokunlong@126.com>

Usage:
    1. Clone libreoffice source codes.
    2. Define the variables within this python code below.
    3. Change "start_commit" and "to_commit" to the range you want to build.
    4. By default, I use "step = 3" below, which means we build one bibisect
       commit for every 3 source commits. You can change this to meet your own needs.
    5. Run.

"""

import os
import time
from datetime import datetime
from subprocess import Popen,PIPE, STDOUT

#---------------------------------------------
# Please define these variables before run:
#---------------------------------------------

# location of the source code
source = "/mnt/data/lo/source/ref-master-bibisect"

# location of the dir which will hold the bibisect git repo.
target = "/mnt/data/lo/bibisect-repos/bibisect-master"

# whether we should push each bibisect commit to a bare repo.
push_to_bare = True
# location of the git bare repo to be pushed to.
target_bare = "/mnt/data/git/lo/bibisect-linux-64-7.0-CN"

# dir in which the log files to be saved
log_dir = "/mnt/data/lo/bibisect_tool/log"

# The range of source commits to be built.
# You do not need to change "start_commit" in each run of this script.
# Commits whcih are already included in the "target" will be automatically ignored.

# 2020-02-21 12:17:32 +0100 tdf#130517 improve accelerators on Print dialog page
start_commit = "5c3604542191b3c69da2d9d912c5c5a20c7143e9"

# master
to_commit = "master"

# The file holding a list of source commits to be ignored in this process,
# the format is: each commit hash one line.
# These are usually source commits which have no impact on our bibisect result,
# such as update git submodules, unit tests / UI tests etc..
# It is a waste of time to build these commits.
igore_list_file = "/mnt/data/lo/bibisect_tool/igore_list.txt"

# For the first run, we will initiate a git repo, 
# so you need to put your user name and email here:
git_name = "Kevin Suo"
git_email = "suokunlong@126.com"

# By default, we do the build for every 3 source commits, to save disk sapce of
# the bibisect repo. You can change this value.
step = 3

# Stop editing.

def run_process(cmd, log=None, wait=True):
    """Run the process as specified in cmd.
    Return list [returnCode, stdout]
    """
    if log:
        log.write(f"Running Process: {cmd}\n")
    print(f"{cmd}")
    
    p = Popen(cmd, shell=True, stdout=PIPE, stderr=STDOUT, encoding='utf8')
    if wait:
        while True:
            output = p.stdout.readline()
            rc = p.poll()
            if output == '' and rc is not None:
                break
            if output:
                output = output.strip()
                if log:
                    log.write(output + "\n")
                else:
                    pass
                    # print(output)
            time.sleep(0.01)
            if log:
                log.flush()
        if rc != 0:
            raise Exception(f"Return code for this process is {rc}")
    else:
        rc = 0
    
    return rc

def generate_range_list(range_text):
    """Process the range text to a list of commits.
    Return: [commit_date, commit_time, commit_zone, commit_hash, commit_msg]
    """
    range_list = range_text.split(b"\n")
    commits = []
    ignore_msgs = [
        "drop unneeded include",
        "Update git submodules",
        "Unit test",
        "unittest",
        "UItest",
        "UI Test",
        "cppcheck",
        "macos",
        "mac os",
        "MAC_OS",
        "android",
        "mobile",
        "update credits",
        "autocorrect",
        "Fix typo",
        "pragma once",
        "executable permission",]
    ignore_list = generate_ignore_list(igore_list_file)
    for range_item in range_list:
        commit_date, commit_time, commit_zone, commit_hash, commit_msg = \
            range_item.split(b" ", maxsplit=4)
        commit_date, commit_time, commit_zone, commit_hash, commit_msg = \
            commit_date.decode(), commit_time.decode(), commit_zone.decode(), \
            commit_hash.decode(), commit_msg.decode()
        ignore = False
        # Decide ignore based on commit message
        for ignore_msg in ignore_msgs:
            if ignore_msg.upper() in commit_msg.upper():
                ignore = True
                break
        if not ignore:
            # Also ignore if the commit is in the ignore_list_file
            if commit_hash[:12] in ignore_list:
                # ignore = True
                continue
            else:
                commits.append([commit_date, commit_time, commit_zone, commit_hash, commit_msg])
    
    # One build for each "step" source commits
    commits_to_be_build = commits[::step]
    # Always include the last commit in the build.
    if commits_to_be_build[-1] != commits[-1]:
        commits_to_be_build.append(commits[-1])
    
    return commits_to_be_build

def commit_in_target(hash_str, target):
    """
    """
    cmd = f"git -C {target} log --pretty='%s'"
    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, encoding='utf8')
    stdout, stderr = p.communicate()
    lines = stdout.strip().splitlines()
    
    target_source_hashes = []
    for line in lines:
        source_info = line.split(sep=" ", maxsplit=4)
        try:
        	target_source_hashes.append(source_info[3])
        except IndexError:
            pass
        
    if hash_str in target_source_hashes:
        return True
    else:
        return False

def generate_ignore_list(igore_list_file):
    """Return a list of source commits to be igored before build
    """
    with open(igore_list_file) as f:
        content = f.read().strip().split("\n")
    
    igore_list = [i[:12] for i in content]
    
    return igore_list
        
if __name__ == "__main__":

    os.chdir(source)
    print(f"Changed to dir {source}")
    
    # Get the commits to be built.
    cmd = f"git log --reverse {start_commit}..{to_commit} --format='%ci %H %s' --boundary"
    p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
    stdout, stderr = p.communicate()
    rc = p.poll()
    if rc != 0:
        raise Exception(stderr)
    range_text = stdout.strip()
    commits_to_be_built = generate_range_list(range_text)
    
    if not commits_to_be_built:
        print("No commits to be build, exiting...")
        print(commits_to_be_built)
        exit(0)
        
    # Create target git repo if not exists.
    if not os.path.exists(os.path.join(target, ".git")):
        print("Iniating target git repo...")
        if not os.path.exists(target):
            os.mkdir(target)
        cmd = f"git init {repr(target)}"
        run_process(cmd)
        cmd = f"git -C {repr(target)} config user.name {repr(git_name)}"
        run_process(cmd)
        cmd = f"git -C {repr(target)} config user.email {repr(git_email)}"
        run_process(cmd)
    
    i = 0
    fail_count = 0
    for commit in commits_to_be_built:
        i += 1
        commit_date, commit_time, commit_zone, commit_hash, commit_msg = commit
        
        # Rescan the ignore list for each commit to be built.
        # New ignores can be added to the list at any time.
        ignore_list = generate_ignore_list(igore_list_file)
        if commit_hash[:12] in ignore_list:
            print(f"{commit_hash} newly detected in ignore list, pass.")
            continue
        
        if commit_in_target(commit_hash, target):
            print(f"{commit_hash} already in target.")
            continue
        else:
            print(f"\nBuilding {commit_hash}, {str(i)} of {str(len(commits_to_be_built))}...")
            
        the_datetime = datetime.now().strftime("%Y-%m-%d-%H%M%S")
        log_file = os.path.join(log_dir, "lo-build-" + the_datetime + "-" + commit_hash + ".log")
        try:
            os.makedirs(log_dir)
        except:
            pass
        log = open(log_file, "w")
        msg = commit_date + " " + commit_time+ " " + commit_zone+ " " + commit_hash
        msg_body = commit_msg.replace("\"", "'")
        log.write("\n-----\n" + msg + "\n" + msg_body + "\n-----\n")
        log.write(the_datetime + "\n")
        print(f"{the_datetime}: {commit_date} - {commit_msg}")
        
        # Checkout
        rc = run_process(f"./g checkout {commit_hash}", log=log)
        if rc != 0:
            raise Exception(f"Error when checking out {commit_hash}")
        
        # Build
        try:
            run_process("./autogen.sh && make build-nocheck", log=log)
        except KeyboardInterrupt:
            exit(0)
        except:
            print(f"Build error in commit {commit_hash}")
            print("Second try:")
            run_process("make clean", log=log)
            try:
                rc = run_process("./autogen.sh && make build-nocheck", log=log)
            except KeyboardInterrupt:
                exit(0)
            except:
                fail_count += 1
                if fail_count >= 5:
                    print("5 fail builds, aborted.")
                    exit(0)
                else:
                    print("Build failed, ignored.")
                    continue
        
        # build successfull, reset fail_count.
        fail_count = 0
        
        # Checkout master on the bibisect repo
        cmd = f"git -C {repr(target)} checkout master"
        try:
            run_process(cmd, log=log)
        except:
            pass
        
        # Sync instdir to target repo
        cmd = f"rsync -vrpEogtl --delete {source}/instdir {target}/"
        run_process(cmd, log=log)
        
        # Copy log file to target.
        log.close()
        cmd = f"cp -f {repr(log_file)} {repr(target)}/build.log"
        run_process(cmd)
        
        # Commit the new build
        cmd = f"git -C {repr(target)} add . && git -C {repr(target)} commit -m {repr(msg)} -m {repr(msg_body)}"
        run_process(cmd)
        
        # Add tag
        #tag_name = f"source-hash-{commit_hash}"
        #cmd = f"git -C {repr(target)} tag {tag_name}"
        #try:
        #    run_process(cmd)
        #except:
        #    cmd = f"git -C {repr(target)} tag -d {tag_name}"; run_process(cmd)
        #    cmd = f"git -C {repr(target)} tag {tag_name}"; run_process(cmd)
        
        # push to bare repo
        if push_to_bare:
            cmd = f"git -C {repr(target)} push {repr(target_bare)} master --tags"
            run_process(cmd)

        print("Sleeping for 5s, type  Ctrl-C to terminate this script")
        time.sleep(5)
    
    print("All Done!")