X-Git-Url: https://git.kernelconcepts.de/?p=karo-tx-uboot.git;a=blobdiff_plain;f=tools%2Fbuildman%2Fbuilder.py;h=1b6517b48849baa9e8aeda73470dff4f7aae2221;hp=0a3900c2f251f6ec07005396d8a726da15b3de1c;hb=4653a8826f33df981a815f91fef7972ec1b37833;hpb=302e609fe653baf1ae3a7573d2f4eafd86ab696b diff --git a/tools/buildman/builder.py b/tools/buildman/builder.py index 0a3900c2f2..1b6517b488 100644 --- a/tools/buildman/builder.py +++ b/tools/buildman/builder.py @@ -6,7 +6,6 @@ # import collections -import errno from datetime import datetime, timedelta import glob import os @@ -15,12 +14,13 @@ import Queue import shutil import string import sys -import threading import time +import builderthread import command import gitutil import terminal +from terminal import Print import toolchain @@ -97,409 +97,6 @@ OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4) trans_valid_chars = string.maketrans("/: ", "---") -def Mkdir(dirname): - """Make a directory if it doesn't already exist. - - Args: - dirname: Directory to create - """ - try: - os.mkdir(dirname) - except OSError as err: - if err.errno == errno.EEXIST: - pass - else: - raise - -class BuilderJob: - """Holds information about a job to be performed by a thread - - Members: - board: Board object to build - commits: List of commit options to build. - """ - def __init__(self): - self.board = None - self.commits = [] - - -class ResultThread(threading.Thread): - """This thread processes results from builder threads. - - It simply passes the results on to the builder. There is only one - result thread, and this helps to serialise the build output. - """ - def __init__(self, builder): - """Set up a new result thread - - Args: - builder: Builder which will be sent each result - """ - threading.Thread.__init__(self) - self.builder = builder - - def run(self): - """Called to start up the result thread. - - We collect the next result job and pass it on to the build. - """ - while True: - result = self.builder.out_queue.get() - self.builder.ProcessResult(result) - self.builder.out_queue.task_done() - - -class BuilderThread(threading.Thread): - """This thread builds U-Boot for a particular board. - - An input queue provides each new job. We run 'make' to build U-Boot - and then pass the results on to the output queue. - - Members: - builder: The builder which contains information we might need - thread_num: Our thread number (0-n-1), used to decide on a - temporary directory - """ - def __init__(self, builder, thread_num): - """Set up a new builder thread""" - threading.Thread.__init__(self) - self.builder = builder - self.thread_num = thread_num - - def Make(self, commit, brd, stage, cwd, *args, **kwargs): - """Run 'make' on a particular commit and board. - - The source code will already be checked out, so the 'commit' - argument is only for information. - - Args: - commit: Commit object that is being built - brd: Board object that is being built - stage: Stage of the build. Valid stages are: - distclean - can be called to clean source - config - called to configure for a board - build - the main make invocation - it does the build - args: A list of arguments to pass to 'make' - kwargs: A list of keyword arguments to pass to command.RunPipe() - - Returns: - CommandResult object - """ - return self.builder.do_make(commit, brd, stage, cwd, *args, - **kwargs) - - def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build, - force_build_failures): - """Build a particular commit. - - If the build is already done, and we are not forcing a build, we skip - the build and just return the previously-saved results. - - Args: - commit_upto: Commit number to build (0...n-1) - brd: Board object to build - work_dir: Directory to which the source will be checked out - do_config: True to run a make _config on the source - force_build: Force a build even if one was previously done - force_build_failures: Force a bulid if the previous result showed - failure - - Returns: - tuple containing: - - CommandResult object containing the results of the build - - boolean indicating whether 'make config' is still needed - """ - # Create a default result - it will be overwritte by the call to - # self.Make() below, in the event that we do a build. - result = command.CommandResult() - result.return_code = 0 - if self.builder.in_tree: - out_dir = work_dir - else: - out_dir = os.path.join(work_dir, 'build') - - # Check if the job was already completed last time - done_file = self.builder.GetDoneFile(commit_upto, brd.target) - result.already_done = os.path.exists(done_file) - will_build = (force_build or force_build_failures or - not result.already_done) - if result.already_done and will_build: - # Get the return code from that build and use it - with open(done_file, 'r') as fd: - result.return_code = int(fd.readline()) - err_file = self.builder.GetErrFile(commit_upto, brd.target) - if os.path.exists(err_file) and os.stat(err_file).st_size: - result.stderr = 'bad' - elif not force_build: - # The build passed, so no need to build it again - will_build = False - - if will_build: - # We are going to have to build it. First, get a toolchain - if not self.toolchain: - try: - self.toolchain = self.builder.toolchains.Select(brd.arch) - except ValueError as err: - result.return_code = 10 - result.stdout = '' - result.stderr = str(err) - # TODO(sjg@chromium.org): This gets swallowed, but needs - # to be reported. - - if self.toolchain: - # Checkout the right commit - if commit_upto is not None: - commit = self.builder.commits[commit_upto] - if self.builder.checkout: - git_dir = os.path.join(work_dir, '.git') - gitutil.Checkout(commit.hash, git_dir, work_dir, - force=True) - else: - commit = self.builder.commit # Ick, fix this for BuildCommits() - - # Set up the environment and command line - env = self.toolchain.MakeEnvironment() - Mkdir(out_dir) - args = [] - if not self.builder.in_tree: - args.append('O=build') - args.append('-s') - if self.builder.num_jobs is not None: - args.extend(['-j', str(self.builder.num_jobs)]) - config_args = ['%s_config' % brd.target] - config_out = '' - args.extend(self.builder.toolchains.GetMakeArguments(brd)) - - # If we need to reconfigure, do that now - if do_config: - result = self.Make(commit, brd, 'distclean', work_dir, - 'distclean', *args, env=env) - result = self.Make(commit, brd, 'config', work_dir, - *(args + config_args), env=env) - config_out = result.combined - do_config = False # No need to configure next time - if result.return_code == 0: - result = self.Make(commit, brd, 'build', work_dir, *args, - env=env) - result.stdout = config_out + result.stdout - else: - result.return_code = 1 - result.stderr = 'No tool chain for %s\n' % brd.arch - result.already_done = False - - result.toolchain = self.toolchain - result.brd = brd - result.commit_upto = commit_upto - result.out_dir = out_dir - return result, do_config - - def _WriteResult(self, result, keep_outputs): - """Write a built result to the output directory. - - Args: - result: CommandResult object containing result to write - keep_outputs: True to store the output binaries, False - to delete them - """ - # Fatal error - if result.return_code < 0: - return - - # Aborted? - if result.stderr and 'No child processes' in result.stderr: - return - - if result.already_done: - return - - # Write the output and stderr - output_dir = self.builder._GetOutputDir(result.commit_upto) - Mkdir(output_dir) - build_dir = self.builder.GetBuildDir(result.commit_upto, - result.brd.target) - Mkdir(build_dir) - - outfile = os.path.join(build_dir, 'log') - with open(outfile, 'w') as fd: - if result.stdout: - fd.write(result.stdout) - - errfile = self.builder.GetErrFile(result.commit_upto, - result.brd.target) - if result.stderr: - with open(errfile, 'w') as fd: - fd.write(result.stderr) - elif os.path.exists(errfile): - os.remove(errfile) - - if result.toolchain: - # Write the build result and toolchain information. - done_file = self.builder.GetDoneFile(result.commit_upto, - result.brd.target) - with open(done_file, 'w') as fd: - fd.write('%s' % result.return_code) - with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: - print >>fd, 'gcc', result.toolchain.gcc - print >>fd, 'path', result.toolchain.path - print >>fd, 'cross', result.toolchain.cross - print >>fd, 'arch', result.toolchain.arch - fd.write('%s' % result.return_code) - - with open(os.path.join(build_dir, 'toolchain'), 'w') as fd: - print >>fd, 'gcc', result.toolchain.gcc - print >>fd, 'path', result.toolchain.path - - # Write out the image and function size information and an objdump - env = result.toolchain.MakeEnvironment() - lines = [] - for fname in ['u-boot', 'spl/u-boot-spl']: - cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname] - nm_result = command.RunPipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) - if nm_result.stdout: - nm = self.builder.GetFuncSizesFile(result.commit_upto, - result.brd.target, fname) - with open(nm, 'w') as fd: - print >>fd, nm_result.stdout, - - cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname] - dump_result = command.RunPipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) - rodata_size = '' - if dump_result.stdout: - objdump = self.builder.GetObjdumpFile(result.commit_upto, - result.brd.target, fname) - with open(objdump, 'w') as fd: - print >>fd, dump_result.stdout, - for line in dump_result.stdout.splitlines(): - fields = line.split() - if len(fields) > 5 and fields[1] == '.rodata': - rodata_size = fields[2] - - cmd = ['%ssize' % self.toolchain.cross, fname] - size_result = command.RunPipe([cmd], capture=True, - capture_stderr=True, cwd=result.out_dir, - raise_on_error=False, env=env) - if size_result.stdout: - lines.append(size_result.stdout.splitlines()[1] + ' ' + - rodata_size) - - # Write out the image sizes file. This is similar to the output - # of binutil's 'size' utility, but it omits the header line and - # adds an additional hex value at the end of each line for the - # rodata size - if len(lines): - sizes = self.builder.GetSizesFile(result.commit_upto, - result.brd.target) - with open(sizes, 'w') as fd: - print >>fd, '\n'.join(lines) - - # Now write the actual build output - if keep_outputs: - patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map', - 'include/autoconf.mk', 'spl/u-boot-spl', - 'spl/u-boot-spl.bin'] - for pattern in patterns: - file_list = glob.glob(os.path.join(result.out_dir, pattern)) - for fname in file_list: - shutil.copy(fname, build_dir) - - - def RunJob(self, job): - """Run a single job - - A job consists of a building a list of commits for a particular board. - - Args: - job: Job to build - """ - brd = job.board - work_dir = self.builder.GetThreadDir(self.thread_num) - self.toolchain = None - if job.commits: - # Run 'make board_config' on the first commit - do_config = True - commit_upto = 0 - force_build = False - for commit_upto in range(0, len(job.commits), job.step): - result, request_config = self.RunCommit(commit_upto, brd, - work_dir, do_config, - force_build or self.builder.force_build, - self.builder.force_build_failures) - failed = result.return_code or result.stderr - did_config = do_config - if failed and not do_config: - # If our incremental build failed, try building again - # with a reconfig. - if self.builder.force_config_on_failure: - result, request_config = self.RunCommit(commit_upto, - brd, work_dir, True, True, False) - did_config = True - if not self.builder.force_reconfig: - do_config = request_config - - # If we built that commit, then config is done. But if we got - # an warning, reconfig next time to force it to build the same - # files that created warnings this time. Otherwise an - # incremental build may not build the same file, and we will - # think that the warning has gone away. - # We could avoid this by using -Werror everywhere... - # For errors, the problem doesn't happen, since presumably - # the build stopped and didn't generate output, so will retry - # that file next time. So we could detect warnings and deal - # with them specially here. For now, we just reconfigure if - # anything goes work. - # Of course this is substantially slower if there are build - # errors/warnings (e.g. 2-3x slower even if only 10% of builds - # have problems). - if (failed and not result.already_done and not did_config and - self.builder.force_config_on_failure): - # If this build failed, try the next one with a - # reconfigure. - # Sometimes if the board_config.h file changes it can mess - # with dependencies, and we get: - # make: *** No rule to make target `include/autoconf.mk', - # needed by `depend'. - do_config = True - force_build = True - else: - force_build = False - if self.builder.force_config_on_failure: - if failed: - do_config = True - result.commit_upto = commit_upto - if result.return_code < 0: - raise ValueError('Interrupt') - - # We have the build results, so output the result - self._WriteResult(result, job.keep_outputs) - self.builder.out_queue.put(result) - else: - # Just build the currently checked-out build - result = self.RunCommit(None, True) - result.commit_upto = self.builder.upto - self.builder.out_queue.put(result) - - def run(self): - """Our thread's run function - - This thread picks a job from the queue, runs it, and then goes to the - next job. - """ - alive = True - while True: - job = self.builder.queue.get() - try: - if self.builder.active and alive: - self.RunJob(job) - except Exception as err: - alive = False - print err - self.builder.queue.task_done() - - class Builder: """Class for building U-Boot for a particular commit. @@ -544,6 +141,7 @@ class Builder: Private members: _base_board_dict: Last-summarised Dict of boards _base_err_lines: Last-summarised list of errors + _base_warn_lines: Last-summarised list of warnings _build_period_us: Time taken for a single build (float object). _complete_delay: Expected delay until completion (timedelta) _next_delay_update: Next time we plan to display a progress update @@ -576,7 +174,7 @@ class Builder: self.func_sizes = func_sizes def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs, - checkout=True, show_unknown=True, step=1): + gnu_make='make', checkout=True, show_unknown=True, step=1): """Create a new Builder object Args: @@ -585,6 +183,7 @@ class Builder: git_dir: Git directory containing source repository num_threads: Number of builder threads to run num_jobs: Number of jobs to run at once (passed to make as -j) + gnu_make: the command name of GNU Make. checkout: True to check out source, False to skip that step. This is used for testing. show_unknown: Show unknown boards (those not built) in summary @@ -596,6 +195,7 @@ class Builder: self.threads = [] self.active = True self.do_make = self.Make + self.gnu_make = gnu_make self.checkout = checkout self.num_threads = num_threads self.num_jobs = num_jobs @@ -612,19 +212,25 @@ class Builder: self.force_reconfig = False self._step = step self.in_tree = False + self._error_lines = 0 self.col = terminal.Color() + self._re_function = re.compile('(.*): In function.*') + self._re_files = re.compile('In file included from.*') + self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*') + self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*') + self.queue = Queue.Queue() self.out_queue = Queue.Queue() for i in range(self.num_threads): - t = BuilderThread(self, i) + t = builderthread.BuilderThread(self, i) t.setDaemon(True) t.start() self.threads.append(t) self.last_line_len = 0 - t = ResultThread(self) + t = builderthread.ResultThread(self) t.setDaemon(True) t.start() self.threads.append(t) @@ -637,6 +243,23 @@ class Builder: for t in self.threads: del t + def SetDisplayOptions(self, show_errors=False, show_sizes=False, + show_detail=False, show_bloat=False, + list_error_boards=False): + """Setup display options for the builder. + + show_errors: True to show summarised error/warning info + show_sizes: Show size deltas + show_detail: Show detail for each board + show_bloat: Show detail for each function + list_error_boards: Show the boards which caused each error/warning + """ + self._show_errors = show_errors + self._show_sizes = show_sizes + self._show_detail = show_detail + self._show_bloat = show_bloat + self._list_error_boards = list_error_boards + def _AddTimestamp(self): """Add a new timestamp to the list and record the build period. @@ -677,8 +300,8 @@ class Builder: length: Length of new line, in characters """ if length < self.last_line_len: - print ' ' * (self.last_line_len - length), - print '\r', + Print(' ' * (self.last_line_len - length), newline=False) + Print('\r', newline=False) self.last_line_len = length sys.stdout.flush() @@ -695,12 +318,12 @@ class Builder: Args: commit: Commit object that is being built brd: Board object that is being built - stage: Stage that we are at (distclean, config, build) + stage: Stage that we are at (mrproper, config, build) cwd: Directory where make should be run args: Arguments to pass to make kwargs: Arguments to pass to command.RunPipe() """ - cmd = ['make'] + list(args) + cmd = [self.gnu_make] + list(args) result = command.RunPipe([cmd], capture=True, capture_stderr=True, cwd=cwd, raise_on_error=False, **kwargs) return result @@ -709,7 +332,8 @@ class Builder: """Process the result of a build, showing progress information Args: - result: A CommandResult object + result: A CommandResult object, which indicates the result for + a single build """ col = terminal.Color() if result: @@ -727,6 +351,13 @@ class Builder: self.warned += 1 if result.already_done: self.already_done += 1 + if self._verbose: + Print('\r', newline=False) + self.ClearLine(0) + boards_selected = {target : result.brd} + self.ResetResultSummary(boards_selected) + self.ProduceResultSummary(result.commit_upto, self.commits, + boards_selected) else: target = '(starting)' @@ -749,8 +380,8 @@ class Builder: self.commit_count) name += target - print line + name, - length = 13 + len(name) + Print(line + name, newline=False) + length = 14 + len(name) self.ClearLine(length) def _GetOutputDir(self, commit_upto): @@ -761,10 +392,13 @@ class Builder: Args: commit_upto: Commit number to use (0..self.count-1) """ - commit = self.commits[commit_upto] - subject = commit.subject.translate(trans_valid_chars) - commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, - self.commit_count, commit.hash, subject[:20])) + if self.commits: + commit = self.commits[commit_upto] + subject = commit.subject.translate(trans_valid_chars) + commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1, + self.commit_count, commit.hash, subject[:20])) + else: + commit_dir = 'current' output_dir = os.path.join(self.base_dir, commit_dir) return output_dir @@ -862,7 +496,7 @@ class Builder: try: size, type, name = line[:-1].split() except: - print "Invalid line in file '%s': '%s'" % (fname, line[:-1]) + Print("Invalid line in file '%s': '%s'" % (fname, line[:-1])) continue if type in 'tTdDbB': # function names begin with '.' on 64-bit powerpc @@ -945,19 +579,57 @@ class Builder: Tuple: Dict containing boards which passed building this commit. keyed by board.target - List containing a summary of error/warning lines + List containing a summary of error lines + Dict keyed by error line, containing a list of the Board + objects with that error + List containing a summary of warning lines + Dict keyed by error line, containing a list of the Board + objects with that warning """ + def AddLine(lines_summary, lines_boards, line, board): + line = line.rstrip() + if line in lines_boards: + lines_boards[line].append(board) + else: + lines_boards[line] = [board] + lines_summary.append(line) + board_dict = {} err_lines_summary = [] + err_lines_boards = {} + warn_lines_summary = [] + warn_lines_boards = {} for board in boards_selected.itervalues(): outcome = self.GetBuildOutcome(commit_upto, board.target, read_func_sizes) board_dict[board.target] = outcome - for err in outcome.err_lines: - if err and not err.rstrip() in err_lines_summary: - err_lines_summary.append(err.rstrip()) - return board_dict, err_lines_summary + last_func = None + last_was_warning = False + for line in outcome.err_lines: + if line: + if (self._re_function.match(line) or + self._re_files.match(line)): + last_func = line + else: + is_warning = self._re_warning.match(line) + is_note = self._re_note.match(line) + if is_warning or (last_was_warning and is_note): + if last_func: + AddLine(warn_lines_summary, warn_lines_boards, + last_func, board) + AddLine(warn_lines_summary, warn_lines_boards, + line, board) + else: + if last_func: + AddLine(err_lines_summary, err_lines_boards, + last_func, board) + AddLine(err_lines_summary, err_lines_boards, + line, board) + last_was_warning = is_warning + last_func = None + return (board_dict, err_lines_summary, err_lines_boards, + warn_lines_summary, warn_lines_boards) def AddOutcome(self, board_dict, arch_list, changes, char, color): """Add an output to our list of outcomes for each architecture @@ -1012,6 +684,9 @@ class Builder: for board in board_selected: self._base_board_dict[board] = Builder.Outcome(0, [], [], {}) self._base_err_lines = [] + self._base_warn_lines = [] + self._base_err_line_boards = {} + self._base_warn_line_boards = {} def PrintFuncSizeDetail(self, fname, old, new): grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0 @@ -1049,16 +724,16 @@ class Builder: return args = [self.ColourNum(x) for x in args] indent = ' ' * 15 - print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % - tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) - print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', - 'delta') + Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' % + tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args)) + Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new', + 'delta')) for diff, name in delta: if diff: color = self.col.RED if diff > 0 else self.col.GREEN msg = '%s %-38s %7s %7s %+7d' % (indent, name, old.get(name, '-'), new.get(name,'-'), diff) - print self.col.Color(color, msg) + Print(msg, colour=color) def PrintSizeDetail(self, target_list, show_bloat): @@ -1083,11 +758,12 @@ class Builder: color = self.col.RED if diff > 0 else self.col.GREEN msg = ' %s %+d' % (name, diff) if not printed_target: - print '%10s %-15s:' % ('', result['_target']), + Print('%10s %-15s:' % ('', result['_target']), + newline=False) printed_target = True - print self.col.Color(color, msg), + Print(msg, colour=color, newline=False) if printed_target: - print + Print() if show_bloat: target = result['_target'] outcome = result['_outcome'] @@ -1192,18 +868,19 @@ class Builder: color = self.col.RED if avg_diff > 0 else self.col.GREEN msg = ' %s %+1.1f' % (name, avg_diff) if not printed_arch: - print '%10s: (for %d/%d boards)' % (arch, count, - arch_count[arch]), + Print('%10s: (for %d/%d boards)' % (arch, count, + arch_count[arch]), newline=False) printed_arch = True - print self.col.Color(color, msg), + Print(msg, colour=color, newline=False) if printed_arch: - print + Print() if show_detail: self.PrintSizeDetail(target_list, show_bloat) def PrintResultSummary(self, board_selected, board_dict, err_lines, + err_line_boards, warn_lines, warn_line_boards, show_sizes, show_detail, show_bloat): """Compare results with the base results and display delta. @@ -1219,10 +896,48 @@ class Builder: commit, keyed by board.target. The value is an Outcome object. err_lines: A list of errors for this commit, or [] if there is none, or we don't want to print errors + err_line_boards: Dict keyed by error line, containing a list of + the Board objects with that error + warn_lines: A list of warnings for this commit, or [] if there is + none, or we don't want to print errors + warn_line_boards: Dict keyed by warning line, containing a list of + the Board objects with that warning show_sizes: Show image size deltas show_detail: Show detail for each board show_bloat: Show detail for each function """ + def _BoardList(line, line_boards): + """Helper function to get a line of boards containing a line + + Args: + line: Error line to search for + Return: + String containing a list of boards with that error line, or + '' if the user has not requested such a list + """ + if self._list_error_boards: + names = [] + for board in line_boards[line]: + names.append(board.target) + names_str = '(%s) ' % ','.join(names) + else: + names_str = '' + return names_str + + def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards, + char): + better_lines = [] + worse_lines = [] + for line in lines: + if line not in base_lines: + worse_lines.append(char + '+' + + _BoardList(line, line_boards) + line) + for line in base_lines: + if line not in lines: + better_lines.append(char + '-' + + _BoardList(line, base_line_boards) + line) + return better_lines, worse_lines + better = [] # List of boards fixed since last commit worse = [] # List of new broken boards since last commit new = [] # List of boards that didn't exist last time @@ -1246,17 +961,14 @@ class Builder: new.append(target) # Get a list of errors that have appeared, and disappeared - better_err = [] - worse_err = [] - for line in err_lines: - if line not in self._base_err_lines: - worse_err.append('+' + line) - for line in self._base_err_lines: - if line not in err_lines: - better_err.append('-' + line) + better_err, worse_err = _CalcErrorDelta(self._base_err_lines, + self._base_err_line_boards, err_lines, err_line_boards, '') + better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines, + self._base_warn_line_boards, warn_lines, warn_line_boards, 'w') # Display results by arch - if better or worse or unknown or new or worse_err or better_err: + if (better or worse or unknown or new or worse_err or better_err + or worse_warn or better_warn): arch_list = {} self.AddOutcome(board_selected, arch_list, better, '', self.col.GREEN) @@ -1267,11 +979,20 @@ class Builder: self.AddOutcome(board_selected, arch_list, unknown, '?', self.col.MAGENTA) for arch, target_list in arch_list.iteritems(): - print '%10s: %s' % (arch, target_list) + Print('%10s: %s' % (arch, target_list)) + self._error_lines += 1 if better_err: - print self.col.Color(self.col.GREEN, '\n'.join(better_err)) + Print('\n'.join(better_err), colour=self.col.GREEN) + self._error_lines += 1 if worse_err: - print self.col.Color(self.col.RED, '\n'.join(worse_err)) + Print('\n'.join(worse_err), colour=self.col.RED) + self._error_lines += 1 + if better_warn: + Print('\n'.join(better_warn), colour=self.col.CYAN) + self._error_lines += 1 + if worse_warn: + Print('\n'.join(worse_warn), colour=self.col.MAGENTA) + self._error_lines += 1 if show_sizes: self.PrintSizeSummary(board_selected, board_dict, show_detail, @@ -1280,6 +1001,9 @@ class Builder: # Save our updated information for the next call to this function self._base_board_dict = board_dict self._base_err_lines = err_lines + self._base_warn_lines = warn_lines + self._base_err_line_boards = err_line_boards + self._base_warn_line_boards = warn_line_boards # Get a list of boards that did not get built, if needed not_built = [] @@ -1287,12 +1011,24 @@ class Builder: if not board in board_dict: not_built.append(board) if not_built: - print "Boards not built (%d): %s" % (len(not_built), - ', '.join(not_built)) - + Print("Boards not built (%d): %s" % (len(not_built), + ', '.join(not_built))) + + def ProduceResultSummary(self, commit_upto, commits, board_selected): + (board_dict, err_lines, err_line_boards, warn_lines, + warn_line_boards) = self.GetResultSummary( + board_selected, commit_upto, + read_func_sizes=self._show_bloat) + if commits: + msg = '%02d: %s' % (commit_upto + 1, + commits[commit_upto].subject) + Print(msg, colour=self.col.BLUE) + self.PrintResultSummary(board_selected, board_dict, + err_lines if self._show_errors else [], err_line_boards, + warn_lines if self._show_errors else [], warn_line_boards, + self._show_sizes, self._show_detail, self._show_bloat) - def ShowSummary(self, commits, board_selected, show_errors, show_sizes, - show_detail, show_bloat): + def ShowSummary(self, commits, board_selected): """Show a build summary for U-Boot for a given board list. Reset the result summary, then repeatedly call GetResultSummary on @@ -1301,23 +1037,16 @@ class Builder: Args: commit: Commit objects to summarise board_selected: Dict containing boards to summarise - show_errors: Show errors that occured - show_sizes: Show size deltas - show_detail: Show detail for each board - show_bloat: Show detail for each function """ - self.commit_count = len(commits) + self.commit_count = len(commits) if commits else 1 self.commits = commits self.ResetResultSummary(board_selected) + self._error_lines = 0 for commit_upto in range(0, self.commit_count, self._step): - board_dict, err_lines = self.GetResultSummary(board_selected, - commit_upto, read_func_sizes=show_bloat) - msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject) - print self.col.Color(self.col.BLUE, msg) - self.PrintResultSummary(board_selected, board_dict, - err_lines if show_errors else [], show_sizes, show_detail, - show_bloat) + self.ProduceResultSummary(commit_upto, commits, board_selected) + if not self._error_lines: + Print('(no errors to report)', colour=self.col.GREEN) def SetupBuild(self, board_selected, commits): @@ -1328,45 +1057,11 @@ class Builder: commits: Selected commits to build """ # First work out how many commits we will build - count = (len(commits) + self._step - 1) / self._step + count = (self.commit_count + self._step - 1) / self._step self.count = len(board_selected) * count self.upto = self.warned = self.fail = 0 self._timestamps = collections.deque() - def BuildBoardsForCommit(self, board_selected, keep_outputs): - """Build all boards for a single commit""" - self.SetupBuild(board_selected) - self.count = len(board_selected) - for brd in board_selected.itervalues(): - job = BuilderJob() - job.board = brd - job.commits = None - job.keep_outputs = keep_outputs - self.queue.put(brd) - - self.queue.join() - self.out_queue.join() - print - self.ClearLine(0) - - def BuildCommits(self, commits, board_selected, show_errors, keep_outputs): - """Build all boards for all commits (non-incremental)""" - self.commit_count = len(commits) - - self.ResetResultSummary(board_selected) - for self.commit_upto in range(self.commit_count): - self.SelectCommit(commits[self.commit_upto]) - self.SelectOutputDir() - Mkdir(self.output_dir) - - self.BuildBoardsForCommit(board_selected, keep_outputs) - board_dict, err_lines = self.GetResultSummary() - self.PrintResultSummary(board_selected, board_dict, - err_lines if show_errors else []) - - if self.already_done: - print '%d builds already done' % self.already_done - def GetThreadDir(self, thread_num): """Get the directory path to the working dir for a thread. @@ -1375,40 +1070,42 @@ class Builder: """ return os.path.join(self._working_dir, '%02d' % thread_num) - def _PrepareThread(self, thread_num): + def _PrepareThread(self, thread_num, setup_git): """Prepare the working directory for a thread. This clones or fetches the repo into the thread's work directory. Args: thread_num: Thread number (0, 1, ...) + setup_git: True to set up a git repo clone """ thread_dir = self.GetThreadDir(thread_num) - Mkdir(thread_dir) + builderthread.Mkdir(thread_dir) git_dir = os.path.join(thread_dir, '.git') # Clone the repo if it doesn't already exist # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so # we have a private index but uses the origin repo's contents? - if self.git_dir: + if setup_git and self.git_dir: src_dir = os.path.abspath(self.git_dir) if os.path.exists(git_dir): gitutil.Fetch(git_dir, thread_dir) else: - print 'Cloning repo for thread %d' % thread_num + Print('Cloning repo for thread %d' % thread_num) gitutil.Clone(src_dir, thread_dir) - def _PrepareWorkingSpace(self, max_threads): + def _PrepareWorkingSpace(self, max_threads, setup_git): """Prepare the working directory for use. Set up the git repo for each thread. Args: max_threads: Maximum number of threads we expect to need. + setup_git: True to set up a git repo clone """ - Mkdir(self._working_dir) + builderthread.Mkdir(self._working_dir) for thread in range(max_threads): - self._PrepareThread(thread) + self._PrepareThread(thread, setup_git) def _PrepareOutputSpace(self): """Get the output directories ready to receive files. @@ -1425,29 +1122,35 @@ class Builder: if dirname not in dir_list: shutil.rmtree(dirname) - def BuildBoards(self, commits, board_selected, show_errors, keep_outputs): + def BuildBoards(self, commits, board_selected, keep_outputs, verbose): """Build all commits for a list of boards Args: commits: List of commits to be build, each a Commit object boards_selected: Dict of selected boards, key is target name, value is Board object - show_errors: True to show summarised error/warning info keep_outputs: True to save build output files + verbose: Display build results as they are completed + Returns: + Tuple containing: + - number of boards that failed to build + - number of boards that issued warnings """ - self.commit_count = len(commits) + self.commit_count = len(commits) if commits else 1 self.commits = commits + self._verbose = verbose self.ResetResultSummary(board_selected) - Mkdir(self.base_dir) - self._PrepareWorkingSpace(min(self.num_threads, len(board_selected))) + builderthread.Mkdir(self.base_dir) + self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)), + commits is not None) self._PrepareOutputSpace() self.SetupBuild(board_selected, commits) self.ProcessResult(None) # Create jobs to build all commits for each board for brd in board_selected.itervalues(): - job = BuilderJob() + job = builderthread.BuilderJob() job.board = brd job.commits = commits job.keep_outputs = keep_outputs @@ -1459,5 +1162,6 @@ class Builder: # Wait until we have processed all output self.out_queue.join() - print + Print() self.ClearLine(0) + return (self.fail, self.warned)