#
import collections
-import errno
from datetime import datetime, timedelta
import glob
import os
import shutil
import string
import sys
-import threading
import time
+import builderthread
import command
import gitutil
import terminal
+from terminal import Print
import toolchain
trans_valid_chars = string.maketrans("/: ", "---")
-def Mkdir(dirname):
- """Make a directory if it doesn't already exist.
-
- Args:
- dirname: Directory to create
- """
- try:
- os.mkdir(dirname)
- except OSError as err:
- if err.errno == errno.EEXIST:
- pass
- else:
- raise
-
-class BuilderJob:
- """Holds information about a job to be performed by a thread
-
- Members:
- board: Board object to build
- commits: List of commit options to build.
- """
- def __init__(self):
- self.board = None
- self.commits = []
-
-
-class ResultThread(threading.Thread):
- """This thread processes results from builder threads.
-
- It simply passes the results on to the builder. There is only one
- result thread, and this helps to serialise the build output.
- """
- def __init__(self, builder):
- """Set up a new result thread
-
- Args:
- builder: Builder which will be sent each result
- """
- threading.Thread.__init__(self)
- self.builder = builder
-
- def run(self):
- """Called to start up the result thread.
-
- We collect the next result job and pass it on to the build.
- """
- while True:
- result = self.builder.out_queue.get()
- self.builder.ProcessResult(result)
- self.builder.out_queue.task_done()
-
-
-class BuilderThread(threading.Thread):
- """This thread builds U-Boot for a particular board.
-
- An input queue provides each new job. We run 'make' to build U-Boot
- and then pass the results on to the output queue.
-
- Members:
- builder: The builder which contains information we might need
- thread_num: Our thread number (0-n-1), used to decide on a
- temporary directory
- """
- def __init__(self, builder, thread_num):
- """Set up a new builder thread"""
- threading.Thread.__init__(self)
- self.builder = builder
- self.thread_num = thread_num
-
- def Make(self, commit, brd, stage, cwd, *args, **kwargs):
- """Run 'make' on a particular commit and board.
-
- The source code will already be checked out, so the 'commit'
- argument is only for information.
-
- Args:
- commit: Commit object that is being built
- brd: Board object that is being built
- stage: Stage of the build. Valid stages are:
- distclean - can be called to clean source
- config - called to configure for a board
- build - the main make invocation - it does the build
- args: A list of arguments to pass to 'make'
- kwargs: A list of keyword arguments to pass to command.RunPipe()
-
- Returns:
- CommandResult object
- """
- return self.builder.do_make(commit, brd, stage, cwd, *args,
- **kwargs)
-
- def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
- force_build_failures):
- """Build a particular commit.
-
- If the build is already done, and we are not forcing a build, we skip
- the build and just return the previously-saved results.
-
- Args:
- commit_upto: Commit number to build (0...n-1)
- brd: Board object to build
- work_dir: Directory to which the source will be checked out
- do_config: True to run a make <board>_config on the source
- force_build: Force a build even if one was previously done
- force_build_failures: Force a bulid if the previous result showed
- failure
-
- Returns:
- tuple containing:
- - CommandResult object containing the results of the build
- - boolean indicating whether 'make config' is still needed
- """
- # Create a default result - it will be overwritte by the call to
- # self.Make() below, in the event that we do a build.
- result = command.CommandResult()
- result.return_code = 0
- out_dir = os.path.join(work_dir, 'build')
-
- # Check if the job was already completed last time
- done_file = self.builder.GetDoneFile(commit_upto, brd.target)
- result.already_done = os.path.exists(done_file)
- will_build = (force_build or force_build_failures or
- not result.already_done)
- if result.already_done and will_build:
- # Get the return code from that build and use it
- with open(done_file, 'r') as fd:
- result.return_code = int(fd.readline())
- err_file = self.builder.GetErrFile(commit_upto, brd.target)
- if os.path.exists(err_file) and os.stat(err_file).st_size:
- result.stderr = 'bad'
- elif not force_build:
- # The build passed, so no need to build it again
- will_build = False
-
- if will_build:
- # We are going to have to build it. First, get a toolchain
- if not self.toolchain:
- try:
- self.toolchain = self.builder.toolchains.Select(brd.arch)
- except ValueError as err:
- result.return_code = 10
- result.stdout = ''
- result.stderr = str(err)
- # TODO(sjg@chromium.org): This gets swallowed, but needs
- # to be reported.
-
- if self.toolchain:
- # Checkout the right commit
- if commit_upto is not None:
- commit = self.builder.commits[commit_upto]
- if self.builder.checkout:
- git_dir = os.path.join(work_dir, '.git')
- gitutil.Checkout(commit.hash, git_dir, work_dir,
- force=True)
- else:
- commit = self.builder.commit # Ick, fix this for BuildCommits()
-
- # Set up the environment and command line
- env = self.toolchain.MakeEnvironment()
- Mkdir(out_dir)
- args = ['O=build', '-s']
- if self.builder.num_jobs is not None:
- args.extend(['-j', str(self.builder.num_jobs)])
- config_args = ['%s_config' % brd.target]
- config_out = ''
- args.extend(self.builder.toolchains.GetMakeArguments(brd))
-
- # If we need to reconfigure, do that now
- if do_config:
- result = self.Make(commit, brd, 'distclean', work_dir,
- 'distclean', *args, env=env)
- result = self.Make(commit, brd, 'config', work_dir,
- *(args + config_args), env=env)
- config_out = result.combined
- do_config = False # No need to configure next time
- if result.return_code == 0:
- result = self.Make(commit, brd, 'build', work_dir, *args,
- env=env)
- result.stdout = config_out + result.stdout
- else:
- result.return_code = 1
- result.stderr = 'No tool chain for %s\n' % brd.arch
- result.already_done = False
-
- result.toolchain = self.toolchain
- result.brd = brd
- result.commit_upto = commit_upto
- result.out_dir = out_dir
- return result, do_config
-
- def _WriteResult(self, result, keep_outputs):
- """Write a built result to the output directory.
-
- Args:
- result: CommandResult object containing result to write
- keep_outputs: True to store the output binaries, False
- to delete them
- """
- # Fatal error
- if result.return_code < 0:
- return
-
- # Aborted?
- if result.stderr and 'No child processes' in result.stderr:
- return
-
- if result.already_done:
- return
-
- # Write the output and stderr
- output_dir = self.builder._GetOutputDir(result.commit_upto)
- Mkdir(output_dir)
- build_dir = self.builder.GetBuildDir(result.commit_upto,
- result.brd.target)
- Mkdir(build_dir)
-
- outfile = os.path.join(build_dir, 'log')
- with open(outfile, 'w') as fd:
- if result.stdout:
- fd.write(result.stdout)
-
- errfile = self.builder.GetErrFile(result.commit_upto,
- result.brd.target)
- if result.stderr:
- with open(errfile, 'w') as fd:
- fd.write(result.stderr)
- elif os.path.exists(errfile):
- os.remove(errfile)
-
- if result.toolchain:
- # Write the build result and toolchain information.
- done_file = self.builder.GetDoneFile(result.commit_upto,
- result.brd.target)
- with open(done_file, 'w') as fd:
- fd.write('%s' % result.return_code)
- with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
- print >>fd, 'gcc', result.toolchain.gcc
- print >>fd, 'path', result.toolchain.path
- print >>fd, 'cross', result.toolchain.cross
- print >>fd, 'arch', result.toolchain.arch
- fd.write('%s' % result.return_code)
-
- with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
- print >>fd, 'gcc', result.toolchain.gcc
- print >>fd, 'path', result.toolchain.path
-
- # Write out the image and function size information and an objdump
- env = result.toolchain.MakeEnvironment()
- lines = []
- for fname in ['u-boot', 'spl/u-boot-spl']:
- cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
- nm_result = command.RunPipe([cmd], capture=True,
- capture_stderr=True, cwd=result.out_dir,
- raise_on_error=False, env=env)
- if nm_result.stdout:
- nm = self.builder.GetFuncSizesFile(result.commit_upto,
- result.brd.target, fname)
- with open(nm, 'w') as fd:
- print >>fd, nm_result.stdout,
-
- cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
- dump_result = command.RunPipe([cmd], capture=True,
- capture_stderr=True, cwd=result.out_dir,
- raise_on_error=False, env=env)
- rodata_size = ''
- if dump_result.stdout:
- objdump = self.builder.GetObjdumpFile(result.commit_upto,
- result.brd.target, fname)
- with open(objdump, 'w') as fd:
- print >>fd, dump_result.stdout,
- for line in dump_result.stdout.splitlines():
- fields = line.split()
- if len(fields) > 5 and fields[1] == '.rodata':
- rodata_size = fields[2]
-
- cmd = ['%ssize' % self.toolchain.cross, fname]
- size_result = command.RunPipe([cmd], capture=True,
- capture_stderr=True, cwd=result.out_dir,
- raise_on_error=False, env=env)
- if size_result.stdout:
- lines.append(size_result.stdout.splitlines()[1] + ' ' +
- rodata_size)
-
- # Write out the image sizes file. This is similar to the output
- # of binutil's 'size' utility, but it omits the header line and
- # adds an additional hex value at the end of each line for the
- # rodata size
- if len(lines):
- sizes = self.builder.GetSizesFile(result.commit_upto,
- result.brd.target)
- with open(sizes, 'w') as fd:
- print >>fd, '\n'.join(lines)
-
- # Now write the actual build output
- if keep_outputs:
- patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
- 'include/autoconf.mk', 'spl/u-boot-spl',
- 'spl/u-boot-spl.bin']
- for pattern in patterns:
- file_list = glob.glob(os.path.join(result.out_dir, pattern))
- for fname in file_list:
- shutil.copy(fname, build_dir)
-
-
- def RunJob(self, job):
- """Run a single job
-
- A job consists of a building a list of commits for a particular board.
-
- Args:
- job: Job to build
- """
- brd = job.board
- work_dir = self.builder.GetThreadDir(self.thread_num)
- self.toolchain = None
- if job.commits:
- # Run 'make board_config' on the first commit
- do_config = True
- commit_upto = 0
- force_build = False
- for commit_upto in range(0, len(job.commits), job.step):
- result, request_config = self.RunCommit(commit_upto, brd,
- work_dir, do_config,
- force_build or self.builder.force_build,
- self.builder.force_build_failures)
- failed = result.return_code or result.stderr
- did_config = do_config
- if failed and not do_config:
- # If our incremental build failed, try building again
- # with a reconfig.
- if self.builder.force_config_on_failure:
- result, request_config = self.RunCommit(commit_upto,
- brd, work_dir, True, True, False)
- did_config = True
- do_config = request_config
-
- # If we built that commit, then config is done. But if we got
- # an warning, reconfig next time to force it to build the same
- # files that created warnings this time. Otherwise an
- # incremental build may not build the same file, and we will
- # think that the warning has gone away.
- # We could avoid this by using -Werror everywhere...
- # For errors, the problem doesn't happen, since presumably
- # the build stopped and didn't generate output, so will retry
- # that file next time. So we could detect warnings and deal
- # with them specially here. For now, we just reconfigure if
- # anything goes work.
- # Of course this is substantially slower if there are build
- # errors/warnings (e.g. 2-3x slower even if only 10% of builds
- # have problems).
- if (failed and not result.already_done and not did_config and
- self.builder.force_config_on_failure):
- # If this build failed, try the next one with a
- # reconfigure.
- # Sometimes if the board_config.h file changes it can mess
- # with dependencies, and we get:
- # make: *** No rule to make target `include/autoconf.mk',
- # needed by `depend'.
- do_config = True
- force_build = True
- else:
- force_build = False
- if self.builder.force_config_on_failure:
- if failed:
- do_config = True
- result.commit_upto = commit_upto
- if result.return_code < 0:
- raise ValueError('Interrupt')
-
- # We have the build results, so output the result
- self._WriteResult(result, job.keep_outputs)
- self.builder.out_queue.put(result)
- else:
- # Just build the currently checked-out build
- result = self.RunCommit(None, True)
- result.commit_upto = self.builder.upto
- self.builder.out_queue.put(result)
-
- def run(self):
- """Our thread's run function
-
- This thread picks a job from the queue, runs it, and then goes to the
- next job.
- """
- alive = True
- while True:
- job = self.builder.queue.get()
- try:
- if self.builder.active and alive:
- self.RunJob(job)
- except Exception as err:
- alive = False
- print err
- self.builder.queue.task_done()
-
-
class Builder:
"""Class for building U-Boot for a particular commit.
toolchains: Toolchains object to use for building
upto: Current commit number we are building (0.count-1)
warned: Number of builds that produced at least one warning
+ force_reconfig: Reconfigure U-Boot on each comiit. This disables
+ incremental building, where buildman reconfigures on the first
+ commit for a baord, and then just does an incremental build for
+ the following commits. In fact buildman will reconfigure and
+ retry for any failing commits, so generally the only effect of
+ this option is to slow things down.
+ in_tree: Build U-Boot in-tree instead of specifying an output
+ directory separate from the source code. This option is really
+ only useful for testing in-tree builds.
Private members:
_base_board_dict: Last-summarised Dict of boards
_base_err_lines: Last-summarised list of errors
+ _base_warn_lines: Last-summarised list of warnings
_build_period_us: Time taken for a single build (float object).
_complete_delay: Expected delay until completion (timedelta)
_next_delay_update: Next time we plan to display a progress update
self.func_sizes = func_sizes
def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
- checkout=True, show_unknown=True, step=1):
+ gnu_make='make', checkout=True, show_unknown=True, step=1):
"""Create a new Builder object
Args:
git_dir: Git directory containing source repository
num_threads: Number of builder threads to run
num_jobs: Number of jobs to run at once (passed to make as -j)
+ gnu_make: the command name of GNU Make.
checkout: True to check out source, False to skip that step.
This is used for testing.
show_unknown: Show unknown boards (those not built) in summary
self.threads = []
self.active = True
self.do_make = self.Make
+ self.gnu_make = gnu_make
self.checkout = checkout
self.num_threads = num_threads
self.num_jobs = num_jobs
self._next_delay_update = datetime.now()
self.force_config_on_failure = True
self.force_build_failures = False
+ self.force_reconfig = False
self._step = step
+ self.in_tree = False
+ self._error_lines = 0
self.col = terminal.Color()
+ self._re_function = re.compile('(.*): In function.*')
+ self._re_files = re.compile('In file included from.*')
+ self._re_warning = re.compile('(.*):(\d*):(\d*): warning: .*')
+ self._re_note = re.compile('(.*):(\d*):(\d*): note: this is the location of the previous.*')
+
self.queue = Queue.Queue()
self.out_queue = Queue.Queue()
for i in range(self.num_threads):
- t = BuilderThread(self, i)
+ t = builderthread.BuilderThread(self, i)
t.setDaemon(True)
t.start()
self.threads.append(t)
self.last_line_len = 0
- t = ResultThread(self)
+ t = builderthread.ResultThread(self)
t.setDaemon(True)
t.start()
self.threads.append(t)
for t in self.threads:
del t
+ def SetDisplayOptions(self, show_errors=False, show_sizes=False,
+ show_detail=False, show_bloat=False,
+ list_error_boards=False):
+ """Setup display options for the builder.
+
+ show_errors: True to show summarised error/warning info
+ show_sizes: Show size deltas
+ show_detail: Show detail for each board
+ show_bloat: Show detail for each function
+ list_error_boards: Show the boards which caused each error/warning
+ """
+ self._show_errors = show_errors
+ self._show_sizes = show_sizes
+ self._show_detail = show_detail
+ self._show_bloat = show_bloat
+ self._list_error_boards = list_error_boards
+
def _AddTimestamp(self):
"""Add a new timestamp to the list and record the build period.
length: Length of new line, in characters
"""
if length < self.last_line_len:
- print ' ' * (self.last_line_len - length),
- print '\r',
+ Print(' ' * (self.last_line_len - length), newline=False)
+ Print('\r', newline=False)
self.last_line_len = length
sys.stdout.flush()
Args:
commit: Commit object that is being built
brd: Board object that is being built
- stage: Stage that we are at (distclean, config, build)
+ stage: Stage that we are at (mrproper, config, build)
cwd: Directory where make should be run
args: Arguments to pass to make
kwargs: Arguments to pass to command.RunPipe()
"""
- cmd = ['make'] + list(args)
+ cmd = [self.gnu_make] + list(args)
result = command.RunPipe([cmd], capture=True, capture_stderr=True,
cwd=cwd, raise_on_error=False, **kwargs)
return result
"""Process the result of a build, showing progress information
Args:
- result: A CommandResult object
+ result: A CommandResult object, which indicates the result for
+ a single build
"""
col = terminal.Color()
if result:
self.warned += 1
if result.already_done:
self.already_done += 1
+ if self._verbose:
+ Print('\r', newline=False)
+ self.ClearLine(0)
+ boards_selected = {target : result.brd}
+ self.ResetResultSummary(boards_selected)
+ self.ProduceResultSummary(result.commit_upto, self.commits,
+ boards_selected)
else:
target = '(starting)'
self.commit_count)
name += target
- print line + name,
- length = 13 + len(name)
+ Print(line + name, newline=False)
+ length = 14 + len(name)
self.ClearLine(length)
def _GetOutputDir(self, commit_upto):
Args:
commit_upto: Commit number to use (0..self.count-1)
"""
- commit = self.commits[commit_upto]
- subject = commit.subject.translate(trans_valid_chars)
- commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
- self.commit_count, commit.hash, subject[:20]))
+ if self.commits:
+ commit = self.commits[commit_upto]
+ subject = commit.subject.translate(trans_valid_chars)
+ commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
+ self.commit_count, commit.hash, subject[:20]))
+ else:
+ commit_dir = 'current'
output_dir = os.path.join(self.base_dir, commit_dir)
return output_dir
try:
size, type, name = line[:-1].split()
except:
- print "Invalid line in file '%s': '%s'" % (fname, line[:-1])
+ Print("Invalid line in file '%s': '%s'" % (fname, line[:-1]))
continue
if type in 'tTdDbB':
# function names begin with '.' on 64-bit powerpc
Tuple:
Dict containing boards which passed building this commit.
keyed by board.target
- List containing a summary of error/warning lines
+ List containing a summary of error lines
+ Dict keyed by error line, containing a list of the Board
+ objects with that error
+ List containing a summary of warning lines
+ Dict keyed by error line, containing a list of the Board
+ objects with that warning
"""
+ def AddLine(lines_summary, lines_boards, line, board):
+ line = line.rstrip()
+ if line in lines_boards:
+ lines_boards[line].append(board)
+ else:
+ lines_boards[line] = [board]
+ lines_summary.append(line)
+
board_dict = {}
err_lines_summary = []
+ err_lines_boards = {}
+ warn_lines_summary = []
+ warn_lines_boards = {}
for board in boards_selected.itervalues():
outcome = self.GetBuildOutcome(commit_upto, board.target,
read_func_sizes)
board_dict[board.target] = outcome
- for err in outcome.err_lines:
- if err and not err.rstrip() in err_lines_summary:
- err_lines_summary.append(err.rstrip())
- return board_dict, err_lines_summary
+ last_func = None
+ last_was_warning = False
+ for line in outcome.err_lines:
+ if line:
+ if (self._re_function.match(line) or
+ self._re_files.match(line)):
+ last_func = line
+ else:
+ is_warning = self._re_warning.match(line)
+ is_note = self._re_note.match(line)
+ if is_warning or (last_was_warning and is_note):
+ if last_func:
+ AddLine(warn_lines_summary, warn_lines_boards,
+ last_func, board)
+ AddLine(warn_lines_summary, warn_lines_boards,
+ line, board)
+ else:
+ if last_func:
+ AddLine(err_lines_summary, err_lines_boards,
+ last_func, board)
+ AddLine(err_lines_summary, err_lines_boards,
+ line, board)
+ last_was_warning = is_warning
+ last_func = None
+ return (board_dict, err_lines_summary, err_lines_boards,
+ warn_lines_summary, warn_lines_boards)
def AddOutcome(self, board_dict, arch_list, changes, char, color):
"""Add an output to our list of outcomes for each architecture
for board in board_selected:
self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
self._base_err_lines = []
+ self._base_warn_lines = []
+ self._base_err_line_boards = {}
+ self._base_warn_line_boards = {}
def PrintFuncSizeDetail(self, fname, old, new):
grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
return
args = [self.ColourNum(x) for x in args]
indent = ' ' * 15
- print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
- tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
- print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
- 'delta')
+ Print('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
+ tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
+ Print('%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
+ 'delta'))
for diff, name in delta:
if diff:
color = self.col.RED if diff > 0 else self.col.GREEN
msg = '%s %-38s %7s %7s %+7d' % (indent, name,
old.get(name, '-'), new.get(name,'-'), diff)
- print self.col.Color(color, msg)
+ Print(msg, colour=color)
def PrintSizeDetail(self, target_list, show_bloat):
color = self.col.RED if diff > 0 else self.col.GREEN
msg = ' %s %+d' % (name, diff)
if not printed_target:
- print '%10s %-15s:' % ('', result['_target']),
+ Print('%10s %-15s:' % ('', result['_target']),
+ newline=False)
printed_target = True
- print self.col.Color(color, msg),
+ Print(msg, colour=color, newline=False)
if printed_target:
- print
+ Print()
if show_bloat:
target = result['_target']
outcome = result['_outcome']
color = self.col.RED if avg_diff > 0 else self.col.GREEN
msg = ' %s %+1.1f' % (name, avg_diff)
if not printed_arch:
- print '%10s: (for %d/%d boards)' % (arch, count,
- arch_count[arch]),
+ Print('%10s: (for %d/%d boards)' % (arch, count,
+ arch_count[arch]), newline=False)
printed_arch = True
- print self.col.Color(color, msg),
+ Print(msg, colour=color, newline=False)
if printed_arch:
- print
+ Print()
if show_detail:
self.PrintSizeDetail(target_list, show_bloat)
def PrintResultSummary(self, board_selected, board_dict, err_lines,
+ err_line_boards, warn_lines, warn_line_boards,
show_sizes, show_detail, show_bloat):
"""Compare results with the base results and display delta.
commit, keyed by board.target. The value is an Outcome object.
err_lines: A list of errors for this commit, or [] if there is
none, or we don't want to print errors
+ err_line_boards: Dict keyed by error line, containing a list of
+ the Board objects with that error
+ warn_lines: A list of warnings for this commit, or [] if there is
+ none, or we don't want to print errors
+ warn_line_boards: Dict keyed by warning line, containing a list of
+ the Board objects with that warning
show_sizes: Show image size deltas
show_detail: Show detail for each board
show_bloat: Show detail for each function
"""
+ def _BoardList(line, line_boards):
+ """Helper function to get a line of boards containing a line
+
+ Args:
+ line: Error line to search for
+ Return:
+ String containing a list of boards with that error line, or
+ '' if the user has not requested such a list
+ """
+ if self._list_error_boards:
+ names = []
+ for board in line_boards[line]:
+ names.append(board.target)
+ names_str = '(%s) ' % ','.join(names)
+ else:
+ names_str = ''
+ return names_str
+
+ def _CalcErrorDelta(base_lines, base_line_boards, lines, line_boards,
+ char):
+ better_lines = []
+ worse_lines = []
+ for line in lines:
+ if line not in base_lines:
+ worse_lines.append(char + '+' +
+ _BoardList(line, line_boards) + line)
+ for line in base_lines:
+ if line not in lines:
+ better_lines.append(char + '-' +
+ _BoardList(line, base_line_boards) + line)
+ return better_lines, worse_lines
+
better = [] # List of boards fixed since last commit
worse = [] # List of new broken boards since last commit
new = [] # List of boards that didn't exist last time
new.append(target)
# Get a list of errors that have appeared, and disappeared
- better_err = []
- worse_err = []
- for line in err_lines:
- if line not in self._base_err_lines:
- worse_err.append('+' + line)
- for line in self._base_err_lines:
- if line not in err_lines:
- better_err.append('-' + line)
+ better_err, worse_err = _CalcErrorDelta(self._base_err_lines,
+ self._base_err_line_boards, err_lines, err_line_boards, '')
+ better_warn, worse_warn = _CalcErrorDelta(self._base_warn_lines,
+ self._base_warn_line_boards, warn_lines, warn_line_boards, 'w')
# Display results by arch
- if better or worse or unknown or new or worse_err or better_err:
+ if (better or worse or unknown or new or worse_err or better_err
+ or worse_warn or better_warn):
arch_list = {}
self.AddOutcome(board_selected, arch_list, better, '',
self.col.GREEN)
self.AddOutcome(board_selected, arch_list, unknown, '?',
self.col.MAGENTA)
for arch, target_list in arch_list.iteritems():
- print '%10s: %s' % (arch, target_list)
+ Print('%10s: %s' % (arch, target_list))
+ self._error_lines += 1
if better_err:
- print self.col.Color(self.col.GREEN, '\n'.join(better_err))
+ Print('\n'.join(better_err), colour=self.col.GREEN)
+ self._error_lines += 1
if worse_err:
- print self.col.Color(self.col.RED, '\n'.join(worse_err))
+ Print('\n'.join(worse_err), colour=self.col.RED)
+ self._error_lines += 1
+ if better_warn:
+ Print('\n'.join(better_warn), colour=self.col.CYAN)
+ self._error_lines += 1
+ if worse_warn:
+ Print('\n'.join(worse_warn), colour=self.col.MAGENTA)
+ self._error_lines += 1
if show_sizes:
self.PrintSizeSummary(board_selected, board_dict, show_detail,
# Save our updated information for the next call to this function
self._base_board_dict = board_dict
self._base_err_lines = err_lines
+ self._base_warn_lines = warn_lines
+ self._base_err_line_boards = err_line_boards
+ self._base_warn_line_boards = warn_line_boards
# Get a list of boards that did not get built, if needed
not_built = []
if not board in board_dict:
not_built.append(board)
if not_built:
- print "Boards not built (%d): %s" % (len(not_built),
- ', '.join(not_built))
-
+ Print("Boards not built (%d): %s" % (len(not_built),
+ ', '.join(not_built)))
+
+ def ProduceResultSummary(self, commit_upto, commits, board_selected):
+ (board_dict, err_lines, err_line_boards, warn_lines,
+ warn_line_boards) = self.GetResultSummary(
+ board_selected, commit_upto,
+ read_func_sizes=self._show_bloat)
+ if commits:
+ msg = '%02d: %s' % (commit_upto + 1,
+ commits[commit_upto].subject)
+ Print(msg, colour=self.col.BLUE)
+ self.PrintResultSummary(board_selected, board_dict,
+ err_lines if self._show_errors else [], err_line_boards,
+ warn_lines if self._show_errors else [], warn_line_boards,
+ self._show_sizes, self._show_detail, self._show_bloat)
- def ShowSummary(self, commits, board_selected, show_errors, show_sizes,
- show_detail, show_bloat):
+ def ShowSummary(self, commits, board_selected):
"""Show a build summary for U-Boot for a given board list.
Reset the result summary, then repeatedly call GetResultSummary on
Args:
commit: Commit objects to summarise
board_selected: Dict containing boards to summarise
- show_errors: Show errors that occured
- show_sizes: Show size deltas
- show_detail: Show detail for each board
- show_bloat: Show detail for each function
"""
- self.commit_count = len(commits)
+ self.commit_count = len(commits) if commits else 1
self.commits = commits
self.ResetResultSummary(board_selected)
+ self._error_lines = 0
for commit_upto in range(0, self.commit_count, self._step):
- board_dict, err_lines = self.GetResultSummary(board_selected,
- commit_upto, read_func_sizes=show_bloat)
- msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject)
- print self.col.Color(self.col.BLUE, msg)
- self.PrintResultSummary(board_selected, board_dict,
- err_lines if show_errors else [], show_sizes, show_detail,
- show_bloat)
+ self.ProduceResultSummary(commit_upto, commits, board_selected)
+ if not self._error_lines:
+ Print('(no errors to report)', colour=self.col.GREEN)
def SetupBuild(self, board_selected, commits):
commits: Selected commits to build
"""
# First work out how many commits we will build
- count = (len(commits) + self._step - 1) / self._step
+ count = (self.commit_count + self._step - 1) / self._step
self.count = len(board_selected) * count
self.upto = self.warned = self.fail = 0
self._timestamps = collections.deque()
- def BuildBoardsForCommit(self, board_selected, keep_outputs):
- """Build all boards for a single commit"""
- self.SetupBuild(board_selected)
- self.count = len(board_selected)
- for brd in board_selected.itervalues():
- job = BuilderJob()
- job.board = brd
- job.commits = None
- job.keep_outputs = keep_outputs
- self.queue.put(brd)
-
- self.queue.join()
- self.out_queue.join()
- print
- self.ClearLine(0)
-
- def BuildCommits(self, commits, board_selected, show_errors, keep_outputs):
- """Build all boards for all commits (non-incremental)"""
- self.commit_count = len(commits)
-
- self.ResetResultSummary(board_selected)
- for self.commit_upto in range(self.commit_count):
- self.SelectCommit(commits[self.commit_upto])
- self.SelectOutputDir()
- Mkdir(self.output_dir)
-
- self.BuildBoardsForCommit(board_selected, keep_outputs)
- board_dict, err_lines = self.GetResultSummary()
- self.PrintResultSummary(board_selected, board_dict,
- err_lines if show_errors else [])
-
- if self.already_done:
- print '%d builds already done' % self.already_done
-
def GetThreadDir(self, thread_num):
"""Get the directory path to the working dir for a thread.
"""
return os.path.join(self._working_dir, '%02d' % thread_num)
- def _PrepareThread(self, thread_num):
+ def _PrepareThread(self, thread_num, setup_git):
"""Prepare the working directory for a thread.
This clones or fetches the repo into the thread's work directory.
Args:
thread_num: Thread number (0, 1, ...)
+ setup_git: True to set up a git repo clone
"""
thread_dir = self.GetThreadDir(thread_num)
- Mkdir(thread_dir)
+ builderthread.Mkdir(thread_dir)
git_dir = os.path.join(thread_dir, '.git')
# Clone the repo if it doesn't already exist
# TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
# we have a private index but uses the origin repo's contents?
- if self.git_dir:
+ if setup_git and self.git_dir:
src_dir = os.path.abspath(self.git_dir)
if os.path.exists(git_dir):
gitutil.Fetch(git_dir, thread_dir)
else:
- print 'Cloning repo for thread %d' % thread_num
+ Print('Cloning repo for thread %d' % thread_num)
gitutil.Clone(src_dir, thread_dir)
- def _PrepareWorkingSpace(self, max_threads):
+ def _PrepareWorkingSpace(self, max_threads, setup_git):
"""Prepare the working directory for use.
Set up the git repo for each thread.
Args:
max_threads: Maximum number of threads we expect to need.
+ setup_git: True to set up a git repo clone
"""
- Mkdir(self._working_dir)
+ builderthread.Mkdir(self._working_dir)
for thread in range(max_threads):
- self._PrepareThread(thread)
+ self._PrepareThread(thread, setup_git)
def _PrepareOutputSpace(self):
"""Get the output directories ready to receive files.
if dirname not in dir_list:
shutil.rmtree(dirname)
- def BuildBoards(self, commits, board_selected, show_errors, keep_outputs):
+ def BuildBoards(self, commits, board_selected, keep_outputs, verbose):
"""Build all commits for a list of boards
Args:
commits: List of commits to be build, each a Commit object
boards_selected: Dict of selected boards, key is target name,
value is Board object
- show_errors: True to show summarised error/warning info
keep_outputs: True to save build output files
+ verbose: Display build results as they are completed
+ Returns:
+ Tuple containing:
+ - number of boards that failed to build
+ - number of boards that issued warnings
"""
- self.commit_count = len(commits)
+ self.commit_count = len(commits) if commits else 1
self.commits = commits
+ self._verbose = verbose
self.ResetResultSummary(board_selected)
- Mkdir(self.base_dir)
- self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)))
+ builderthread.Mkdir(self.base_dir)
+ self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)),
+ commits is not None)
self._PrepareOutputSpace()
self.SetupBuild(board_selected, commits)
self.ProcessResult(None)
# Create jobs to build all commits for each board
for brd in board_selected.itervalues():
- job = BuilderJob()
+ job = builderthread.BuilderJob()
job.board = brd
job.commits = commits
job.keep_outputs = keep_outputs
# Wait until we have processed all output
self.out_queue.join()
- print
+ Print()
self.ClearLine(0)
+ return (self.fail, self.warned)
projects / karo-tx-uboot.git / blobdiff