1 # Copyright (c) 2013 The Chromium OS Authors.
3 # Bloat-o-meter code used here Copyright 2004 Matt Mackall <mpm@selenic.com>
5 # SPDX-License-Identifier: GPL-2.0+
10 from datetime import datetime, timedelta
30 Please see README for user documentation, and you should be familiar with
31 that before trying to make sense of this.
33 Buildman works by keeping the machine as busy as possible, building different
34 commits for different boards on multiple CPUs at once.
36 The source repo (self.git_dir) contains all the commits to be built. Each
37 thread works on a single board at a time. It checks out the first commit,
38 configures it for that board, then builds it. Then it checks out the next
39 commit and builds it (typically without re-configuring). When it runs out
40 of commits, it gets another job from the builder and starts again with that
43 Clearly the builder threads could work either way - they could check out a
44 commit and then built it for all boards. Using separate directories for each
45 commit/board pair they could leave their build product around afterwards
48 The intent behind building a single board for multiple commits, is to make
49 use of incremental builds. Since each commit is built incrementally from
50 the previous one, builds are faster. Reconfiguring for a different board
51 removes all intermediate object files.
53 Many threads can be working at once, but each has its own working directory.
54 When a thread finishes a build, it puts the output files into a result
57 The base directory used by buildman is normally '../<branch>', i.e.
58 a directory higher than the source repository and named after the branch
61 Within the base directory, we have one subdirectory for each commit. Within
62 that is one subdirectory for each board. Within that is the build output for
63 that commit/board combination.
65 Buildman also create working directories for each thread, in a .bm-work/
66 subdirectory in the base dir.
68 As an example, say we are building branch 'us-net' for boards 'sandbox' and
69 'seaboard', and say that us-net has two commits. We will have directories
72 us-net/ base directory
73 01_of_02_g4ed4ebc_net--Add-tftp-speed-/
78 02_of_02_g4ed4ebc_net--Check-tftp-comp/
84 00/ working directory for thread 0 (contains source checkout)
86 01/ working directory for thread 1
89 u-boot/ source directory
93 # Possible build outcomes
94 OUTCOME_OK, OUTCOME_WARNING, OUTCOME_ERROR, OUTCOME_UNKNOWN = range(4)
96 # Translate a commit subject into a valid filename
97 trans_valid_chars = string.maketrans("/: ", "---")
101 """Make a directory if it doesn't already exist.
104 dirname: Directory to create
108 except OSError as err:
109 if err.errno == errno.EEXIST:
115 """Holds information about a job to be performed by a thread
118 board: Board object to build
119 commits: List of commit options to build.
126 class ResultThread(threading.Thread):
127 """This thread processes results from builder threads.
129 It simply passes the results on to the builder. There is only one
130 result thread, and this helps to serialise the build output.
132 def __init__(self, builder):
133 """Set up a new result thread
136 builder: Builder which will be sent each result
138 threading.Thread.__init__(self)
139 self.builder = builder
142 """Called to start up the result thread.
144 We collect the next result job and pass it on to the build.
147 result = self.builder.out_queue.get()
148 self.builder.ProcessResult(result)
149 self.builder.out_queue.task_done()
152 class BuilderThread(threading.Thread):
153 """This thread builds U-Boot for a particular board.
155 An input queue provides each new job. We run 'make' to build U-Boot
156 and then pass the results on to the output queue.
159 builder: The builder which contains information we might need
160 thread_num: Our thread number (0-n-1), used to decide on a
163 def __init__(self, builder, thread_num):
164 """Set up a new builder thread"""
165 threading.Thread.__init__(self)
166 self.builder = builder
167 self.thread_num = thread_num
169 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
170 """Run 'make' on a particular commit and board.
172 The source code will already be checked out, so the 'commit'
173 argument is only for information.
176 commit: Commit object that is being built
177 brd: Board object that is being built
178 stage: Stage of the build. Valid stages are:
179 distclean - can be called to clean source
180 config - called to configure for a board
181 build - the main make invocation - it does the build
182 args: A list of arguments to pass to 'make'
183 kwargs: A list of keyword arguments to pass to command.RunPipe()
188 return self.builder.do_make(commit, brd, stage, cwd, *args,
191 def RunCommit(self, commit_upto, brd, work_dir, do_config, force_build,
192 force_build_failures):
193 """Build a particular commit.
195 If the build is already done, and we are not forcing a build, we skip
196 the build and just return the previously-saved results.
199 commit_upto: Commit number to build (0...n-1)
200 brd: Board object to build
201 work_dir: Directory to which the source will be checked out
202 do_config: True to run a make <board>_config on the source
203 force_build: Force a build even if one was previously done
204 force_build_failures: Force a bulid if the previous result showed
209 - CommandResult object containing the results of the build
210 - boolean indicating whether 'make config' is still needed
212 # Create a default result - it will be overwritte by the call to
213 # self.Make() below, in the event that we do a build.
214 result = command.CommandResult()
215 result.return_code = 0
216 if self.builder.in_tree:
219 out_dir = os.path.join(work_dir, 'build')
221 # Check if the job was already completed last time
222 done_file = self.builder.GetDoneFile(commit_upto, brd.target)
223 result.already_done = os.path.exists(done_file)
224 will_build = (force_build or force_build_failures or
225 not result.already_done)
226 if result.already_done and will_build:
227 # Get the return code from that build and use it
228 with open(done_file, 'r') as fd:
229 result.return_code = int(fd.readline())
230 err_file = self.builder.GetErrFile(commit_upto, brd.target)
231 if os.path.exists(err_file) and os.stat(err_file).st_size:
232 result.stderr = 'bad'
233 elif not force_build:
234 # The build passed, so no need to build it again
238 # We are going to have to build it. First, get a toolchain
239 if not self.toolchain:
241 self.toolchain = self.builder.toolchains.Select(brd.arch)
242 except ValueError as err:
243 result.return_code = 10
245 result.stderr = str(err)
246 # TODO(sjg@chromium.org): This gets swallowed, but needs
250 # Checkout the right commit
251 if commit_upto is not None:
252 commit = self.builder.commits[commit_upto]
253 if self.builder.checkout:
254 git_dir = os.path.join(work_dir, '.git')
255 gitutil.Checkout(commit.hash, git_dir, work_dir,
258 commit = self.builder.commit # Ick, fix this for BuildCommits()
260 # Set up the environment and command line
261 env = self.toolchain.MakeEnvironment()
264 if not self.builder.in_tree:
265 args.append('O=build')
267 if self.builder.num_jobs is not None:
268 args.extend(['-j', str(self.builder.num_jobs)])
269 config_args = ['%s_config' % brd.target]
271 args.extend(self.builder.toolchains.GetMakeArguments(brd))
273 # If we need to reconfigure, do that now
275 result = self.Make(commit, brd, 'distclean', work_dir,
276 'distclean', *args, env=env)
277 result = self.Make(commit, brd, 'config', work_dir,
278 *(args + config_args), env=env)
279 config_out = result.combined
280 do_config = False # No need to configure next time
281 if result.return_code == 0:
282 result = self.Make(commit, brd, 'build', work_dir, *args,
284 result.stdout = config_out + result.stdout
286 result.return_code = 1
287 result.stderr = 'No tool chain for %s\n' % brd.arch
288 result.already_done = False
290 result.toolchain = self.toolchain
292 result.commit_upto = commit_upto
293 result.out_dir = out_dir
294 return result, do_config
296 def _WriteResult(self, result, keep_outputs):
297 """Write a built result to the output directory.
300 result: CommandResult object containing result to write
301 keep_outputs: True to store the output binaries, False
305 if result.return_code < 0:
309 if result.stderr and 'No child processes' in result.stderr:
312 if result.already_done:
315 # Write the output and stderr
316 output_dir = self.builder._GetOutputDir(result.commit_upto)
318 build_dir = self.builder.GetBuildDir(result.commit_upto,
322 outfile = os.path.join(build_dir, 'log')
323 with open(outfile, 'w') as fd:
325 fd.write(result.stdout)
327 errfile = self.builder.GetErrFile(result.commit_upto,
330 with open(errfile, 'w') as fd:
331 fd.write(result.stderr)
332 elif os.path.exists(errfile):
336 # Write the build result and toolchain information.
337 done_file = self.builder.GetDoneFile(result.commit_upto,
339 with open(done_file, 'w') as fd:
340 fd.write('%s' % result.return_code)
341 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
342 print >>fd, 'gcc', result.toolchain.gcc
343 print >>fd, 'path', result.toolchain.path
344 print >>fd, 'cross', result.toolchain.cross
345 print >>fd, 'arch', result.toolchain.arch
346 fd.write('%s' % result.return_code)
348 with open(os.path.join(build_dir, 'toolchain'), 'w') as fd:
349 print >>fd, 'gcc', result.toolchain.gcc
350 print >>fd, 'path', result.toolchain.path
352 # Write out the image and function size information and an objdump
353 env = result.toolchain.MakeEnvironment()
355 for fname in ['u-boot', 'spl/u-boot-spl']:
356 cmd = ['%snm' % self.toolchain.cross, '--size-sort', fname]
357 nm_result = command.RunPipe([cmd], capture=True,
358 capture_stderr=True, cwd=result.out_dir,
359 raise_on_error=False, env=env)
361 nm = self.builder.GetFuncSizesFile(result.commit_upto,
362 result.brd.target, fname)
363 with open(nm, 'w') as fd:
364 print >>fd, nm_result.stdout,
366 cmd = ['%sobjdump' % self.toolchain.cross, '-h', fname]
367 dump_result = command.RunPipe([cmd], capture=True,
368 capture_stderr=True, cwd=result.out_dir,
369 raise_on_error=False, env=env)
371 if dump_result.stdout:
372 objdump = self.builder.GetObjdumpFile(result.commit_upto,
373 result.brd.target, fname)
374 with open(objdump, 'w') as fd:
375 print >>fd, dump_result.stdout,
376 for line in dump_result.stdout.splitlines():
377 fields = line.split()
378 if len(fields) > 5 and fields[1] == '.rodata':
379 rodata_size = fields[2]
381 cmd = ['%ssize' % self.toolchain.cross, fname]
382 size_result = command.RunPipe([cmd], capture=True,
383 capture_stderr=True, cwd=result.out_dir,
384 raise_on_error=False, env=env)
385 if size_result.stdout:
386 lines.append(size_result.stdout.splitlines()[1] + ' ' +
389 # Write out the image sizes file. This is similar to the output
390 # of binutil's 'size' utility, but it omits the header line and
391 # adds an additional hex value at the end of each line for the
394 sizes = self.builder.GetSizesFile(result.commit_upto,
396 with open(sizes, 'w') as fd:
397 print >>fd, '\n'.join(lines)
399 # Now write the actual build output
401 patterns = ['u-boot', '*.bin', 'u-boot.dtb', '*.map',
402 'include/autoconf.mk', 'spl/u-boot-spl',
403 'spl/u-boot-spl.bin']
404 for pattern in patterns:
405 file_list = glob.glob(os.path.join(result.out_dir, pattern))
406 for fname in file_list:
407 shutil.copy(fname, build_dir)
410 def RunJob(self, job):
413 A job consists of a building a list of commits for a particular board.
419 work_dir = self.builder.GetThreadDir(self.thread_num)
420 self.toolchain = None
422 # Run 'make board_config' on the first commit
426 for commit_upto in range(0, len(job.commits), job.step):
427 result, request_config = self.RunCommit(commit_upto, brd,
429 force_build or self.builder.force_build,
430 self.builder.force_build_failures)
431 failed = result.return_code or result.stderr
432 did_config = do_config
433 if failed and not do_config:
434 # If our incremental build failed, try building again
436 if self.builder.force_config_on_failure:
437 result, request_config = self.RunCommit(commit_upto,
438 brd, work_dir, True, True, False)
440 if not self.builder.force_reconfig:
441 do_config = request_config
443 # If we built that commit, then config is done. But if we got
444 # an warning, reconfig next time to force it to build the same
445 # files that created warnings this time. Otherwise an
446 # incremental build may not build the same file, and we will
447 # think that the warning has gone away.
448 # We could avoid this by using -Werror everywhere...
449 # For errors, the problem doesn't happen, since presumably
450 # the build stopped and didn't generate output, so will retry
451 # that file next time. So we could detect warnings and deal
452 # with them specially here. For now, we just reconfigure if
453 # anything goes work.
454 # Of course this is substantially slower if there are build
455 # errors/warnings (e.g. 2-3x slower even if only 10% of builds
457 if (failed and not result.already_done and not did_config and
458 self.builder.force_config_on_failure):
459 # If this build failed, try the next one with a
461 # Sometimes if the board_config.h file changes it can mess
462 # with dependencies, and we get:
463 # make: *** No rule to make target `include/autoconf.mk',
464 # needed by `depend'.
469 if self.builder.force_config_on_failure:
472 result.commit_upto = commit_upto
473 if result.return_code < 0:
474 raise ValueError('Interrupt')
476 # We have the build results, so output the result
477 self._WriteResult(result, job.keep_outputs)
478 self.builder.out_queue.put(result)
480 # Just build the currently checked-out build
481 result = self.RunCommit(None, True)
482 result.commit_upto = self.builder.upto
483 self.builder.out_queue.put(result)
486 """Our thread's run function
488 This thread picks a job from the queue, runs it, and then goes to the
493 job = self.builder.queue.get()
495 if self.builder.active and alive:
497 except Exception as err:
500 self.builder.queue.task_done()
504 """Class for building U-Boot for a particular commit.
506 Public members: (many should ->private)
507 active: True if the builder is active and has not been stopped
508 already_done: Number of builds already completed
509 base_dir: Base directory to use for builder
510 checkout: True to check out source, False to skip that step.
511 This is used for testing.
512 col: terminal.Color() object
513 count: Number of commits to build
514 do_make: Method to call to invoke Make
515 fail: Number of builds that failed due to error
516 force_build: Force building even if a build already exists
517 force_config_on_failure: If a commit fails for a board, disable
518 incremental building for the next commit we build for that
519 board, so that we will see all warnings/errors again.
520 force_build_failures: If a previously-built build (i.e. built on
521 a previous run of buildman) is marked as failed, rebuild it.
522 git_dir: Git directory containing source repository
523 last_line_len: Length of the last line we printed (used for erasing
524 it with new progress information)
525 num_jobs: Number of jobs to run at once (passed to make as -j)
526 num_threads: Number of builder threads to run
527 out_queue: Queue of results to process
528 re_make_err: Compiled regular expression for ignore_lines
529 queue: Queue of jobs to run
530 threads: List of active threads
531 toolchains: Toolchains object to use for building
532 upto: Current commit number we are building (0.count-1)
533 warned: Number of builds that produced at least one warning
534 force_reconfig: Reconfigure U-Boot on each comiit. This disables
535 incremental building, where buildman reconfigures on the first
536 commit for a baord, and then just does an incremental build for
537 the following commits. In fact buildman will reconfigure and
538 retry for any failing commits, so generally the only effect of
539 this option is to slow things down.
540 in_tree: Build U-Boot in-tree instead of specifying an output
541 directory separate from the source code. This option is really
542 only useful for testing in-tree builds.
545 _base_board_dict: Last-summarised Dict of boards
546 _base_err_lines: Last-summarised list of errors
547 _build_period_us: Time taken for a single build (float object).
548 _complete_delay: Expected delay until completion (timedelta)
549 _next_delay_update: Next time we plan to display a progress update
551 _show_unknown: Show unknown boards (those not built) in summary
552 _timestamps: List of timestamps for the completion of the last
553 last _timestamp_count builds. Each is a datetime object.
554 _timestamp_count: Number of timestamps to keep in our list.
555 _working_dir: Base working directory containing all threads
558 """Records a build outcome for a single make invocation
561 rc: Outcome value (OUTCOME_...)
562 err_lines: List of error lines or [] if none
563 sizes: Dictionary of image size information, keyed by filename
564 - Each value is itself a dictionary containing
565 values for 'text', 'data' and 'bss', being the integer
566 size in bytes of each section.
567 func_sizes: Dictionary keyed by filename - e.g. 'u-boot'. Each
568 value is itself a dictionary:
570 value: Size of function in bytes
572 def __init__(self, rc, err_lines, sizes, func_sizes):
574 self.err_lines = err_lines
576 self.func_sizes = func_sizes
578 def __init__(self, toolchains, base_dir, git_dir, num_threads, num_jobs,
579 checkout=True, show_unknown=True, step=1):
580 """Create a new Builder object
583 toolchains: Toolchains object to use for building
584 base_dir: Base directory to use for builder
585 git_dir: Git directory containing source repository
586 num_threads: Number of builder threads to run
587 num_jobs: Number of jobs to run at once (passed to make as -j)
588 checkout: True to check out source, False to skip that step.
589 This is used for testing.
590 show_unknown: Show unknown boards (those not built) in summary
591 step: 1 to process every commit, n to process every nth commit
593 self.toolchains = toolchains
594 self.base_dir = base_dir
595 self._working_dir = os.path.join(base_dir, '.bm-work')
598 self.do_make = self.Make
599 self.checkout = checkout
600 self.num_threads = num_threads
601 self.num_jobs = num_jobs
602 self.already_done = 0
603 self.force_build = False
604 self.git_dir = git_dir
605 self._show_unknown = show_unknown
606 self._timestamp_count = 10
607 self._build_period_us = None
608 self._complete_delay = None
609 self._next_delay_update = datetime.now()
610 self.force_config_on_failure = True
611 self.force_build_failures = False
612 self.force_reconfig = False
616 self.col = terminal.Color()
618 self.queue = Queue.Queue()
619 self.out_queue = Queue.Queue()
620 for i in range(self.num_threads):
621 t = BuilderThread(self, i)
624 self.threads.append(t)
626 self.last_line_len = 0
627 t = ResultThread(self)
630 self.threads.append(t)
632 ignore_lines = ['(make.*Waiting for unfinished)', '(Segmentation fault)']
633 self.re_make_err = re.compile('|'.join(ignore_lines))
636 """Get rid of all threads created by the builder"""
637 for t in self.threads:
640 def _AddTimestamp(self):
641 """Add a new timestamp to the list and record the build period.
643 The build period is the length of time taken to perform a single
644 build (one board, one commit).
647 self._timestamps.append(now)
648 count = len(self._timestamps)
649 delta = self._timestamps[-1] - self._timestamps[0]
650 seconds = delta.total_seconds()
652 # If we have enough data, estimate build period (time taken for a
653 # single build) and therefore completion time.
654 if count > 1 and self._next_delay_update < now:
655 self._next_delay_update = now + timedelta(seconds=2)
657 self._build_period = float(seconds) / count
658 todo = self.count - self.upto
659 self._complete_delay = timedelta(microseconds=
660 self._build_period * todo * 1000000)
662 self._complete_delay -= timedelta(
663 microseconds=self._complete_delay.microseconds)
666 self._timestamps.popleft()
669 def ClearLine(self, length):
670 """Clear any characters on the current line
672 Make way for a new line of length 'length', by outputting enough
673 spaces to clear out the old line. Then remember the new length for
677 length: Length of new line, in characters
679 if length < self.last_line_len:
680 print ' ' * (self.last_line_len - length),
682 self.last_line_len = length
685 def SelectCommit(self, commit, checkout=True):
686 """Checkout the selected commit for this build
689 if checkout and self.checkout:
690 gitutil.Checkout(commit.hash)
692 def Make(self, commit, brd, stage, cwd, *args, **kwargs):
696 commit: Commit object that is being built
697 brd: Board object that is being built
698 stage: Stage that we are at (distclean, config, build)
699 cwd: Directory where make should be run
700 args: Arguments to pass to make
701 kwargs: Arguments to pass to command.RunPipe()
703 cmd = ['make'] + list(args)
704 result = command.RunPipe([cmd], capture=True, capture_stderr=True,
705 cwd=cwd, raise_on_error=False, **kwargs)
708 def ProcessResult(self, result):
709 """Process the result of a build, showing progress information
712 result: A CommandResult object
714 col = terminal.Color()
716 target = result.brd.target
718 if result.return_code < 0:
724 if result.return_code != 0:
728 if result.already_done:
729 self.already_done += 1
731 target = '(starting)'
733 # Display separate counts for ok, warned and fail
734 ok = self.upto - self.warned - self.fail
735 line = '\r' + self.col.Color(self.col.GREEN, '%5d' % ok)
736 line += self.col.Color(self.col.YELLOW, '%5d' % self.warned)
737 line += self.col.Color(self.col.RED, '%5d' % self.fail)
739 name = ' /%-5d ' % self.count
741 # Add our current completion time estimate
743 if self._complete_delay:
744 name += '%s : ' % self._complete_delay
745 # When building all boards for a commit, we can print a commit
747 if result and result.commit_upto is None:
748 name += 'commit %2d/%-3d' % (self.commit_upto + 1,
753 length = 13 + len(name)
754 self.ClearLine(length)
756 def _GetOutputDir(self, commit_upto):
757 """Get the name of the output directory for a commit number
759 The output directory is typically .../<branch>/<commit>.
762 commit_upto: Commit number to use (0..self.count-1)
764 commit = self.commits[commit_upto]
765 subject = commit.subject.translate(trans_valid_chars)
766 commit_dir = ('%02d_of_%02d_g%s_%s' % (commit_upto + 1,
767 self.commit_count, commit.hash, subject[:20]))
768 output_dir = os.path.join(self.base_dir, commit_dir)
771 def GetBuildDir(self, commit_upto, target):
772 """Get the name of the build directory for a commit number
774 The build directory is typically .../<branch>/<commit>/<target>.
777 commit_upto: Commit number to use (0..self.count-1)
780 output_dir = self._GetOutputDir(commit_upto)
781 return os.path.join(output_dir, target)
783 def GetDoneFile(self, commit_upto, target):
784 """Get the name of the done file for a commit number
787 commit_upto: Commit number to use (0..self.count-1)
790 return os.path.join(self.GetBuildDir(commit_upto, target), 'done')
792 def GetSizesFile(self, commit_upto, target):
793 """Get the name of the sizes file for a commit number
796 commit_upto: Commit number to use (0..self.count-1)
799 return os.path.join(self.GetBuildDir(commit_upto, target), 'sizes')
801 def GetFuncSizesFile(self, commit_upto, target, elf_fname):
802 """Get the name of the funcsizes file for a commit number and ELF file
805 commit_upto: Commit number to use (0..self.count-1)
807 elf_fname: Filename of elf image
809 return os.path.join(self.GetBuildDir(commit_upto, target),
810 '%s.sizes' % elf_fname.replace('/', '-'))
812 def GetObjdumpFile(self, commit_upto, target, elf_fname):
813 """Get the name of the objdump file for a commit number and ELF file
816 commit_upto: Commit number to use (0..self.count-1)
818 elf_fname: Filename of elf image
820 return os.path.join(self.GetBuildDir(commit_upto, target),
821 '%s.objdump' % elf_fname.replace('/', '-'))
823 def GetErrFile(self, commit_upto, target):
824 """Get the name of the err file for a commit number
827 commit_upto: Commit number to use (0..self.count-1)
830 output_dir = self.GetBuildDir(commit_upto, target)
831 return os.path.join(output_dir, 'err')
833 def FilterErrors(self, lines):
834 """Filter out errors in which we have no interest
836 We should probably use map().
839 lines: List of error lines, each a string
841 New list with only interesting lines included
845 if not self.re_make_err.search(line):
846 out_lines.append(line)
849 def ReadFuncSizes(self, fname, fd):
850 """Read function sizes from the output of 'nm'
853 fd: File containing data to read
854 fname: Filename we are reading from (just for errors)
857 Dictionary containing size of each function in bytes, indexed by
861 for line in fd.readlines():
863 size, type, name = line[:-1].split()
865 print "Invalid line in file '%s': '%s'" % (fname, line[:-1])
868 # function names begin with '.' on 64-bit powerpc
870 name = 'static.' + name.split('.')[0]
871 sym[name] = sym.get(name, 0) + int(size, 16)
874 def GetBuildOutcome(self, commit_upto, target, read_func_sizes):
875 """Work out the outcome of a build.
878 commit_upto: Commit number to check (0..n-1)
879 target: Target board to check
880 read_func_sizes: True to read function size information
885 done_file = self.GetDoneFile(commit_upto, target)
886 sizes_file = self.GetSizesFile(commit_upto, target)
889 if os.path.exists(done_file):
890 with open(done_file, 'r') as fd:
891 return_code = int(fd.readline())
893 err_file = self.GetErrFile(commit_upto, target)
894 if os.path.exists(err_file):
895 with open(err_file, 'r') as fd:
896 err_lines = self.FilterErrors(fd.readlines())
898 # Decide whether the build was ok, failed or created warnings
906 # Convert size information to our simple format
907 if os.path.exists(sizes_file):
908 with open(sizes_file, 'r') as fd:
909 for line in fd.readlines():
910 values = line.split()
913 rodata = int(values[6], 16)
915 'all' : int(values[0]) + int(values[1]) +
917 'text' : int(values[0]) - rodata,
918 'data' : int(values[1]),
919 'bss' : int(values[2]),
922 sizes[values[5]] = size_dict
925 pattern = self.GetFuncSizesFile(commit_upto, target, '*')
926 for fname in glob.glob(pattern):
927 with open(fname, 'r') as fd:
928 dict_name = os.path.basename(fname).replace('.sizes',
930 func_sizes[dict_name] = self.ReadFuncSizes(fname, fd)
932 return Builder.Outcome(rc, err_lines, sizes, func_sizes)
934 return Builder.Outcome(OUTCOME_UNKNOWN, [], {}, {})
936 def GetResultSummary(self, boards_selected, commit_upto, read_func_sizes):
937 """Calculate a summary of the results of building a commit.
940 board_selected: Dict containing boards to summarise
941 commit_upto: Commit number to summarize (0..self.count-1)
942 read_func_sizes: True to read function size information
946 Dict containing boards which passed building this commit.
947 keyed by board.target
948 List containing a summary of error/warning lines
951 err_lines_summary = []
953 for board in boards_selected.itervalues():
954 outcome = self.GetBuildOutcome(commit_upto, board.target,
956 board_dict[board.target] = outcome
957 for err in outcome.err_lines:
958 if err and not err.rstrip() in err_lines_summary:
959 err_lines_summary.append(err.rstrip())
960 return board_dict, err_lines_summary
962 def AddOutcome(self, board_dict, arch_list, changes, char, color):
963 """Add an output to our list of outcomes for each architecture
965 This simple function adds failing boards (changes) to the
966 relevant architecture string, so we can print the results out
967 sorted by architecture.
970 board_dict: Dict containing all boards
971 arch_list: Dict keyed by arch name. Value is a string containing
972 a list of board names which failed for that arch.
973 changes: List of boards to add to arch_list
974 color: terminal.Colour object
977 for target in changes:
978 if target in board_dict:
979 arch = board_dict[target].arch
982 str = self.col.Color(color, ' ' + target)
983 if not arch in done_arch:
984 str = self.col.Color(color, char) + ' ' + str
985 done_arch[arch] = True
986 if not arch in arch_list:
987 arch_list[arch] = str
989 arch_list[arch] += str
992 def ColourNum(self, num):
993 color = self.col.RED if num > 0 else self.col.GREEN
996 return self.col.Color(color, str(num))
998 def ResetResultSummary(self, board_selected):
999 """Reset the results summary ready for use.
1001 Set up the base board list to be all those selected, and set the
1002 error lines to empty.
1004 Following this, calls to PrintResultSummary() will use this
1005 information to work out what has changed.
1008 board_selected: Dict containing boards to summarise, keyed by
1011 self._base_board_dict = {}
1012 for board in board_selected:
1013 self._base_board_dict[board] = Builder.Outcome(0, [], [], {})
1014 self._base_err_lines = []
1016 def PrintFuncSizeDetail(self, fname, old, new):
1017 grow, shrink, add, remove, up, down = 0, 0, 0, 0, 0, 0
1018 delta, common = [], {}
1025 if name not in common:
1028 delta.append([-old[name], name])
1031 if name not in common:
1034 delta.append([new[name], name])
1037 diff = new.get(name, 0) - old.get(name, 0)
1039 grow, up = grow + 1, up + diff
1041 shrink, down = shrink + 1, down - diff
1042 delta.append([diff, name])
1047 args = [add, -remove, grow, -shrink, up, -down, up - down]
1050 args = [self.ColourNum(x) for x in args]
1052 print ('%s%s: add: %s/%s, grow: %s/%s bytes: %s/%s (%s)' %
1053 tuple([indent, self.col.Color(self.col.YELLOW, fname)] + args))
1054 print '%s %-38s %7s %7s %+7s' % (indent, 'function', 'old', 'new',
1056 for diff, name in delta:
1058 color = self.col.RED if diff > 0 else self.col.GREEN
1059 msg = '%s %-38s %7s %7s %+7d' % (indent, name,
1060 old.get(name, '-'), new.get(name,'-'), diff)
1061 print self.col.Color(color, msg)
1064 def PrintSizeDetail(self, target_list, show_bloat):
1065 """Show details size information for each board
1068 target_list: List of targets, each a dict containing:
1069 'target': Target name
1070 'total_diff': Total difference in bytes across all areas
1071 <part_name>: Difference for that part
1072 show_bloat: Show detail for each function
1074 targets_by_diff = sorted(target_list, reverse=True,
1075 key=lambda x: x['_total_diff'])
1076 for result in targets_by_diff:
1077 printed_target = False
1078 for name in sorted(result):
1080 if name.startswith('_'):
1083 color = self.col.RED if diff > 0 else self.col.GREEN
1084 msg = ' %s %+d' % (name, diff)
1085 if not printed_target:
1086 print '%10s %-15s:' % ('', result['_target']),
1087 printed_target = True
1088 print self.col.Color(color, msg),
1092 target = result['_target']
1093 outcome = result['_outcome']
1094 base_outcome = self._base_board_dict[target]
1095 for fname in outcome.func_sizes:
1096 self.PrintFuncSizeDetail(fname,
1097 base_outcome.func_sizes[fname],
1098 outcome.func_sizes[fname])
1101 def PrintSizeSummary(self, board_selected, board_dict, show_detail,
1103 """Print a summary of image sizes broken down by section.
1105 The summary takes the form of one line per architecture. The
1106 line contains deltas for each of the sections (+ means the section
1107 got bigger, - means smaller). The nunmbers are the average number
1108 of bytes that a board in this section increased by.
1111 powerpc: (622 boards) text -0.0
1112 arm: (285 boards) text -0.0
1113 nds32: (3 boards) text -8.0
1116 board_selected: Dict containing boards to summarise, keyed by
1118 board_dict: Dict containing boards for which we built this
1119 commit, keyed by board.target. The value is an Outcome object.
1120 show_detail: Show detail for each board
1121 show_bloat: Show detail for each function
1126 # Calculate changes in size for different image parts
1127 # The previous sizes are in Board.sizes, for each board
1128 for target in board_dict:
1129 if target not in board_selected:
1131 base_sizes = self._base_board_dict[target].sizes
1132 outcome = board_dict[target]
1133 sizes = outcome.sizes
1135 # Loop through the list of images, creating a dict of size
1136 # changes for each image/part. We end up with something like
1137 # {'target' : 'snapper9g45, 'data' : 5, 'u-boot-spl:text' : -4}
1138 # which means that U-Boot data increased by 5 bytes and SPL
1139 # text decreased by 4.
1140 err = {'_target' : target}
1142 if image in base_sizes:
1143 base_image = base_sizes[image]
1144 # Loop through the text, data, bss parts
1145 for part in sorted(sizes[image]):
1146 diff = sizes[image][part] - base_image[part]
1149 if image == 'u-boot':
1152 name = image + ':' + part
1154 arch = board_selected[target].arch
1155 if not arch in arch_count:
1156 arch_count[arch] = 1
1158 arch_count[arch] += 1
1160 pass # Only add to our list when we have some stats
1161 elif not arch in arch_list:
1162 arch_list[arch] = [err]
1164 arch_list[arch].append(err)
1166 # We now have a list of image size changes sorted by arch
1167 # Print out a summary of these
1168 for arch, target_list in arch_list.iteritems():
1169 # Get total difference for each type
1171 for result in target_list:
1173 for name, diff in result.iteritems():
1174 if name.startswith('_'):
1178 totals[name] += diff
1181 result['_total_diff'] = total
1182 result['_outcome'] = board_dict[result['_target']]
1184 count = len(target_list)
1185 printed_arch = False
1186 for name in sorted(totals):
1189 # Display the average difference in this name for this
1191 avg_diff = float(diff) / count
1192 color = self.col.RED if avg_diff > 0 else self.col.GREEN
1193 msg = ' %s %+1.1f' % (name, avg_diff)
1194 if not printed_arch:
1195 print '%10s: (for %d/%d boards)' % (arch, count,
1198 print self.col.Color(color, msg),
1203 self.PrintSizeDetail(target_list, show_bloat)
1206 def PrintResultSummary(self, board_selected, board_dict, err_lines,
1207 show_sizes, show_detail, show_bloat):
1208 """Compare results with the base results and display delta.
1210 Only boards mentioned in board_selected will be considered. This
1211 function is intended to be called repeatedly with the results of
1212 each commit. It therefore shows a 'diff' between what it saw in
1213 the last call and what it sees now.
1216 board_selected: Dict containing boards to summarise, keyed by
1218 board_dict: Dict containing boards for which we built this
1219 commit, keyed by board.target. The value is an Outcome object.
1220 err_lines: A list of errors for this commit, or [] if there is
1221 none, or we don't want to print errors
1222 show_sizes: Show image size deltas
1223 show_detail: Show detail for each board
1224 show_bloat: Show detail for each function
1226 better = [] # List of boards fixed since last commit
1227 worse = [] # List of new broken boards since last commit
1228 new = [] # List of boards that didn't exist last time
1229 unknown = [] # List of boards that were not built
1231 for target in board_dict:
1232 if target not in board_selected:
1235 # If the board was built last time, add its outcome to a list
1236 if target in self._base_board_dict:
1237 base_outcome = self._base_board_dict[target].rc
1238 outcome = board_dict[target]
1239 if outcome.rc == OUTCOME_UNKNOWN:
1240 unknown.append(target)
1241 elif outcome.rc < base_outcome:
1242 better.append(target)
1243 elif outcome.rc > base_outcome:
1244 worse.append(target)
1248 # Get a list of errors that have appeared, and disappeared
1251 for line in err_lines:
1252 if line not in self._base_err_lines:
1253 worse_err.append('+' + line)
1254 for line in self._base_err_lines:
1255 if line not in err_lines:
1256 better_err.append('-' + line)
1258 # Display results by arch
1259 if better or worse or unknown or new or worse_err or better_err:
1261 self.AddOutcome(board_selected, arch_list, better, '',
1263 self.AddOutcome(board_selected, arch_list, worse, '+',
1265 self.AddOutcome(board_selected, arch_list, new, '*', self.col.BLUE)
1266 if self._show_unknown:
1267 self.AddOutcome(board_selected, arch_list, unknown, '?',
1269 for arch, target_list in arch_list.iteritems():
1270 print '%10s: %s' % (arch, target_list)
1272 print self.col.Color(self.col.GREEN, '\n'.join(better_err))
1274 print self.col.Color(self.col.RED, '\n'.join(worse_err))
1277 self.PrintSizeSummary(board_selected, board_dict, show_detail,
1280 # Save our updated information for the next call to this function
1281 self._base_board_dict = board_dict
1282 self._base_err_lines = err_lines
1284 # Get a list of boards that did not get built, if needed
1286 for board in board_selected:
1287 if not board in board_dict:
1288 not_built.append(board)
1290 print "Boards not built (%d): %s" % (len(not_built),
1291 ', '.join(not_built))
1294 def ShowSummary(self, commits, board_selected, show_errors, show_sizes,
1295 show_detail, show_bloat):
1296 """Show a build summary for U-Boot for a given board list.
1298 Reset the result summary, then repeatedly call GetResultSummary on
1299 each commit's results, then display the differences we see.
1302 commit: Commit objects to summarise
1303 board_selected: Dict containing boards to summarise
1304 show_errors: Show errors that occured
1305 show_sizes: Show size deltas
1306 show_detail: Show detail for each board
1307 show_bloat: Show detail for each function
1309 self.commit_count = len(commits)
1310 self.commits = commits
1311 self.ResetResultSummary(board_selected)
1313 for commit_upto in range(0, self.commit_count, self._step):
1314 board_dict, err_lines = self.GetResultSummary(board_selected,
1315 commit_upto, read_func_sizes=show_bloat)
1316 msg = '%02d: %s' % (commit_upto + 1, commits[commit_upto].subject)
1317 print self.col.Color(self.col.BLUE, msg)
1318 self.PrintResultSummary(board_selected, board_dict,
1319 err_lines if show_errors else [], show_sizes, show_detail,
1323 def SetupBuild(self, board_selected, commits):
1324 """Set up ready to start a build.
1327 board_selected: Selected boards to build
1328 commits: Selected commits to build
1330 # First work out how many commits we will build
1331 count = (len(commits) + self._step - 1) / self._step
1332 self.count = len(board_selected) * count
1333 self.upto = self.warned = self.fail = 0
1334 self._timestamps = collections.deque()
1336 def BuildBoardsForCommit(self, board_selected, keep_outputs):
1337 """Build all boards for a single commit"""
1338 self.SetupBuild(board_selected)
1339 self.count = len(board_selected)
1340 for brd in board_selected.itervalues():
1344 job.keep_outputs = keep_outputs
1348 self.out_queue.join()
1352 def BuildCommits(self, commits, board_selected, show_errors, keep_outputs):
1353 """Build all boards for all commits (non-incremental)"""
1354 self.commit_count = len(commits)
1356 self.ResetResultSummary(board_selected)
1357 for self.commit_upto in range(self.commit_count):
1358 self.SelectCommit(commits[self.commit_upto])
1359 self.SelectOutputDir()
1360 Mkdir(self.output_dir)
1362 self.BuildBoardsForCommit(board_selected, keep_outputs)
1363 board_dict, err_lines = self.GetResultSummary()
1364 self.PrintResultSummary(board_selected, board_dict,
1365 err_lines if show_errors else [])
1367 if self.already_done:
1368 print '%d builds already done' % self.already_done
1370 def GetThreadDir(self, thread_num):
1371 """Get the directory path to the working dir for a thread.
1374 thread_num: Number of thread to check.
1376 return os.path.join(self._working_dir, '%02d' % thread_num)
1378 def _PrepareThread(self, thread_num):
1379 """Prepare the working directory for a thread.
1381 This clones or fetches the repo into the thread's work directory.
1384 thread_num: Thread number (0, 1, ...)
1386 thread_dir = self.GetThreadDir(thread_num)
1388 git_dir = os.path.join(thread_dir, '.git')
1390 # Clone the repo if it doesn't already exist
1391 # TODO(sjg@chromium): Perhaps some git hackery to symlink instead, so
1392 # we have a private index but uses the origin repo's contents?
1394 src_dir = os.path.abspath(self.git_dir)
1395 if os.path.exists(git_dir):
1396 gitutil.Fetch(git_dir, thread_dir)
1398 print 'Cloning repo for thread %d' % thread_num
1399 gitutil.Clone(src_dir, thread_dir)
1401 def _PrepareWorkingSpace(self, max_threads):
1402 """Prepare the working directory for use.
1404 Set up the git repo for each thread.
1407 max_threads: Maximum number of threads we expect to need.
1409 Mkdir(self._working_dir)
1410 for thread in range(max_threads):
1411 self._PrepareThread(thread)
1413 def _PrepareOutputSpace(self):
1414 """Get the output directories ready to receive files.
1416 We delete any output directories which look like ones we need to
1417 create. Having left over directories is confusing when the user wants
1418 to check the output manually.
1421 for commit_upto in range(self.commit_count):
1422 dir_list.append(self._GetOutputDir(commit_upto))
1424 for dirname in glob.glob(os.path.join(self.base_dir, '*')):
1425 if dirname not in dir_list:
1426 shutil.rmtree(dirname)
1428 def BuildBoards(self, commits, board_selected, show_errors, keep_outputs):
1429 """Build all commits for a list of boards
1432 commits: List of commits to be build, each a Commit object
1433 boards_selected: Dict of selected boards, key is target name,
1434 value is Board object
1435 show_errors: True to show summarised error/warning info
1436 keep_outputs: True to save build output files
1438 self.commit_count = len(commits)
1439 self.commits = commits
1441 self.ResetResultSummary(board_selected)
1442 Mkdir(self.base_dir)
1443 self._PrepareWorkingSpace(min(self.num_threads, len(board_selected)))
1444 self._PrepareOutputSpace()
1445 self.SetupBuild(board_selected, commits)
1446 self.ProcessResult(None)
1448 # Create jobs to build all commits for each board
1449 for brd in board_selected.itervalues():
1452 job.commits = commits
1453 job.keep_outputs = keep_outputs
1454 job.step = self._step
1457 # Wait until all jobs are started
1460 # Wait until we have processed all output
1461 self.out_queue.join()