tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 import toolchain
  17 import command
  18 import subprocess
  19
  20 def GetPlural(count):
  21     """Returns a plural 's' if count is not 1"""
  22     return 's' if count != 1 else ''
  23
  24 def GetActionSummary(is_summary, commits, selected, options):
  25     """Return a string summarising the intended action.
  26
  27     Returns:
  28         Summary string.
  29     """
  30     if commits:
  31         count = len(commits)
  32         count = (count + options.step - 1) / options.step
  33         commit_str = '%d commit%s' % (count, GetPlural(count))
  34     else:
  35         commit_str = 'current source'
  36     str = '%s %s for %d boards' % (
  37         'Summary of' if is_summary else 'Building', commit_str,
  38         len(selected))
  39     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  40             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  41     return str
  42
  43 def ShowActions(series, why_selected, boards_selected, builder, options):
  44     """Display a list of actions that we would take, if not a dry run.
  45
  46     Args:
  47         series: Series object
  48         why_selected: Dictionary where each key is a buildman argument
  49                 provided by the user, and the value is the boards brought
  50                 in by that argument. For example, 'arm' might bring in
  51                 400 boards, so in this case the key would be 'arm' and
  52                 the value would be a list of board names.
  53         boards_selected: Dict of selected boards, key is target name,
  54                 value is Board object
  55         builder: The builder that will be used to build the commits
  56         options: Command line options object
  57     """
  58     col = terminal.Color()
  59     print 'Dry run, so not doing much. But I would do this:'
  60     print
  61     if series:
  62         commits = series.commits
  63     else:
  64         commits = None
  65     print GetActionSummary(False, commits, boards_selected,
  66             options)
  67     print 'Build directory: %s' % builder.base_dir
  68     if commits:
  69         for upto in range(0, len(series.commits), options.step):
  70             commit = series.commits[upto]
  71             print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
  72             print commit.subject
  73     print
  74     for arg in why_selected:
  75         if arg != 'all':
  76             print arg, ': %d boards' % why_selected[arg]
  77     print ('Total boards to build for each commit: %d\n' %
  78             why_selected['all'])
  79
  80 def DoBuildman(options, args):
  81     """The main control code for buildman
  82
  83     Args:
  84         options: Command line options object
  85         args: Command line arguments (list of strings)
  86     """
  87     gitutil.Setup()
  88
  89     bsettings.Setup(options.config_file)
  90     options.git_dir = os.path.join(options.git, '.git')
  91
  92     toolchains = toolchain.Toolchains()
  93     toolchains.Scan(options.list_tool_chains)
  94     if options.list_tool_chains:
  95         toolchains.List()
  96         print
  97         return 0
  98
  99     # Work out how many commits to build. We want to build everything on the
 100     # branch. We also build the upstream commit as a control so we can see
 101     # problems introduced by the first commit on the branch.
 102     col = terminal.Color()
 103     count = options.count
 104     if count == -1:
 105         if not options.branch:
 106             count = 1
 107         else:
 108             count = gitutil.CountCommitsInBranch(options.git_dir,
 109                                                  options.branch)
 110             if count is None:
 111                 str = ("Branch '%s' not found or has no upstream" %
 112                        options.branch)
 113                 sys.exit(col.Color(col.RED, str))
 114             count += 1   # Build upstream commit also
 115
 116     if not count:
 117         str = ("No commits found to process in branch '%s': "
 118                "set branch's upstream or use -c flag" % options.branch)
 119         sys.exit(col.Color(col.RED, str))
 120
 121     # Work out what subset of the boards we are building
 122     board_file = os.path.join(options.git, 'boards.cfg')
 123     status = subprocess.call([os.path.join(options.git,
 124                                            'tools/genboardscfg.py')])
 125     if status != 0:
 126         sys.exit("Failed to generate boards.cfg")
 127
 128     boards = board.Boards()
 129     boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 130
 131     exclude = []
 132     if options.exclude:
 133         for arg in options.exclude:
 134             exclude += arg.split(',')
 135
 136     why_selected = boards.SelectBoards(args, exclude)
 137     selected = boards.GetSelected()
 138     if not len(selected):
 139         sys.exit(col.Color(col.RED, 'No matching boards found'))
 140
 141     # Read the metadata from the commits. First look at the upstream commit,
 142     # then the ones in the branch. We would like to do something like
 143     # upstream/master~..branch but that isn't possible if upstream/master is
 144     # a merge commit (it will list all the commits that form part of the
 145     # merge)
 146     if options.branch:
 147         if count == -1:
 148             range_expr = gitutil.GetRangeInBranch(options.git_dir,
 149                                                   options.branch)
 150             upstream_commit = gitutil.GetUpstream(options.git_dir,
 151                                                   options.branch)
 152             series = patchstream.GetMetaDataForList(upstream_commit,
 153                 options.git_dir, 1)
 154
 155             # Conflicting tags are not a problem for buildman, since it does
 156             # not use them. For example, Series-version is not useful for
 157             # buildman. On the other hand conflicting tags will cause an
 158             # error. So allow later tags to overwrite earlier ones.
 159             series.allow_overwrite = True
 160             series = patchstream.GetMetaDataForList(range_expr,
 161                                               options.git_dir, None, series)
 162         else:
 163             # Honour the count
 164             series = patchstream.GetMetaDataForList(options.branch,
 165                                                     options.git_dir, count)
 166     else:
 167         series = None
 168         options.verbose = True
 169         options.show_errors = True
 170
 171     # By default we have one thread per CPU. But if there are not enough jobs
 172     # we can have fewer threads and use a high '-j' value for make.
 173     if not options.threads:
 174         options.threads = min(multiprocessing.cpu_count(), len(selected))
 175     if not options.jobs:
 176         options.jobs = max(1, (multiprocessing.cpu_count() +
 177                 len(selected) - 1) / len(selected))
 178
 179     if not options.step:
 180         options.step = len(series.commits) - 1
 181
 182     gnu_make = command.Output(os.path.join(options.git,
 183                                            'scripts/show-gnu-make')).rstrip()
 184     if not gnu_make:
 185         sys.exit('GNU Make not found')
 186
 187     # Create a new builder with the selected options
 188     if options.branch:
 189         dirname = options.branch
 190     else:
 191         dirname = 'current'
 192     output_dir = os.path.join(options.output_dir, dirname)
 193     builder = Builder(toolchains, output_dir, options.git_dir,
 194             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 195             show_unknown=options.show_unknown, step=options.step)
 196     builder.force_config_on_failure = not options.quick
 197
 198     # For a dry run, just show our actions as a sanity check
 199     if options.dry_run:
 200         ShowActions(series, why_selected, selected, builder, options)
 201     else:
 202         builder.force_build = options.force_build
 203         builder.force_build_failures = options.force_build_failures
 204         builder.force_reconfig = options.force_reconfig
 205         builder.in_tree = options.in_tree
 206
 207         # Work out which boards to build
 208         board_selected = boards.GetSelectedDict()
 209
 210         if series:
 211             commits = series.commits
 212         else:
 213             commits = None
 214
 215         print GetActionSummary(options.summary, commits, board_selected,
 216                                options)
 217
 218         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 219                                   options.show_detail, options.show_bloat,
 220                                   options.list_error_boards)
 221         if options.summary:
 222             # We can't show function sizes without board details at present
 223             if options.show_bloat:
 224                 options.show_detail = True
 225             builder.ShowSummary(commits, board_selected)
 226         else:
 227             fail, warned = builder.BuildBoards(commits, board_selected,
 228                                 options.keep_outputs, options.verbose)
 229             if fail:
 230                 return 128
 231             elif warned:
 232                 return 129
 233     return 0