tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 import toolchain
  17 import command
  18 import subprocess
  19
  20 def GetPlural(count):
  21     """Returns a plural 's' if count is not 1"""
  22     return 's' if count != 1 else ''
  23
  24 def GetActionSummary(is_summary, commits, selected, options):
  25     """Return a string summarising the intended action.
  26
  27     Returns:
  28         Summary string.
  29     """
  30     if commits:
  31         count = len(commits)
  32         count = (count + options.step - 1) / options.step
  33         commit_str = '%d commit%s' % (count, GetPlural(count))
  34     else:
  35         commit_str = 'current source'
  36     str = '%s %s for %d boards' % (
  37         'Summary of' if is_summary else 'Building', commit_str,
  38         len(selected))
  39     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  40             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  41     return str
  42
  43 def ShowActions(series, why_selected, boards_selected, builder, options):
  44     """Display a list of actions that we would take, if not a dry run.
  45
  46     Args:
  47         series: Series object
  48         why_selected: Dictionary where each key is a buildman argument
  49                 provided by the user, and the value is the boards brought
  50                 in by that argument. For example, 'arm' might bring in
  51                 400 boards, so in this case the key would be 'arm' and
  52                 the value would be a list of board names.
  53         boards_selected: Dict of selected boards, key is target name,
  54                 value is Board object
  55         builder: The builder that will be used to build the commits
  56         options: Command line options object
  57     """
  58     col = terminal.Color()
  59     print 'Dry run, so not doing much. But I would do this:'
  60     print
  61     if series:
  62         commits = series.commits
  63     else:
  64         commits = None
  65     print GetActionSummary(False, commits, boards_selected,
  66             options)
  67     print 'Build directory: %s' % builder.base_dir
  68     if commits:
  69         for upto in range(0, len(series.commits), options.step):
  70             commit = series.commits[upto]
  71             print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
  72             print commit.subject
  73     print
  74     for arg in why_selected:
  75         if arg != 'all':
  76             print arg, ': %d boards' % why_selected[arg]
  77     print ('Total boards to build for each commit: %d\n' %
  78             why_selected['all'])
  79
  80 def DoBuildman(options, args):
  81     """The main control code for buildman
  82
  83     Args:
  84         options: Command line options object
  85         args: Command line arguments (list of strings)
  86     """
  87     if options.full_help:
  88         pager = os.getenv('PAGER')
  89         if not pager:
  90             pager = 'more'
  91         fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
  92         command.Run(pager, fname)
  93         return 0
  94
  95     gitutil.Setup()
  96
  97     bsettings.Setup(options.config_file)
  98     options.git_dir = os.path.join(options.git, '.git')
  99
 100     toolchains = toolchain.Toolchains()
 101     toolchains.Scan(options.list_tool_chains)
 102     if options.list_tool_chains:
 103         toolchains.List()
 104         print
 105         return 0
 106
 107     # Work out how many commits to build. We want to build everything on the
 108     # branch. We also build the upstream commit as a control so we can see
 109     # problems introduced by the first commit on the branch.
 110     col = terminal.Color()
 111     count = options.count
 112     if count == -1:
 113         if not options.branch:
 114             count = 1
 115         else:
 116             count = gitutil.CountCommitsInBranch(options.git_dir,
 117                                                  options.branch)
 118             if count is None:
 119                 str = ("Branch '%s' not found or has no upstream" %
 120                        options.branch)
 121                 sys.exit(col.Color(col.RED, str))
 122             count += 1   # Build upstream commit also
 123
 124     if not count:
 125         str = ("No commits found to process in branch '%s': "
 126                "set branch's upstream or use -c flag" % options.branch)
 127         sys.exit(col.Color(col.RED, str))
 128
 129     # Work out what subset of the boards we are building
 130     board_file = os.path.join(options.git, 'boards.cfg')
 131     status = subprocess.call([os.path.join(options.git,
 132                                            'tools/genboardscfg.py')])
 133     if status != 0:
 134         sys.exit("Failed to generate boards.cfg")
 135
 136     boards = board.Boards()
 137     boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 138
 139     exclude = []
 140     if options.exclude:
 141         for arg in options.exclude:
 142             exclude += arg.split(',')
 143
 144     why_selected = boards.SelectBoards(args, exclude)
 145     selected = boards.GetSelected()
 146     if not len(selected):
 147         sys.exit(col.Color(col.RED, 'No matching boards found'))
 148
 149     # Read the metadata from the commits. First look at the upstream commit,
 150     # then the ones in the branch. We would like to do something like
 151     # upstream/master~..branch but that isn't possible if upstream/master is
 152     # a merge commit (it will list all the commits that form part of the
 153     # merge)
 154     if options.branch:
 155         if count == -1:
 156             range_expr = gitutil.GetRangeInBranch(options.git_dir,
 157                                                   options.branch)
 158             upstream_commit = gitutil.GetUpstream(options.git_dir,
 159                                                   options.branch)
 160             series = patchstream.GetMetaDataForList(upstream_commit,
 161                 options.git_dir, 1)
 162
 163             # Conflicting tags are not a problem for buildman, since it does
 164             # not use them. For example, Series-version is not useful for
 165             # buildman. On the other hand conflicting tags will cause an
 166             # error. So allow later tags to overwrite earlier ones.
 167             series.allow_overwrite = True
 168             series = patchstream.GetMetaDataForList(range_expr,
 169                                               options.git_dir, None, series)
 170         else:
 171             # Honour the count
 172             series = patchstream.GetMetaDataForList(options.branch,
 173                                                     options.git_dir, count)
 174     else:
 175         series = None
 176         options.verbose = True
 177         options.show_errors = True
 178
 179     # By default we have one thread per CPU. But if there are not enough jobs
 180     # we can have fewer threads and use a high '-j' value for make.
 181     if not options.threads:
 182         options.threads = min(multiprocessing.cpu_count(), len(selected))
 183     if not options.jobs:
 184         options.jobs = max(1, (multiprocessing.cpu_count() +
 185                 len(selected) - 1) / len(selected))
 186
 187     if not options.step:
 188         options.step = len(series.commits) - 1
 189
 190     gnu_make = command.Output(os.path.join(options.git,
 191                                            'scripts/show-gnu-make')).rstrip()
 192     if not gnu_make:
 193         sys.exit('GNU Make not found')
 194
 195     # Create a new builder with the selected options
 196     if options.branch:
 197         dirname = options.branch
 198     else:
 199         dirname = 'current'
 200     output_dir = os.path.join(options.output_dir, dirname)
 201     builder = Builder(toolchains, output_dir, options.git_dir,
 202             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 203             show_unknown=options.show_unknown, step=options.step)
 204     builder.force_config_on_failure = not options.quick
 205
 206     # For a dry run, just show our actions as a sanity check
 207     if options.dry_run:
 208         ShowActions(series, why_selected, selected, builder, options)
 209     else:
 210         builder.force_build = options.force_build
 211         builder.force_build_failures = options.force_build_failures
 212         builder.force_reconfig = options.force_reconfig
 213         builder.in_tree = options.in_tree
 214
 215         # Work out which boards to build
 216         board_selected = boards.GetSelectedDict()
 217
 218         if series:
 219             commits = series.commits
 220         else:
 221             commits = None
 222
 223         print GetActionSummary(options.summary, commits, board_selected,
 224                                options)
 225
 226         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 227                                   options.show_detail, options.show_bloat,
 228                                   options.list_error_boards)
 229         if options.summary:
 230             # We can't show function sizes without board details at present
 231             if options.show_bloat:
 232                 options.show_detail = True
 233             builder.ShowSummary(commits, board_selected)
 234         else:
 235             fail, warned = builder.BuildBoards(commits, board_selected,
 236                                 options.keep_outputs, options.verbose)
 237             if fail:
 238                 return 128
 239             elif warned:
 240                 return 129
 241     return 0