tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 import toolchain
  17 import command
  18 import subprocess
  19
  20 def GetPlural(count):
  21     """Returns a plural 's' if count is not 1"""
  22     return 's' if count != 1 else ''
  23
  24 def GetActionSummary(is_summary, commits, selected, options):
  25     """Return a string summarising the intended action.
  26
  27     Returns:
  28         Summary string.
  29     """
  30     if commits:
  31         count = len(commits)
  32         count = (count + options.step - 1) / options.step
  33         commit_str = '%d commit%s' % (count, GetPlural(count))
  34     else:
  35         commit_str = 'current source'
  36     str = '%s %s for %d boards' % (
  37         'Summary of' if is_summary else 'Building', commit_str,
  38         len(selected))
  39     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  40             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  41     return str
  42
  43 def ShowActions(series, why_selected, boards_selected, builder, options):
  44     """Display a list of actions that we would take, if not a dry run.
  45
  46     Args:
  47         series: Series object
  48         why_selected: Dictionary where each key is a buildman argument
  49                 provided by the user, and the value is the boards brought
  50                 in by that argument. For example, 'arm' might bring in
  51                 400 boards, so in this case the key would be 'arm' and
  52                 the value would be a list of board names.
  53         boards_selected: Dict of selected boards, key is target name,
  54                 value is Board object
  55         builder: The builder that will be used to build the commits
  56         options: Command line options object
  57     """
  58     col = terminal.Color()
  59     print 'Dry run, so not doing much. But I would do this:'
  60     print
  61     if series:
  62         commits = series.commits
  63     else:
  64         commits = None
  65     print GetActionSummary(False, commits, boards_selected,
  66             options)
  67     print 'Build directory: %s' % builder.base_dir
  68     if commits:
  69         for upto in range(0, len(series.commits), options.step):
  70             commit = series.commits[upto]
  71             print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
  72             print commit.subject
  73     print
  74     for arg in why_selected:
  75         if arg != 'all':
  76             print arg, ': %d boards' % why_selected[arg]
  77     print ('Total boards to build for each commit: %d\n' %
  78             why_selected['all'])
  79
  80 def DoBuildman(options, args):
  81     """The main control code for buildman
  82
  83     Args:
  84         options: Command line options object
  85         args: Command line arguments (list of strings)
  86     """
  87     gitutil.Setup()
  88
  89     bsettings.Setup(options.config_file)
  90     options.git_dir = os.path.join(options.git, '.git')
  91
  92     toolchains = toolchain.Toolchains()
  93     toolchains.Scan(options.list_tool_chains)
  94     if options.list_tool_chains:
  95         toolchains.List()
  96         print
  97         return
  98
  99     # Work out how many commits to build. We want to build everything on the
 100     # branch. We also build the upstream commit as a control so we can see
 101     # problems introduced by the first commit on the branch.
 102     col = terminal.Color()
 103     count = options.count
 104     if count == -1:
 105         if not options.branch:
 106             count = 1
 107         else:
 108             count = gitutil.CountCommitsInBranch(options.git_dir,
 109                                                  options.branch)
 110             if count is None:
 111                 str = ("Branch '%s' not found or has no upstream" %
 112                        options.branch)
 113                 sys.exit(col.Color(col.RED, str))
 114             count += 1   # Build upstream commit also
 115
 116     if not count:
 117         str = ("No commits found to process in branch '%s': "
 118                "set branch's upstream or use -c flag" % options.branch)
 119         sys.exit(col.Color(col.RED, str))
 120
 121     # Work out what subset of the boards we are building
 122     board_file = os.path.join(options.git, 'boards.cfg')
 123     status = subprocess.call([os.path.join(options.git,
 124                                            'tools/genboardscfg.py')])
 125     if status != 0:
 126         sys.exit("Failed to generate boards.cfg")
 127
 128     boards = board.Boards()
 129     boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 130     why_selected = boards.SelectBoards(args)
 131     selected = boards.GetSelected()
 132     if not len(selected):
 133         sys.exit(col.Color(col.RED, 'No matching boards found'))
 134
 135     # Read the metadata from the commits. First look at the upstream commit,
 136     # then the ones in the branch. We would like to do something like
 137     # upstream/master~..branch but that isn't possible if upstream/master is
 138     # a merge commit (it will list all the commits that form part of the
 139     # merge)
 140     if options.branch:
 141         if count == -1:
 142             range_expr = gitutil.GetRangeInBranch(options.git_dir,
 143                                                   options.branch)
 144             upstream_commit = gitutil.GetUpstream(options.git_dir,
 145                                                   options.branch)
 146             series = patchstream.GetMetaDataForList(upstream_commit,
 147                 options.git_dir, 1)
 148
 149             # Conflicting tags are not a problem for buildman, since it does
 150             # not use them. For example, Series-version is not useful for
 151             # buildman. On the other hand conflicting tags will cause an
 152             # error. So allow later tags to overwrite earlier ones.
 153             series.allow_overwrite = True
 154             series = patchstream.GetMetaDataForList(range_expr,
 155                                               options.git_dir, None, series)
 156         else:
 157             # Honour the count
 158             series = patchstream.GetMetaDataForList(options.branch,
 159                                                     options.git_dir, count)
 160     else:
 161         series = None
 162         options.verbose = True
 163         options.show_errors = True
 164
 165     # By default we have one thread per CPU. But if there are not enough jobs
 166     # we can have fewer threads and use a high '-j' value for make.
 167     if not options.threads:
 168         options.threads = min(multiprocessing.cpu_count(), len(selected))
 169     if not options.jobs:
 170         options.jobs = max(1, (multiprocessing.cpu_count() +
 171                 len(selected) - 1) / len(selected))
 172
 173     if not options.step:
 174         options.step = len(series.commits) - 1
 175
 176     gnu_make = command.Output(os.path.join(options.git,
 177                                            'scripts/show-gnu-make')).rstrip()
 178     if not gnu_make:
 179         sys.exit('GNU Make not found')
 180
 181     # Create a new builder with the selected options
 182     if options.branch:
 183         dirname = options.branch
 184     else:
 185         dirname = 'current'
 186     output_dir = os.path.join(options.output_dir, dirname)
 187     builder = Builder(toolchains, output_dir, options.git_dir,
 188             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 189             show_unknown=options.show_unknown, step=options.step)
 190     builder.force_config_on_failure = not options.quick
 191
 192     # For a dry run, just show our actions as a sanity check
 193     if options.dry_run:
 194         ShowActions(series, why_selected, selected, builder, options)
 195     else:
 196         builder.force_build = options.force_build
 197         builder.force_build_failures = options.force_build_failures
 198         builder.force_reconfig = options.force_reconfig
 199         builder.in_tree = options.in_tree
 200
 201         # Work out which boards to build
 202         board_selected = boards.GetSelectedDict()
 203
 204         if series:
 205             commits = series.commits
 206         else:
 207             commits = None
 208
 209         print GetActionSummary(options.summary, commits, board_selected,
 210                                options)
 211
 212         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 213                                   options.show_detail, options.show_bloat)
 214         if options.summary:
 215             # We can't show function sizes without board details at present
 216             if options.show_bloat:
 217                 options.show_detail = True
 218             builder.ShowSummary(commits, board_selected)
 219         else:
 220             builder.BuildBoards(commits, board_selected,
 221                                 options.keep_outputs, options.verbose)