tools/buildman/control.py

   1 # Copyright (c) 2013 The Chromium OS Authors.
   2 #
   3 # SPDX-License-Identifier:      GPL-2.0+
   4 #
   5
   6 import multiprocessing
   7 import os
   8 import sys
   9
  10 import board
  11 import bsettings
  12 from builder import Builder
  13 import gitutil
  14 import patchstream
  15 import terminal
  16 import toolchain
  17 import command
  18 import subprocess
  19
  20 def GetPlural(count):
  21     """Returns a plural 's' if count is not 1"""
  22     return 's' if count != 1 else ''
  23
  24 def GetActionSummary(is_summary, commits, selected, options):
  25     """Return a string summarising the intended action.
  26
  27     Returns:
  28         Summary string.
  29     """
  30     if commits:
  31         count = len(commits)
  32         count = (count + options.step - 1) / options.step
  33         commit_str = '%d commit%s' % (count, GetPlural(count))
  34     else:
  35         commit_str = 'current source'
  36     str = '%s %s for %d boards' % (
  37         'Summary of' if is_summary else 'Building', commit_str,
  38         len(selected))
  39     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
  40             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
  41     return str
  42
  43 def ShowActions(series, why_selected, boards_selected, builder, options):
  44     """Display a list of actions that we would take, if not a dry run.
  45
  46     Args:
  47         series: Series object
  48         why_selected: Dictionary where each key is a buildman argument
  49                 provided by the user, and the value is the boards brought
  50                 in by that argument. For example, 'arm' might bring in
  51                 400 boards, so in this case the key would be 'arm' and
  52                 the value would be a list of board names.
  53         boards_selected: Dict of selected boards, key is target name,
  54                 value is Board object
  55         builder: The builder that will be used to build the commits
  56         options: Command line options object
  57     """
  58     col = terminal.Color()
  59     print 'Dry run, so not doing much. But I would do this:'
  60     print
  61     if series:
  62         commits = series.commits
  63     else:
  64         commits = None
  65     print GetActionSummary(False, commits, boards_selected,
  66             options)
  67     print 'Build directory: %s' % builder.base_dir
  68     if commits:
  69         for upto in range(0, len(series.commits), options.step):
  70             commit = series.commits[upto]
  71             print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
  72             print commit.subject
  73     print
  74     for arg in why_selected:
  75         if arg != 'all':
  76             print arg, ': %d boards' % why_selected[arg]
  77     print ('Total boards to build for each commit: %d\n' %
  78             why_selected['all'])
  79
  80 def DoBuildman(options, args):
  81     """The main control code for buildman
  82
  83     Args:
  84         options: Command line options object
  85         args: Command line arguments (list of strings)
  86     """
  87     gitutil.Setup()
  88
  89     bsettings.Setup(options.config_file)
  90     options.git_dir = os.path.join(options.git, '.git')
  91
  92     toolchains = toolchain.Toolchains()
  93     toolchains.Scan(options.list_tool_chains)
  94     if options.list_tool_chains:
  95         toolchains.List()
  96         print
  97         return
  98
  99     # Work out how many commits to build. We want to build everything on the
 100     # branch. We also build the upstream commit as a control so we can see
 101     # problems introduced by the first commit on the branch.
 102     col = terminal.Color()
 103     count = options.count
 104     if count == -1:
 105         if not options.branch:
 106             count = 1
 107         else:
 108             count = gitutil.CountCommitsInBranch(options.git_dir,
 109                                                  options.branch)
 110             if count is None:
 111                 str = ("Branch '%s' not found or has no upstream" %
 112                        options.branch)
 113                 sys.exit(col.Color(col.RED, str))
 114             count += 1   # Build upstream commit also
 115
 116     if not count:
 117         str = ("No commits found to process in branch '%s': "
 118                "set branch's upstream or use -c flag" % options.branch)
 119         sys.exit(col.Color(col.RED, str))
 120
 121     # Work out what subset of the boards we are building
 122     board_file = os.path.join(options.git, 'boards.cfg')
 123     if not os.path.exists(board_file):
 124         print 'Could not find %s' % board_file
 125         status = subprocess.call([os.path.join(options.git,
 126                                                'tools/genboardscfg.py')])
 127         if status != 0:
 128             sys.exit("Failed to generate boards.cfg")
 129
 130     boards = board.Boards()
 131     boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
 132     why_selected = boards.SelectBoards(args)
 133     selected = boards.GetSelected()
 134     if not len(selected):
 135         sys.exit(col.Color(col.RED, 'No matching boards found'))
 136
 137     # Read the metadata from the commits. First look at the upstream commit,
 138     # then the ones in the branch. We would like to do something like
 139     # upstream/master~..branch but that isn't possible if upstream/master is
 140     # a merge commit (it will list all the commits that form part of the
 141     # merge)
 142     if options.branch:
 143         if count == -1:
 144             range_expr = gitutil.GetRangeInBranch(options.git_dir,
 145                                                   options.branch)
 146             upstream_commit = gitutil.GetUpstream(options.git_dir,
 147                                                   options.branch)
 148             series = patchstream.GetMetaDataForList(upstream_commit,
 149                 options.git_dir, 1)
 150
 151             # Conflicting tags are not a problem for buildman, since it does
 152             # not use them. For example, Series-version is not useful for
 153             # buildman. On the other hand conflicting tags will cause an
 154             # error. So allow later tags to overwrite earlier ones.
 155             series.allow_overwrite = True
 156             series = patchstream.GetMetaDataForList(range_expr,
 157                                               options.git_dir, None, series)
 158         else:
 159             # Honour the count
 160             series = patchstream.GetMetaDataForList(options.branch,
 161                                                     options.git_dir, count)
 162     else:
 163         series = None
 164         options.verbose = True
 165         options.show_errors = True
 166
 167     # By default we have one thread per CPU. But if there are not enough jobs
 168     # we can have fewer threads and use a high '-j' value for make.
 169     if not options.threads:
 170         options.threads = min(multiprocessing.cpu_count(), len(selected))
 171     if not options.jobs:
 172         options.jobs = max(1, (multiprocessing.cpu_count() +
 173                 len(selected) - 1) / len(selected))
 174
 175     if not options.step:
 176         options.step = len(series.commits) - 1
 177
 178     gnu_make = command.Output(os.path.join(options.git,
 179                                            'scripts/show-gnu-make')).rstrip()
 180     if not gnu_make:
 181         sys.exit('GNU Make not found')
 182
 183     # Create a new builder with the selected options
 184     if options.branch:
 185         dirname = options.branch
 186     else:
 187         dirname = 'current'
 188     output_dir = os.path.join(options.output_dir, dirname)
 189     builder = Builder(toolchains, output_dir, options.git_dir,
 190             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
 191             show_unknown=options.show_unknown, step=options.step)
 192     builder.force_config_on_failure = not options.quick
 193
 194     # For a dry run, just show our actions as a sanity check
 195     if options.dry_run:
 196         ShowActions(series, why_selected, selected, builder, options)
 197     else:
 198         builder.force_build = options.force_build
 199         builder.force_build_failures = options.force_build_failures
 200         builder.force_reconfig = options.force_reconfig
 201         builder.in_tree = options.in_tree
 202
 203         # Work out which boards to build
 204         board_selected = boards.GetSelectedDict()
 205
 206         if series:
 207             commits = series.commits
 208         else:
 209             commits = None
 210
 211         print GetActionSummary(options.summary, commits, board_selected,
 212                                options)
 213
 214         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
 215                                   options.show_detail, options.show_bloat)
 216         if options.summary:
 217             # We can't show function sizes without board details at present
 218             if options.show_bloat:
 219                 options.show_detail = True
 220             builder.ShowSummary(commits, board_selected)
 221         else:
 222             builder.BuildBoards(commits, board_selected,
 223                                 options.keep_outputs, options.verbose)