buildman: Move full help code into the control module
[karo-tx-uboot.git] / tools / buildman / control.py
1 # Copyright (c) 2013 The Chromium OS Authors.
2 #
3 # SPDX-License-Identifier:      GPL-2.0+
4 #
5
6 import multiprocessing
7 import os
8 import sys
9
10 import board
11 import bsettings
12 from builder import Builder
13 import gitutil
14 import patchstream
15 import terminal
16 import toolchain
17 import command
18 import subprocess
19
20 def GetPlural(count):
21     """Returns a plural 's' if count is not 1"""
22     return 's' if count != 1 else ''
23
24 def GetActionSummary(is_summary, commits, selected, options):
25     """Return a string summarising the intended action.
26
27     Returns:
28         Summary string.
29     """
30     if commits:
31         count = len(commits)
32         count = (count + options.step - 1) / options.step
33         commit_str = '%d commit%s' % (count, GetPlural(count))
34     else:
35         commit_str = 'current source'
36     str = '%s %s for %d boards' % (
37         'Summary of' if is_summary else 'Building', commit_str,
38         len(selected))
39     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
40             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
41     return str
42
43 def ShowActions(series, why_selected, boards_selected, builder, options):
44     """Display a list of actions that we would take, if not a dry run.
45
46     Args:
47         series: Series object
48         why_selected: Dictionary where each key is a buildman argument
49                 provided by the user, and the value is the boards brought
50                 in by that argument. For example, 'arm' might bring in
51                 400 boards, so in this case the key would be 'arm' and
52                 the value would be a list of board names.
53         boards_selected: Dict of selected boards, key is target name,
54                 value is Board object
55         builder: The builder that will be used to build the commits
56         options: Command line options object
57     """
58     col = terminal.Color()
59     print 'Dry run, so not doing much. But I would do this:'
60     print
61     if series:
62         commits = series.commits
63     else:
64         commits = None
65     print GetActionSummary(False, commits, boards_selected,
66             options)
67     print 'Build directory: %s' % builder.base_dir
68     if commits:
69         for upto in range(0, len(series.commits), options.step):
70             commit = series.commits[upto]
71             print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
72             print commit.subject
73     print
74     for arg in why_selected:
75         if arg != 'all':
76             print arg, ': %d boards' % why_selected[arg]
77     print ('Total boards to build for each commit: %d\n' %
78             why_selected['all'])
79
80 def DoBuildman(options, args):
81     """The main control code for buildman
82
83     Args:
84         options: Command line options object
85         args: Command line arguments (list of strings)
86     """
87     if options.full_help:
88         pager = os.getenv('PAGER')
89         if not pager:
90             pager = 'more'
91         fname = os.path.join(os.path.dirname(sys.argv[0]), 'README')
92         command.Run(pager, fname)
93         return 0
94
95     gitutil.Setup()
96
97     bsettings.Setup(options.config_file)
98     options.git_dir = os.path.join(options.git, '.git')
99
100     toolchains = toolchain.Toolchains()
101     toolchains.Scan(options.list_tool_chains)
102     if options.list_tool_chains:
103         toolchains.List()
104         print
105         return 0
106
107     # Work out how many commits to build. We want to build everything on the
108     # branch. We also build the upstream commit as a control so we can see
109     # problems introduced by the first commit on the branch.
110     col = terminal.Color()
111     count = options.count
112     if count == -1:
113         if not options.branch:
114             count = 1
115         else:
116             count = gitutil.CountCommitsInBranch(options.git_dir,
117                                                  options.branch)
118             if count is None:
119                 str = ("Branch '%s' not found or has no upstream" %
120                        options.branch)
121                 sys.exit(col.Color(col.RED, str))
122             count += 1   # Build upstream commit also
123
124     if not count:
125         str = ("No commits found to process in branch '%s': "
126                "set branch's upstream or use -c flag" % options.branch)
127         sys.exit(col.Color(col.RED, str))
128
129     # Work out what subset of the boards we are building
130     board_file = os.path.join(options.git, 'boards.cfg')
131     status = subprocess.call([os.path.join(options.git,
132                                            'tools/genboardscfg.py')])
133     if status != 0:
134         sys.exit("Failed to generate boards.cfg")
135
136     boards = board.Boards()
137     boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
138
139     exclude = []
140     if options.exclude:
141         for arg in options.exclude:
142             exclude += arg.split(',')
143
144     why_selected = boards.SelectBoards(args, exclude)
145     selected = boards.GetSelected()
146     if not len(selected):
147         sys.exit(col.Color(col.RED, 'No matching boards found'))
148
149     # Read the metadata from the commits. First look at the upstream commit,
150     # then the ones in the branch. We would like to do something like
151     # upstream/master~..branch but that isn't possible if upstream/master is
152     # a merge commit (it will list all the commits that form part of the
153     # merge)
154     if options.branch:
155         if count == -1:
156             range_expr = gitutil.GetRangeInBranch(options.git_dir,
157                                                   options.branch)
158             upstream_commit = gitutil.GetUpstream(options.git_dir,
159                                                   options.branch)
160             series = patchstream.GetMetaDataForList(upstream_commit,
161                 options.git_dir, 1)
162
163             # Conflicting tags are not a problem for buildman, since it does
164             # not use them. For example, Series-version is not useful for
165             # buildman. On the other hand conflicting tags will cause an
166             # error. So allow later tags to overwrite earlier ones.
167             series.allow_overwrite = True
168             series = patchstream.GetMetaDataForList(range_expr,
169                                               options.git_dir, None, series)
170         else:
171             # Honour the count
172             series = patchstream.GetMetaDataForList(options.branch,
173                                                     options.git_dir, count)
174     else:
175         series = None
176         options.verbose = True
177         options.show_errors = True
178
179     # By default we have one thread per CPU. But if there are not enough jobs
180     # we can have fewer threads and use a high '-j' value for make.
181     if not options.threads:
182         options.threads = min(multiprocessing.cpu_count(), len(selected))
183     if not options.jobs:
184         options.jobs = max(1, (multiprocessing.cpu_count() +
185                 len(selected) - 1) / len(selected))
186
187     if not options.step:
188         options.step = len(series.commits) - 1
189
190     gnu_make = command.Output(os.path.join(options.git,
191                                            'scripts/show-gnu-make')).rstrip()
192     if not gnu_make:
193         sys.exit('GNU Make not found')
194
195     # Create a new builder with the selected options
196     if options.branch:
197         dirname = options.branch
198     else:
199         dirname = 'current'
200     output_dir = os.path.join(options.output_dir, dirname)
201     builder = Builder(toolchains, output_dir, options.git_dir,
202             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
203             show_unknown=options.show_unknown, step=options.step)
204     builder.force_config_on_failure = not options.quick
205
206     # For a dry run, just show our actions as a sanity check
207     if options.dry_run:
208         ShowActions(series, why_selected, selected, builder, options)
209     else:
210         builder.force_build = options.force_build
211         builder.force_build_failures = options.force_build_failures
212         builder.force_reconfig = options.force_reconfig
213         builder.in_tree = options.in_tree
214
215         # Work out which boards to build
216         board_selected = boards.GetSelectedDict()
217
218         if series:
219             commits = series.commits
220         else:
221             commits = None
222
223         print GetActionSummary(options.summary, commits, board_selected,
224                                options)
225
226         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
227                                   options.show_detail, options.show_bloat,
228                                   options.list_error_boards)
229         if options.summary:
230             # We can't show function sizes without board details at present
231             if options.show_bloat:
232                 options.show_detail = True
233             builder.ShowSummary(commits, board_selected)
234         else:
235             fail, warned = builder.BuildBoards(commits, board_selected,
236                                 options.keep_outputs, options.verbose)
237             if fail:
238                 return 128
239             elif warned:
240                 return 129
241     return 0