cb01158b60225182ba9cbbf62c60194eb8f8323d
[karo-tx-uboot.git] / tools / buildman / control.py
1 # Copyright (c) 2013 The Chromium OS Authors.
2 #
3 # SPDX-License-Identifier:      GPL-2.0+
4 #
5
6 import multiprocessing
7 import os
8 import sys
9
10 import board
11 import bsettings
12 from builder import Builder
13 import gitutil
14 import patchstream
15 import terminal
16 import toolchain
17 import command
18 import subprocess
19
20 def GetPlural(count):
21     """Returns a plural 's' if count is not 1"""
22     return 's' if count != 1 else ''
23
24 def GetActionSummary(is_summary, commits, selected, options):
25     """Return a string summarising the intended action.
26
27     Returns:
28         Summary string.
29     """
30     if commits:
31         count = len(commits)
32         count = (count + options.step - 1) / options.step
33         commit_str = '%d commit%s' % (count, GetPlural(count))
34     else:
35         commit_str = 'current source'
36     str = '%s %s for %d boards' % (
37         'Summary of' if is_summary else 'Building', commit_str,
38         len(selected))
39     str += ' (%d thread%s, %d job%s per thread)' % (options.threads,
40             GetPlural(options.threads), options.jobs, GetPlural(options.jobs))
41     return str
42
43 def ShowActions(series, why_selected, boards_selected, builder, options):
44     """Display a list of actions that we would take, if not a dry run.
45
46     Args:
47         series: Series object
48         why_selected: Dictionary where each key is a buildman argument
49                 provided by the user, and the value is the boards brought
50                 in by that argument. For example, 'arm' might bring in
51                 400 boards, so in this case the key would be 'arm' and
52                 the value would be a list of board names.
53         boards_selected: Dict of selected boards, key is target name,
54                 value is Board object
55         builder: The builder that will be used to build the commits
56         options: Command line options object
57     """
58     col = terminal.Color()
59     print 'Dry run, so not doing much. But I would do this:'
60     print
61     if series:
62         commits = series.commits
63     else:
64         commits = None
65     print GetActionSummary(False, commits, boards_selected,
66             options)
67     print 'Build directory: %s' % builder.base_dir
68     if commits:
69         for upto in range(0, len(series.commits), options.step):
70             commit = series.commits[upto]
71             print '   ', col.Color(col.YELLOW, commit.hash, bright=False),
72             print commit.subject
73     print
74     for arg in why_selected:
75         if arg != 'all':
76             print arg, ': %d boards' % why_selected[arg]
77     print ('Total boards to build for each commit: %d\n' %
78             why_selected['all'])
79
80 def DoBuildman(options, args):
81     """The main control code for buildman
82
83     Args:
84         options: Command line options object
85         args: Command line arguments (list of strings)
86     """
87     gitutil.Setup()
88
89     bsettings.Setup(options.config_file)
90     options.git_dir = os.path.join(options.git, '.git')
91
92     toolchains = toolchain.Toolchains()
93     toolchains.Scan(options.list_tool_chains)
94     if options.list_tool_chains:
95         toolchains.List()
96         print
97         return 0
98
99     # Work out how many commits to build. We want to build everything on the
100     # branch. We also build the upstream commit as a control so we can see
101     # problems introduced by the first commit on the branch.
102     col = terminal.Color()
103     count = options.count
104     if count == -1:
105         if not options.branch:
106             count = 1
107         else:
108             count = gitutil.CountCommitsInBranch(options.git_dir,
109                                                  options.branch)
110             if count is None:
111                 str = ("Branch '%s' not found or has no upstream" %
112                        options.branch)
113                 sys.exit(col.Color(col.RED, str))
114             count += 1   # Build upstream commit also
115
116     if not count:
117         str = ("No commits found to process in branch '%s': "
118                "set branch's upstream or use -c flag" % options.branch)
119         sys.exit(col.Color(col.RED, str))
120
121     # Work out what subset of the boards we are building
122     board_file = os.path.join(options.git, 'boards.cfg')
123     status = subprocess.call([os.path.join(options.git,
124                                            'tools/genboardscfg.py')])
125     if status != 0:
126         sys.exit("Failed to generate boards.cfg")
127
128     boards = board.Boards()
129     boards.ReadBoards(os.path.join(options.git, 'boards.cfg'))
130
131     exclude = []
132     if options.exclude:
133         for arg in options.exclude:
134             exclude += arg.split(',')
135
136     why_selected = boards.SelectBoards(args, exclude)
137     selected = boards.GetSelected()
138     if not len(selected):
139         sys.exit(col.Color(col.RED, 'No matching boards found'))
140
141     # Read the metadata from the commits. First look at the upstream commit,
142     # then the ones in the branch. We would like to do something like
143     # upstream/master~..branch but that isn't possible if upstream/master is
144     # a merge commit (it will list all the commits that form part of the
145     # merge)
146     if options.branch:
147         if count == -1:
148             range_expr = gitutil.GetRangeInBranch(options.git_dir,
149                                                   options.branch)
150             upstream_commit = gitutil.GetUpstream(options.git_dir,
151                                                   options.branch)
152             series = patchstream.GetMetaDataForList(upstream_commit,
153                 options.git_dir, 1)
154
155             # Conflicting tags are not a problem for buildman, since it does
156             # not use them. For example, Series-version is not useful for
157             # buildman. On the other hand conflicting tags will cause an
158             # error. So allow later tags to overwrite earlier ones.
159             series.allow_overwrite = True
160             series = patchstream.GetMetaDataForList(range_expr,
161                                               options.git_dir, None, series)
162         else:
163             # Honour the count
164             series = patchstream.GetMetaDataForList(options.branch,
165                                                     options.git_dir, count)
166     else:
167         series = None
168         options.verbose = True
169         options.show_errors = True
170
171     # By default we have one thread per CPU. But if there are not enough jobs
172     # we can have fewer threads and use a high '-j' value for make.
173     if not options.threads:
174         options.threads = min(multiprocessing.cpu_count(), len(selected))
175     if not options.jobs:
176         options.jobs = max(1, (multiprocessing.cpu_count() +
177                 len(selected) - 1) / len(selected))
178
179     if not options.step:
180         options.step = len(series.commits) - 1
181
182     gnu_make = command.Output(os.path.join(options.git,
183                                            'scripts/show-gnu-make')).rstrip()
184     if not gnu_make:
185         sys.exit('GNU Make not found')
186
187     # Create a new builder with the selected options
188     if options.branch:
189         dirname = options.branch
190     else:
191         dirname = 'current'
192     output_dir = os.path.join(options.output_dir, dirname)
193     builder = Builder(toolchains, output_dir, options.git_dir,
194             options.threads, options.jobs, gnu_make=gnu_make, checkout=True,
195             show_unknown=options.show_unknown, step=options.step)
196     builder.force_config_on_failure = not options.quick
197
198     # For a dry run, just show our actions as a sanity check
199     if options.dry_run:
200         ShowActions(series, why_selected, selected, builder, options)
201     else:
202         builder.force_build = options.force_build
203         builder.force_build_failures = options.force_build_failures
204         builder.force_reconfig = options.force_reconfig
205         builder.in_tree = options.in_tree
206
207         # Work out which boards to build
208         board_selected = boards.GetSelectedDict()
209
210         if series:
211             commits = series.commits
212         else:
213             commits = None
214
215         print GetActionSummary(options.summary, commits, board_selected,
216                                options)
217
218         builder.SetDisplayOptions(options.show_errors, options.show_sizes,
219                                   options.show_detail, options.show_bloat)
220         if options.summary:
221             # We can't show function sizes without board details at present
222             if options.show_bloat:
223                 options.show_detail = True
224             builder.ShowSummary(commits, board_selected)
225         else:
226             fail, warned = builder.BuildBoards(commits, board_selected,
227                                 options.keep_outputs, options.verbose)
228             if fail:
229                 return 128
230             elif warned:
231                 return 129
232     return 0