3 # Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
5 # SPDX-License-Identifier: GPL-2.0+
9 Converter from Kconfig and MAINTAINERS to a board database.
11 Run 'tools/genboardscfg.py' to create a board database.
13 Run 'tools/genboardscfg.py -h' for available options.
15 Python 2.6 or later, but not Python 3.x is necessary to run this script.
21 import multiprocessing
29 sys.path.append(os.path.join(os.path.dirname(__file__), 'buildman'))
32 ### constant variables ###
33 OUTPUT_FILE = 'boards.cfg'
34 CONFIG_DIR = 'configs'
38 # Automatically generated by %s: don't edit
40 # Status, Arch, CPU, SoC, Vendor, Board, Target, Options, Maintainers
44 ### helper functions ###
46 """Remove a file ignoring 'No such file or directory' error."""
49 except OSError as exception:
50 # Ignore 'No such file or directory' error
51 if exception.errno != errno.ENOENT:
54 def check_top_directory():
55 """Exit if we are not at the top of source directory."""
56 for f in ('README', 'Licenses'):
57 if not os.path.exists(f):
58 sys.exit('Please run at the top of source directory.')
60 def output_is_new(output):
61 """Check if the output file is up to date.
64 True if the given output file exists and is newer than any of
65 *_defconfig, MAINTAINERS and Kconfig*. False otherwise.
68 ctime = os.path.getctime(output)
69 except OSError as exception:
70 if exception.errno == errno.ENOENT:
71 # return False on 'No such file or directory' error
76 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
77 for filename in fnmatch.filter(filenames, '*_defconfig'):
78 if fnmatch.fnmatch(filename, '.*'):
80 filepath = os.path.join(dirpath, filename)
81 if ctime < os.path.getctime(filepath):
84 for (dirpath, dirnames, filenames) in os.walk('.'):
85 for filename in filenames:
86 if (fnmatch.fnmatch(filename, '*~') or
87 not fnmatch.fnmatch(filename, 'Kconfig*') and
88 not filename == 'MAINTAINERS'):
90 filepath = os.path.join(dirpath, filename)
91 if ctime < os.path.getctime(filepath):
94 # Detect a board that has been removed since the current board database
96 with open(output) as f:
98 if line[0] == '#' or line == '\n':
100 defconfig = line.split()[6] + '_defconfig'
101 if not os.path.exists(os.path.join(CONFIG_DIR, defconfig)):
107 class KconfigScanner:
109 """Kconfig scanner."""
111 ### constant variable only used in this class ###
116 'vendor' : 'SYS_VENDOR',
117 'board' : 'SYS_BOARD',
118 'config' : 'SYS_CONFIG_NAME',
119 'options' : 'SYS_EXTRA_OPTIONS'
123 """Scan all the Kconfig files and create a Config object."""
124 # Define environment variables referenced from Kconfig
125 os.environ['srctree'] = os.getcwd()
126 os.environ['UBOOTVERSION'] = 'dummy'
127 os.environ['KCONFIG_OBJDIR'] = ''
128 self._conf = kconfiglib.Config()
131 """Delete a leftover temporary file before exit.
133 The scan() method of this class creates a temporay file and deletes
134 it on success. If scan() method throws an exception on the way,
135 the temporary file might be left over. In that case, it should be
136 deleted in this destructor.
138 if hasattr(self, '_tmpfile') and self._tmpfile:
139 try_remove(self._tmpfile)
141 def scan(self, defconfig):
142 """Load a defconfig file to obtain board parameters.
145 defconfig: path to the defconfig file to be processed
148 A dictionary of board parameters. It has a form of:
153 'vendor': <vendor_name>,
154 'board': <board_name>,
155 'target': <target_name>,
156 'config': <config_header_name>,
157 'options': <extra_options>
160 # strip special prefixes and save it in a temporary file
161 fd, self._tmpfile = tempfile.mkstemp()
162 with os.fdopen(fd, 'w') as f:
163 for line in open(defconfig):
164 colon = line.find(':CONFIG_')
168 f.write(line[colon + 1:])
170 self._conf.load_config(self._tmpfile)
172 try_remove(self._tmpfile)
177 # Get the value of CONFIG_SYS_ARCH, CONFIG_SYS_CPU, ... etc.
178 # Set '-' if the value is empty.
179 for key, symbol in self._SYMBOL_TABLE.items():
180 value = self._conf.get_symbol(symbol).get_value()
186 defconfig = os.path.basename(defconfig)
187 params['target'], match, rear = defconfig.partition('_defconfig')
188 assert match and not rear, '%s : invalid defconfig' % defconfig
191 if params['arch'] == 'arm' and params['cpu'] == 'armv8':
192 params['arch'] = 'aarch64'
194 # fix-up options field. It should have the form:
195 # <config name>[:comma separated config options]
196 if params['options'] != '-':
197 params['options'] = params['config'] + ':' + \
198 params['options'].replace(r'\"', '"')
199 elif params['config'] != params['target']:
200 params['options'] = params['config']
204 def scan_defconfigs_for_multiprocess(queue, defconfigs):
205 """Scan defconfig files and queue their board parameters
207 This function is intended to be passed to
208 multiprocessing.Process() constructor.
211 queue: An instance of multiprocessing.Queue().
212 The resulting board parameters are written into it.
213 defconfigs: A sequence of defconfig files to be scanned.
215 kconf_scanner = KconfigScanner()
216 for defconfig in defconfigs:
217 queue.put(kconf_scanner.scan(defconfig))
219 def read_queues(queues, params_list):
220 """Read the queues and append the data to the paramers list"""
223 params_list.append(q.get())
225 def scan_defconfigs(jobs=1):
226 """Collect board parameters for all defconfig files.
228 This function invokes multiple processes for faster processing.
231 jobs: The number of jobs to run simultaneously
234 for (dirpath, dirnames, filenames) in os.walk(CONFIG_DIR):
235 for filename in fnmatch.filter(filenames, '*_defconfig'):
236 if fnmatch.fnmatch(filename, '.*'):
238 all_defconfigs.append(os.path.join(dirpath, filename))
240 total_boards = len(all_defconfigs)
243 for i in range(jobs):
244 defconfigs = all_defconfigs[total_boards * i / jobs :
245 total_boards * (i + 1) / jobs]
246 q = multiprocessing.Queue(maxsize=-1)
247 p = multiprocessing.Process(target=scan_defconfigs_for_multiprocess,
248 args=(q, defconfigs))
253 # The resulting data should be accumulated to this list
256 # Data in the queues should be retrieved preriodically.
257 # Otherwise, the queues would become full and subprocesses would get stuck.
258 while any([p.is_alive() for p in processes]):
259 read_queues(queues, params_list)
260 # sleep for a while until the queues are filled
261 time.sleep(SLEEP_TIME)
263 # Joining subprocesses just in case
264 # (All subprocesses should already have been finished)
268 # retrieve leftover data
269 read_queues(queues, params_list)
273 class MaintainersDatabase:
275 """The database of board status and maintainers."""
278 """Create an empty database."""
281 def get_status(self, target):
282 """Return the status of the given board.
284 The board status is generally either 'Active' or 'Orphan'.
285 Display a warning message and return '-' if status information
289 'Active', 'Orphan' or '-'.
291 if not target in self.database:
292 print >> sys.stderr, "WARNING: no status info for '%s'" % target
295 tmp = self.database[target][0]
296 if tmp.startswith('Maintained'):
298 elif tmp.startswith('Orphan'):
301 print >> sys.stderr, ("WARNING: %s: unknown status for '%s'" %
305 def get_maintainers(self, target):
306 """Return the maintainers of the given board.
309 Maintainers of the board. If the board has two or more maintainers,
310 they are separated with colons.
312 if not target in self.database:
313 print >> sys.stderr, "WARNING: no maintainers for '%s'" % target
316 return ':'.join(self.database[target][1])
318 def parse_file(self, file):
319 """Parse a MAINTAINERS file.
321 Parse a MAINTAINERS file and accumulates board status and
322 maintainers information.
325 file: MAINTAINERS file to be parsed
330 for line in open(file):
331 tag, rest = line[:2], line[2:].strip()
333 maintainers.append(rest)
335 # expand wildcard and filter by 'configs/*_defconfig'
336 for f in glob.glob(rest):
337 front, match, rear = f.partition('configs/')
338 if not front and match:
339 front, match, rear = rear.rpartition('_defconfig')
340 if match and not rear:
341 targets.append(front)
345 for target in targets:
346 self.database[target] = (status, maintainers)
351 for target in targets:
352 self.database[target] = (status, maintainers)
354 def insert_maintainers_info(params_list):
355 """Add Status and Maintainers information to the board parameters list.
358 params_list: A list of the board parameters
360 database = MaintainersDatabase()
361 for (dirpath, dirnames, filenames) in os.walk('.'):
362 if 'MAINTAINERS' in filenames:
363 database.parse_file(os.path.join(dirpath, 'MAINTAINERS'))
365 for i, params in enumerate(params_list):
366 target = params['target']
367 params['status'] = database.get_status(target)
368 params['maintainers'] = database.get_maintainers(target)
369 params_list[i] = params
371 def format_and_output(params_list, output):
372 """Write board parameters into a file.
374 Columnate the board parameters, sort lines alphabetically,
375 and then write them to a file.
378 params_list: The list of board parameters
379 output: The path to the output file
381 FIELDS = ('status', 'arch', 'cpu', 'soc', 'vendor', 'board', 'target',
382 'options', 'maintainers')
384 # First, decide the width of each column
385 max_length = dict([ (f, 0) for f in FIELDS])
386 for params in params_list:
388 max_length[f] = max(max_length[f], len(params[f]))
391 for params in params_list:
394 # insert two spaces between fields like column -t would
395 line += ' ' + params[f].ljust(max_length[f])
396 output_lines.append(line.strip())
398 # ignore case when sorting
399 output_lines.sort(key=str.lower)
401 with open(output, 'w') as f:
402 f.write(COMMENT_BLOCK + '\n'.join(output_lines) + '\n')
404 def gen_boards_cfg(output, jobs=1, force=False):
405 """Generate a board database file.
408 output: The name of the output file
409 jobs: The number of jobs to run simultaneously
410 force: Force to generate the output even if it is new
412 check_top_directory()
414 if not force and output_is_new(output):
415 print "%s is up to date. Nothing to do." % output
418 params_list = scan_defconfigs(jobs)
419 insert_maintainers_info(params_list)
420 format_and_output(params_list, output)
424 cpu_count = multiprocessing.cpu_count()
425 except NotImplementedError:
428 parser = optparse.OptionParser()
430 parser.add_option('-f', '--force', action="store_true", default=False,
431 help='regenerate the output even if it is new')
432 parser.add_option('-j', '--jobs', type='int', default=cpu_count,
433 help='the number of jobs to run simultaneously')
434 parser.add_option('-o', '--output', default=OUTPUT_FILE,
435 help='output file [default=%s]' % OUTPUT_FILE)
436 (options, args) = parser.parse_args()
438 gen_boards_cfg(options.output, jobs=options.jobs, force=options.force)
440 if __name__ == '__main__':