tools/buildman/func_test.py

   1 #
   2 # Copyright (c) 2014 Google, Inc
   3 #
   4 # SPDX-License-Identifier:      GPL-2.0+
   5 #
   6
   7 import os
   8 import shutil
   9 import sys
  10 import tempfile
  11 import unittest
  12
  13 import board
  14 import bsettings
  15 import cmdline
  16 import command
  17 import control
  18 import gitutil
  19 import terminal
  20 import toolchain
  21
  22 settings_data = '''
  23 # Buildman settings file
  24
  25 [toolchain]
  26
  27 [toolchain-alias]
  28
  29 [make-flags]
  30 src=/home/sjg/c/src
  31 chroot=/home/sjg/c/chroot
  32 vboot=USE_STDINT=1 VBOOT_DEBUG=1 MAKEFLAGS_VBOOT=DEBUG=1 CFLAGS_EXTRA_VBOOT=-DUNROLL_LOOPS VBOOT_SOURCE=${src}/platform/vboot_reference
  33 chromeos_coreboot=VBOOT=${chroot}/build/link/usr ${vboot}
  34 chromeos_daisy=VBOOT=${chroot}/build/daisy/usr ${vboot}
  35 chromeos_peach=VBOOT=${chroot}/build/peach_pit/usr ${vboot}
  36 '''
  37
  38 boards = [
  39     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 1', 'board0',  ''],
  40     ['Active', 'arm', 'armv7', '', 'Tester', 'ARM Board 2', 'board1', ''],
  41     ['Active', 'powerpc', 'powerpc', '', 'Tester', 'PowerPC board 1', 'board2', ''],
  42     ['Active', 'powerpc', 'mpc5xx', '', 'Tester', 'PowerPC board 2', 'board3', ''],
  43     ['Active', 'sandbox', 'sandbox', '', 'Tester', 'Sandbox board', 'board4', ''],
  44 ]
  45
  46 commit_shortlog = """4aca821 patman: Avoid changing the order of tags
  47 39403bb patman: Use --no-pager' to stop git from forking a pager
  48 db6e6f2 patman: Remove the -a option
  49 f2ccf03 patman: Correct unit tests to run correctly
  50 1d097f9 patman: Fix indentation in terminal.py
  51 d073747 patman: Support the 'reverse' option for 'git log
  52 """
  53
  54 commit_log = ["""commit 7f6b8315d18f683c5181d0c3694818c1b2a20dcd
  55 Author: Masahiro Yamada <yamada.m@jp.panasonic.com>
  56 Date:   Fri Aug 22 19:12:41 2014 +0900
  57
  58     buildman: refactor help message
  59
  60     "buildman [options]" is displayed by default.
  61
  62     Append the rest of help messages to parser.usage
  63     instead of replacing it.
  64
  65     Besides, "-b <branch>" is not mandatory since commit fea5858e.
  66     Drop it from the usage.
  67
  68     Signed-off-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
  69 """,
  70 """commit d0737479be6baf4db5e2cdbee123e96bc5ed0ba8
  71 Author: Simon Glass <sjg@chromium.org>
  72 Date:   Thu Aug 14 16:48:25 2014 -0600
  73
  74     patman: Support the 'reverse' option for 'git log'
  75
  76     This option is currently not supported, but needs to be, for buildman to
  77     operate as expected.
  78
  79     Series-changes: 7
  80     - Add new patch to fix the 'reverse' bug
  81
  82
  83     Change-Id: I79078f792e8b390b8a1272a8023537821d45feda
  84     Reported-by: York Sun <yorksun@freescale.com>
  85     Signed-off-by: Simon Glass <sjg@chromium.org>
  86
  87 """,
  88 """commit 1d097f9ab487c5019152fd47bda126839f3bf9fc
  89 Author: Simon Glass <sjg@chromium.org>
  90 Date:   Sat Aug 9 11:44:32 2014 -0600
  91
  92     patman: Fix indentation in terminal.py
  93
  94     This code came from a different project with 2-character indentation. Fix
  95     it for U-Boot.
  96
  97     Series-changes: 6
  98     - Add new patch to fix indentation in teminal.py
  99
 100     Change-Id: I5a74d2ebbb3cc12a665f5c725064009ac96e8a34
 101     Signed-off-by: Simon Glass <sjg@chromium.org>
 102
 103 """,
 104 """commit f2ccf03869d1e152c836515a3ceb83cdfe04a105
 105 Author: Simon Glass <sjg@chromium.org>
 106 Date:   Sat Aug 9 11:08:24 2014 -0600
 107
 108     patman: Correct unit tests to run correctly
 109
 110     It seems that doctest behaves differently now, and some of the unit tests
 111     do not run. Adjust the tests to work correctly.
 112
 113      ./tools/patman/patman --test
 114     <unittest.result.TestResult run=10 errors=0 failures=0>
 115
 116     Series-changes: 6
 117     - Add new patch to fix patman unit tests
 118
 119     Change-Id: I3d2ca588f4933e1f9d6b1665a00e4ae58269ff3b
 120
 121 """,
 122 """commit db6e6f2f9331c5a37647d6668768d4a40b8b0d1c
 123 Author: Simon Glass <sjg@chromium.org>
 124 Date:   Sat Aug 9 12:06:02 2014 -0600
 125
 126     patman: Remove the -a option
 127
 128     It seems that this is no longer needed, since checkpatch.pl will catch
 129     whitespace problems in patches. Also the option is not widely used, so
 130     it seems safe to just remove it.
 131
 132     Series-changes: 6
 133     - Add new patch to remove patman's -a option
 134
 135     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 136     Change-Id: I5821a1c75154e532c46513486ca40b808de7e2cc
 137
 138 """,
 139 """commit 39403bb4f838153028a6f21ca30bf100f3791133
 140 Author: Simon Glass <sjg@chromium.org>
 141 Date:   Thu Aug 14 21:50:52 2014 -0600
 142
 143     patman: Use --no-pager' to stop git from forking a pager
 144
 145 """,
 146 """commit 4aca821e27e97925c039e69fd37375b09c6f129c
 147 Author: Simon Glass <sjg@chromium.org>
 148 Date:   Fri Aug 22 15:57:39 2014 -0600
 149
 150     patman: Avoid changing the order of tags
 151
 152     patman collects tags that it sees in the commit and places them nicely
 153     sorted at the end of the patch. However, this is not really necessary and
 154     in fact is apparently not desirable.
 155
 156     Series-changes: 9
 157     - Add new patch to avoid changing the order of tags
 158
 159     Suggested-by: Masahiro Yamada <yamada.m@jp.panasonic.com>
 160     Change-Id: Ib1518588c1a189ad5c3198aae76f8654aed8d0db
 161 """]
 162
 163 TEST_BRANCH = '__testbranch'
 164
 165 class TestFunctional(unittest.TestCase):
 166     """Functional test for buildman.
 167
 168     This aims to test from just below the invocation of buildman (parsing
 169     of arguments) to 'make' and 'git' invocation. It is not a true
 170     emd-to-end test, as it mocks git, make and the tool chain. But this
 171     makes it easier to detect when the builder is doing the wrong thing,
 172     since in many cases this test code will fail. For example, only a
 173     very limited subset of 'git' arguments is supported - anything
 174     unexpected will fail.
 175     """
 176     def setUp(self):
 177         self._base_dir = tempfile.mkdtemp()
 178         self._git_dir = os.path.join(self._base_dir, 'src')
 179         self._buildman_pathname = sys.argv[0]
 180         self._buildman_dir = os.path.dirname(sys.argv[0])
 181         command.test_result = self._HandleCommand
 182         self.setupToolchains()
 183         self._toolchains.Add('arm-gcc', test=False)
 184         self._toolchains.Add('powerpc-gcc', test=False)
 185         bsettings.Setup(None)
 186         bsettings.AddFile(settings_data)
 187         self._boards = board.Boards()
 188         for brd in boards:
 189             self._boards.AddBoard(board.Board(*brd))
 190
 191         # Directories where the source been cloned
 192         self._clone_dirs = []
 193         self._commits = len(commit_shortlog.splitlines()) + 1
 194         self._total_builds = self._commits * len(boards)
 195
 196         # Number of calls to make
 197         self._make_calls = 0
 198
 199         # Map of [board, commit] to error messages
 200         self._error = {}
 201
 202         self._test_branch = TEST_BRANCH
 203
 204         # Avoid sending any output and clear all terminal output
 205         terminal.SetPrintTestMode()
 206         terminal.GetPrintTestLines()
 207
 208     def tearDown(self):
 209         shutil.rmtree(self._base_dir)
 210
 211     def setupToolchains(self):
 212         self._toolchains = toolchain.Toolchains()
 213         self._toolchains.Add('gcc', test=False)
 214
 215     def _RunBuildman(self, *args):
 216         return command.RunPipe([[self._buildman_pathname] + list(args)],
 217                 capture=True, capture_stderr=True)
 218
 219     def _RunControl(self, *args, **kwargs):
 220         sys.argv = [sys.argv[0]] + list(args)
 221         options, args = cmdline.ParseArgs()
 222         result = control.DoBuildman(options, args, toolchains=self._toolchains,
 223                 make_func=self._HandleMake, boards=self._boards,
 224                 clean_dir=kwargs.get('clean_dir', True))
 225         self._builder = control.builder
 226         return result
 227
 228     def testFullHelp(self):
 229         command.test_result = None
 230         result = self._RunBuildman('-H')
 231         help_file = os.path.join(self._buildman_dir, 'README')
 232         self.assertEqual(len(result.stdout), os.path.getsize(help_file))
 233         self.assertEqual(0, len(result.stderr))
 234         self.assertEqual(0, result.return_code)
 235
 236     def testHelp(self):
 237         command.test_result = None
 238         result = self._RunBuildman('-h')
 239         help_file = os.path.join(self._buildman_dir, 'README')
 240         self.assertTrue(len(result.stdout) > 1000)
 241         self.assertEqual(0, len(result.stderr))
 242         self.assertEqual(0, result.return_code)
 243
 244     def testGitSetup(self):
 245         """Test gitutils.Setup(), from outside the module itself"""
 246         command.test_result = command.CommandResult(return_code=1)
 247         gitutil.Setup()
 248         self.assertEqual(gitutil.use_no_decorate, False)
 249
 250         command.test_result = command.CommandResult(return_code=0)
 251         gitutil.Setup()
 252         self.assertEqual(gitutil.use_no_decorate, True)
 253
 254     def _HandleCommandGitLog(self, args):
 255         if '-n0' in args:
 256             return command.CommandResult(return_code=0)
 257         elif args[-1] == 'upstream/master..%s' % self._test_branch:
 258             return command.CommandResult(return_code=0, stdout=commit_shortlog)
 259         elif args[:3] == ['--no-color', '--no-decorate', '--reverse']:
 260             if args[-1] == self._test_branch:
 261                 count = int(args[3][2:])
 262                 return command.CommandResult(return_code=0,
 263                                             stdout=''.join(commit_log[:count]))
 264
 265         # Not handled, so abort
 266         print 'git log', args
 267         sys.exit(1)
 268
 269     def _HandleCommandGitConfig(self, args):
 270         config = args[0]
 271         if config == 'sendemail.aliasesfile':
 272             return command.CommandResult(return_code=0)
 273         elif config.startswith('branch.badbranch'):
 274             return command.CommandResult(return_code=1)
 275         elif config == 'branch.%s.remote' % self._test_branch:
 276             return command.CommandResult(return_code=0, stdout='upstream\n')
 277         elif config == 'branch.%s.merge' % self._test_branch:
 278             return command.CommandResult(return_code=0,
 279                                          stdout='refs/heads/master\n')
 280
 281         # Not handled, so abort
 282         print 'git config', args
 283         sys.exit(1)
 284
 285     def _HandleCommandGit(self, in_args):
 286         """Handle execution of a git command
 287
 288         This uses a hacked-up parser.
 289
 290         Args:
 291             in_args: Arguments after 'git' from the command line
 292         """
 293         git_args = []           # Top-level arguments to git itself
 294         sub_cmd = None          # Git sub-command selected
 295         args = []               # Arguments to the git sub-command
 296         for arg in in_args:
 297             if sub_cmd:
 298                 args.append(arg)
 299             elif arg[0] == '-':
 300                 git_args.append(arg)
 301             else:
 302                 if git_args and git_args[-1] in ['--git-dir', '--work-tree']:
 303                     git_args.append(arg)
 304                 else:
 305                     sub_cmd = arg
 306         if sub_cmd == 'config':
 307             return self._HandleCommandGitConfig(args)
 308         elif sub_cmd == 'log':
 309             return self._HandleCommandGitLog(args)
 310         elif sub_cmd == 'clone':
 311             return command.CommandResult(return_code=0)
 312         elif sub_cmd == 'checkout':
 313             return command.CommandResult(return_code=0)
 314
 315         # Not handled, so abort
 316         print 'git', git_args, sub_cmd, args
 317         sys.exit(1)
 318
 319     def _HandleCommandNm(self, args):
 320         return command.CommandResult(return_code=0)
 321
 322     def _HandleCommandObjdump(self, args):
 323         return command.CommandResult(return_code=0)
 324
 325     def _HandleCommandSize(self, args):
 326         return command.CommandResult(return_code=0)
 327
 328     def _HandleCommand(self, **kwargs):
 329         """Handle a command execution.
 330
 331         The command is in kwargs['pipe-list'], as a list of pipes, each a
 332         list of commands. The command should be emulated as required for
 333         testing purposes.
 334
 335         Returns:
 336             A CommandResult object
 337         """
 338         pipe_list = kwargs['pipe_list']
 339         wc = False
 340         if len(pipe_list) != 1:
 341             if pipe_list[1] == ['wc', '-l']:
 342                 wc = True
 343             else:
 344                 print 'invalid pipe', kwargs
 345                 sys.exit(1)
 346         cmd = pipe_list[0][0]
 347         args = pipe_list[0][1:]
 348         result = None
 349         if cmd == 'git':
 350             result = self._HandleCommandGit(args)
 351         elif cmd == './scripts/show-gnu-make':
 352             return command.CommandResult(return_code=0, stdout='make')
 353         elif cmd.endswith('nm'):
 354             return self._HandleCommandNm(args)
 355         elif cmd.endswith('objdump'):
 356             return self._HandleCommandObjdump(args)
 357         elif cmd.endswith( 'size'):
 358             return self._HandleCommandSize(args)
 359
 360         if not result:
 361             # Not handled, so abort
 362             print 'unknown command', kwargs
 363             sys.exit(1)
 364
 365         if wc:
 366             result.stdout = len(result.stdout.splitlines())
 367         return result
 368
 369     def _HandleMake(self, commit, brd, stage, cwd, *args, **kwargs):
 370         """Handle execution of 'make'
 371
 372         Args:
 373             commit: Commit object that is being built
 374             brd: Board object that is being built
 375             stage: Stage that we are at (mrproper, config, build)
 376             cwd: Directory where make should be run
 377             args: Arguments to pass to make
 378             kwargs: Arguments to pass to command.RunPipe()
 379         """
 380         self._make_calls += 1
 381         if stage == 'mrproper':
 382             return command.CommandResult(return_code=0)
 383         elif stage == 'config':
 384             return command.CommandResult(return_code=0,
 385                     combined='Test configuration complete')
 386         elif stage == 'build':
 387             stderr = ''
 388             if type(commit) is not str:
 389                 stderr = self._error.get((brd.target, commit.sequence))
 390             if stderr:
 391                 return command.CommandResult(return_code=1, stderr=stderr)
 392             return command.CommandResult(return_code=0)
 393
 394         # Not handled, so abort
 395         print 'make', stage
 396         sys.exit(1)
 397
 398     # Example function to print output lines
 399     def print_lines(self, lines):
 400         print len(lines)
 401         for line in lines:
 402             print line
 403         #self.print_lines(terminal.GetPrintTestLines())
 404
 405     def testNoBoards(self):
 406         """Test that buildman aborts when there are no boards"""
 407         self._boards = board.Boards()
 408         with self.assertRaises(SystemExit):
 409             self._RunControl()
 410
 411     def testCurrentSource(self):
 412         """Very simple test to invoke buildman on the current source"""
 413         self.setupToolchains();
 414         self._RunControl()
 415         lines = terminal.GetPrintTestLines()
 416         self.assertIn('Building current source for %d boards' % len(boards),
 417                       lines[0].text)
 418
 419     def testBadBranch(self):
 420         """Test that we can detect an invalid branch"""
 421         with self.assertRaises(ValueError):
 422             self._RunControl('-b', 'badbranch')
 423
 424     def testBadToolchain(self):
 425         """Test that missing toolchains are detected"""
 426         self.setupToolchains();
 427         ret_code = self._RunControl('-b', TEST_BRANCH)
 428         lines = terminal.GetPrintTestLines()
 429
 430         # Buildman always builds the upstream commit as well
 431         self.assertIn('Building %d commits for %d boards' %
 432                 (self._commits, len(boards)), lines[0].text)
 433         self.assertEqual(self._builder.count, self._total_builds)
 434
 435         # Only sandbox should succeed, the others don't have toolchains
 436         self.assertEqual(self._builder.fail,
 437                          self._total_builds - self._commits)
 438         self.assertEqual(ret_code, 128)
 439
 440         for commit in range(self._commits):
 441             for board in self._boards.GetList():
 442                 if board.arch != 'sandbox':
 443                   errfile = self._builder.GetErrFile(commit, board.target)
 444                   fd = open(errfile)
 445                   self.assertEqual(fd.readlines(),
 446                           ['No tool chain for %s\n' % board.arch])
 447                   fd.close()
 448
 449     def testBranch(self):
 450         """Test building a branch with all toolchains present"""
 451         self._RunControl('-b', TEST_BRANCH)
 452         self.assertEqual(self._builder.count, self._total_builds)
 453         self.assertEqual(self._builder.fail, 0)
 454
 455     def testCount(self):
 456         """Test building a specific number of commitst"""
 457         self._RunControl('-b', TEST_BRANCH, '-c2')
 458         self.assertEqual(self._builder.count, 2 * len(boards))
 459         self.assertEqual(self._builder.fail, 0)
 460         # Each board has a mrproper, config, and then one make per commit
 461         self.assertEqual(self._make_calls, len(boards) * (2 + 2))
 462
 463     def testIncremental(self):
 464         """Test building a branch twice - the second time should do nothing"""
 465         self._RunControl('-b', TEST_BRANCH)
 466
 467         # Each board has a mrproper, config, and then one make per commit
 468         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 469         self._make_calls = 0
 470         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 471         self.assertEqual(self._make_calls, 0)
 472         self.assertEqual(self._builder.count, self._total_builds)
 473         self.assertEqual(self._builder.fail, 0)
 474
 475     def testForceBuild(self):
 476         """The -f flag should force a rebuild"""
 477         self._RunControl('-b', TEST_BRANCH)
 478         self._make_calls = 0
 479         self._RunControl('-b', TEST_BRANCH, '-f', clean_dir=False)
 480         # Each board has a mrproper, config, and then one make per commit
 481         self.assertEqual(self._make_calls, len(boards) * (self._commits + 2))
 482
 483     def testForceReconfigure(self):
 484         """The -f flag should force a rebuild"""
 485         self._RunControl('-b', TEST_BRANCH, '-C')
 486         # Each commit has a mrproper, config and make
 487         self.assertEqual(self._make_calls, len(boards) * self._commits * 3)
 488
 489     def testErrors(self):
 490         """Test handling of build errors"""
 491         self._error['board2', 1] = 'fred\n'
 492         self._RunControl('-b', TEST_BRANCH)
 493         self.assertEqual(self._builder.count, self._total_builds)
 494         self.assertEqual(self._builder.fail, 1)
 495
 496         # Remove the error. This should have no effect since the commit will
 497         # not be rebuilt
 498         del self._error['board2', 1]
 499         self._make_calls = 0
 500         self._RunControl('-b', TEST_BRANCH, clean_dir=False)
 501         self.assertEqual(self._builder.count, self._total_builds)
 502         self.assertEqual(self._make_calls, 0)
 503         self.assertEqual(self._builder.fail, 1)
 504
 505         # Now use the -F flag to force rebuild of the bad commit
 506         self._RunControl('-b', TEST_BRANCH, '-F', clean_dir=False)
 507         self.assertEqual(self._builder.count, self._total_builds)
 508         self.assertEqual(self._builder.fail, 0)
 509         self.assertEqual(self._make_calls, 3)
 510
 511     def testBranchWithSlash(self):
 512         """Test building a branch with a '/' in the name"""
 513         self._test_branch = '/__dev/__testbranch'
 514         self._RunControl('-b', self._test_branch, clean_dir=False)
 515         self.assertEqual(self._builder.count, self._total_builds)
 516         self.assertEqual(self._builder.fail, 0)