Minki/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2024-11-23 12:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

linux-kselftest-kunit-fixes-5.10-rc1

Browse Source 
This Kunit fixes update consists of several kunit tool bug fixes in
 flag handling, run outside kernel tree, make errors, and generating
 results.
 -----BEGIN PGP SIGNATURE-----
 
 iQIzBAABCgAdFiEEPZKym/RZuOCGeA/kCwJExA0NQxwFAl+It8sACgkQCwJExA0N
 QxzwMhAAncxGyhPX0JoYJMpz6fFjITt0Bnrv21OcfO1VeIGExOoHfmpmsXVK1dbv
 VIlLEAmjAlX6UIpEpi/DN3DfUFi9E0oeCZd4EOnehti12fs4sFYXeJvYK2TrKyNV
 zCaUxHb6xxL4Toy4XuNRdD6016f39Vax4QcM/puNm5CNVMboWeyzZaqooASxLya8
 SjZSthmS+kAqvHcB6aqPx7GhBib/k6T+Br5w6paM79tYk9OSMhvhtIx/+twQFw4e
 gkT5ZcTdk+t6mjBUY+w1l28W9VlAQi5kjB+22M8loRv2spu3YyxlVqDuQI0MUy+N
 RntNsxIwJOSt6N0DvUbXDEZwn/SvX0KS40nI5SSJjkcS1orAE2FQwjY2WzHCZp7T
 stQ+Wp7nHuzlVTRUz1pd6ZFluUshptkMwEQfznMIqdxgbWZSQWPc9ypGp6dU9hti
 jM5jGUnzYqIe5qdccImRd8fijLKJ/BOvmOauijhJPavhBmCqwseERhb0XN4vpiH+
 YCkdT7HHVIZWOr6ECovKizeElTfZN3MBNZV30oDeOzsQSyp5W5vke9u+weKN7NF6
 Y2+DACvT87B8miIUPv2HNlAi74nBx6RLDyqLbRgarjNOfDsDfkqw5OHxFBr29TDb
 u7THjs63LouVGFgaON7yPMgnsYep4irYuaT7kM9QDW1Af0xyQSs=
 =drcb
 -----END PGP SIGNATURE-----

Merge tag 'linux-kselftest-kunit-fixes-5.10-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest

Pull Kunit updates from Shuah Khan:
 "Several kunit tool bug fixes in flag handling, run outside kernel
  tree, make errors, and generating results"

* tag 'linux-kselftest-kunit-fixes-5.10-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/shuah/linux-kselftest:
  kunit: tool: fix display of make errors
  kunit: tool: handle when .kunit exists but .kunitconfig does not
  kunit: tool: fix --alltests flag
  kunit: tool: allow generating test results in JSON
  kunit: tool: fix running kunit_tool from outside kernel tree
This commit is contained in:
Linus Torvalds 2020-10-15 15:17:06 -07:00
commit 578a7155c5
5 changed files with 154 additions and 28 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
tools/testing/kunit/configs/broken_on_uml.config
View File

@ -39,3 +39,4 @@
# CONFIG_QCOM_CPR is not set
# CONFIG_RESET_BRCMSTB_RESCAL is not set
# CONFIG_RESET_INTEL_GW is not set
# CONFIG_ADI_AXI_ADC is not set

56
tools/testing/kunit/kunit.py
View File

@ -17,6 +17,7 @@ from collections import namedtuple
from enum import Enum, auto
import kunit_config
import kunit_json
import kunit_kernel
import kunit_parser
@ -30,9 +31,9 @@ KunitBuildRequest = namedtuple('KunitBuildRequest',
KunitExecRequest = namedtuple('KunitExecRequest',
['timeout', 'build_dir', 'alltests'])
KunitParseRequest = namedtuple('KunitParseRequest',
['raw_output', 'input_data'])
['raw_output', 'input_data', 'build_dir', 'json'])
KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs',
'build_dir', 'alltests',
'build_dir', 'alltests', 'json',
'make_options'])
KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0]
@ -113,12 +114,22 @@ def parse_tests(request: KunitParseRequest) -> KunitResult:
test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS,
[],
'Tests not Parsed.')
if request.raw_output:
kunit_parser.raw_output(request.input_data)
else:
test_result = kunit_parser.parse_run_tests(request.input_data)
parse_end = time.time()
if request.json:
json_obj = kunit_json.get_json_result(
test_result=test_result,
def_config='kunit_defconfig',
build_dir=request.build_dir,
json_path=request.json)
if request.json == 'stdout':
print(json_obj)
if test_result.status != kunit_parser.TestStatus.SUCCESS:
return KunitResult(KunitStatus.TEST_FAILURE, test_result,
parse_end - parse_start)
@ -151,7 +162,9 @@ def run_tests(linux: kunit_kernel.LinuxSourceTree,
return exec_result
parse_request = KunitParseRequest(request.raw_output,
exec_result.result)
exec_result.result,
request.build_dir,
request.json)
parse_result = parse_tests(parse_request)
run_end = time.time()
@ -195,7 +208,12 @@ def add_exec_opts(parser):
def add_parse_opts(parser):
parser.add_argument('--raw_output', help='don\'t format output from kernel',
action='store_true')
parser.add_argument('--json',
nargs='?',
help='Stores test results in a JSON, and either '
'prints to stdout or saves to file if a '
'filename is specified',
type=str, const='stdout', default=None)
def main(argv, linux=None):
parser = argparse.ArgumentParser(
@ -237,10 +255,16 @@ def main(argv, linux=None):
cli_args = parser.parse_args(argv)
if get_kernel_root_path():
os.chdir(get_kernel_root_path())
if cli_args.subcommand == 'run':
if not os.path.exists(cli_args.build_dir):
os.mkdir(cli_args.build_dir)
if not os.path.exists(kunit_kernel.kunitconfig_path):
create_default_kunitconfig()
if not linux:
linux = kunit_kernel.LinuxSourceTree()
@ -249,15 +273,19 @@ def main(argv, linux=None):
cli_args.jobs,
cli_args.build_dir,
cli_args.alltests,
cli_args.json,
cli_args.make_options)
result = run_tests(linux, request)
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'config':
if cli_args.build_dir:
if not os.path.exists(cli_args.build_dir):
if cli_args.build_dir and (
not os.path.exists(cli_args.build_dir)):
os.mkdir(cli_args.build_dir)
if not os.path.exists(kunit_kernel.kunitconfig_path):
create_default_kunitconfig()
if not linux:
linux = kunit_kernel.LinuxSourceTree()
@ -270,10 +298,6 @@ def main(argv, linux=None):
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'build':
if cli_args.build_dir:
if not os.path.exists(cli_args.build_dir):
os.mkdir(cli_args.build_dir)
if not linux:
linux = kunit_kernel.LinuxSourceTree()
@ -288,10 +312,6 @@ def main(argv, linux=None):
if result.status != KunitStatus.SUCCESS:
sys.exit(1)
elif cli_args.subcommand == 'exec':
if cli_args.build_dir:
if not os.path.exists(cli_args.build_dir):
os.mkdir(cli_args.build_dir)
if not linux:
linux = kunit_kernel.LinuxSourceTree()
@ -300,7 +320,9 @@ def main(argv, linux=None):
cli_args.alltests)
exec_result = exec_tests(linux, exec_request)
parse_request = KunitParseRequest(cli_args.raw_output,
exec_result.result)
exec_result.result,
cli_args.build_dir,
cli_args.json)
result = parse_tests(parse_request)
kunit_parser.print_with_timestamp((
'Elapsed time: %.3fs\n') % (
@ -314,7 +336,9 @@ def main(argv, linux=None):
with open(cli_args.file, 'r') as f:
kunit_output = f.read().splitlines()
request = KunitParseRequest(cli_args.raw_output,
kunit_output)
kunit_output,
cli_args.build_dir,
cli_args.json)
result = parse_tests(request)
if result.status != KunitStatus.SUCCESS:
sys.exit(1)

63
tools/testing/kunit/kunit_json.py Normal file
View File

@ -0,0 +1,63 @@
# SPDX-License-Identifier: GPL-2.0
#
# Generates JSON from KUnit results according to
# KernelCI spec: https://github.com/kernelci/kernelci-doc/wiki/Test-API
#
# Copyright (C) 2020, Google LLC.
# Author: Heidi Fahim <heidifahim@google.com>
import json
import os
import kunit_parser
from kunit_parser import TestStatus
def get_json_result(test_result, def_config, build_dir, json_path):
sub_groups = []
# Each test suite is mapped to a KernelCI sub_group
for test_suite in test_result.suites:
sub_group = {
"name": test_suite.name,
"arch": "UM",
"defconfig": def_config,
"build_environment": build_dir,
"test_cases": [],
"lab_name": None,
"kernel": None,
"job": None,
"git_branch": "kselftest",
}
test_cases = []
# TODO: Add attachments attribute in test_case with detailed
# failure message, see https://api.kernelci.org/schema-test-case.html#get
for case in test_suite.cases:
test_case = {"name": case.name, "status": "FAIL"}
if case.status == TestStatus.SUCCESS:
test_case["status"] = "PASS"
elif case.status == TestStatus.TEST_CRASHED:
test_case["status"] = "ERROR"
test_cases.append(test_case)
sub_group["test_cases"] = test_cases
sub_groups.append(sub_group)
test_group = {
"name": "KUnit Test Group",
"arch": "UM",
"defconfig": def_config,
"build_environment": build_dir,
"sub_groups": sub_groups,
"lab_name": None,
"kernel": None,
"job": None,
"git_branch": "kselftest",
}
json_obj = json.dumps(test_group, indent=4)
if json_path != 'stdout':
with open(json_path, 'w') as result_path:
result_path.write(json_obj)
root = __file__.split('tools/testing/kunit/')[0]
kunit_parser.print_with_timestamp(
"Test results stored in %s" %
os.path.join(root, result_path.name))
return json_obj

27
tools/testing/kunit/kunit_kernel.py
View File

@ -36,9 +36,9 @@ class LinuxSourceTreeOperations(object):
try:
subprocess.check_output(['make', 'mrproper'], stderr=subprocess.STDOUT)
except OSError as e:
raise ConfigError('Could not call make command: ' + e)
raise ConfigError('Could not call make command: ' + str(e))
except subprocess.CalledProcessError as e:
raise ConfigError(e.output)
raise ConfigError(e.output.decode())
def make_olddefconfig(self, build_dir, make_options):
command = ['make', 'ARCH=um', 'olddefconfig']
@ -49,22 +49,27 @@ class LinuxSourceTreeOperations(object):
try:
subprocess.check_output(command, stderr=subprocess.STDOUT)
except OSError as e:
raise ConfigError('Could not call make command: ' + e)
raise ConfigError('Could not call make command: ' + str(e))
except subprocess.CalledProcessError as e:
raise ConfigError(e.output)
raise ConfigError(e.output.decode())
def make_allyesconfig(self):
def make_allyesconfig(self, build_dir, make_options):
kunit_parser.print_with_timestamp(
'Enabling all CONFIGs for UML...')
command = ['make', 'ARCH=um', 'allyesconfig']
if make_options:
command.extend(make_options)
if build_dir:
command += ['O=' + build_dir]
process = subprocess.Popen(
['make', 'ARCH=um', 'allyesconfig'],
command,
stdout=subprocess.DEVNULL,
stderr=subprocess.STDOUT)
process.wait()
kunit_parser.print_with_timestamp(
'Disabling broken configs to run KUnit tests...')
with ExitStack() as es:
config = open(KCONFIG_PATH, 'a')
config = open(get_kconfig_path(build_dir), 'a')
disable = open(BROKEN_ALLCONFIG_PATH, 'r').read()
config.write(disable)
kunit_parser.print_with_timestamp(
@ -79,9 +84,9 @@ class LinuxSourceTreeOperations(object):
try:
subprocess.check_output(command, stderr=subprocess.STDOUT)
except OSError as e:
raise BuildError('Could not call execute make: ' + e)
raise BuildError('Could not call execute make: ' + str(e))
except subprocess.CalledProcessError as e:
raise BuildError(e.output)
raise BuildError(e.output.decode())
def linux_bin(self, params, timeout, build_dir, outfile):
"""Runs the Linux UML binary. Must be named 'linux'."""
@ -161,9 +166,9 @@ class LinuxSourceTree(object):
return self.build_config(build_dir, make_options)
def build_um_kernel(self, alltests, jobs, build_dir, make_options):
if alltests:
self._ops.make_allyesconfig()
try:
if alltests:
self._ops.make_allyesconfig(build_dir, make_options)
self._ops.make_olddefconfig(build_dir, make_options)
self._ops.make(jobs, build_dir, make_options)
except (ConfigError, BuildError) as e:

33
tools/testing/kunit/kunit_tool_test.py
View File

@ -11,11 +11,13 @@ from unittest import mock
import tempfile, shutil # Handling test_tmpdir
import json
import os
import kunit_config
import kunit_parser
import kunit_kernel
import kunit_json
import kunit
test_tmpdir = ''
@ -230,6 +232,37 @@ class KUnitParserTest(unittest.TestCase):
result = kunit_parser.parse_run_tests(file.readlines())
self.assertEqual('kunit-resource-test', result.suites[0].name)
class KUnitJsonTest(unittest.TestCase):
def _json_for(self, log_file):
with(open(get_absolute_path(log_file))) as file:
test_result = kunit_parser.parse_run_tests(file)
json_obj = kunit_json.get_json_result(
test_result=test_result,
def_config='kunit_defconfig',
build_dir=None,
json_path='stdout')
return json.loads(json_obj)
def test_failed_test_json(self):
result = self._json_for(
'test_data/test_is_test_passed-failure.log')
self.assertEqual(
{'name': 'example_simple_test', 'status': 'FAIL'},
result["sub_groups"][1]["test_cases"][0])
def test_crashed_test_json(self):
result = self._json_for(
'test_data/test_is_test_passed-crash.log')
self.assertEqual(
{'name': 'example_simple_test', 'status': 'ERROR'},
result["sub_groups"][1]["test_cases"][0])
def test_no_tests_json(self):
result = self._json_for(
'test_data/test_is_test_passed-no_tests_run.log')
self.assertEqual(0, len(result['sub_groups']))
class StrContains(str):
def __eq__(self, other):
return self in other