linux/tools/testing/ktest/ktest.pl
Steven Rostedt 2b7d9b2142 ktest: Added continuing on success, clear log and timeout
Add option to continue after a test fails.

Add option to reset the log at start of running ktest.

Update default timeout to 2 minutes.

Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
2010-11-18 11:23:08 -05:00

904 lines
18 KiB
Perl

#!/usr/bin/perl -w
#
# Copywrite 2010 - Steven Rostedt <srostedt@redhat.com>, Red Hat Inc.
# Licensed under the terms of the GNU GPL License version 2
#
use strict;
use IPC::Open2;
use Fcntl qw(F_GETFL F_SETFL O_NONBLOCK);
use FileHandle;
$#ARGV >= 0 || die "usage: autotest.pl config-file\n";
$| = 1;
my %opt;
#default opts
$opt{"NUM_BUILDS"} = 5;
$opt{"DEFAULT_BUILD_TYPE"} = "randconfig";
$opt{"MAKE_CMD"} = "make";
$opt{"TIMEOUT"} = 120;
$opt{"TMP_DIR"} = "/tmp/autotest";
$opt{"SLEEP_TIME"} = 60; # sleep time between tests
$opt{"BUILD_NOCLEAN"} = 0;
$opt{"REBOOT_ON_ERROR"} = 0;
$opt{"POWEROFF_ON_ERROR"} = 0;
$opt{"REBOOT_ON_SUCCESS"} = 1;
$opt{"POWEROFF_ON_SUCCESS"} = 0;
$opt{"BUILD_OPTIONS"} = "";
$opt{"BISECT_SLEEP_TIME"} = 10; # sleep time between bisects
$opt{"CLEAR_LOG"} = 0;
$opt{"SUCCESS_LINE"} = "login:";
$opt{"BOOTED_TIMEOUT"} = 1;
$opt{"DIE_ON_FAILURE"} = 1;
my $version;
my $grub_number;
my $target;
my $make;
my $noclean;
my $minconfig;
my $addconfig;
my $in_bisect = 0;
my $bisect_bad = "";
my $reverse_bisect;
my $in_patchcheck = 0;
my $run_test;
my $redirect;
sub read_config {
my ($config) = @_;
open(IN, $config) || die "can't read file $config";
while (<IN>) {
# ignore blank lines and comments
next if (/^\s*$/ || /\s*\#/);
if (/^\s*(\S+)\s*=\s*(.*?)\s*$/) {
my $lvalue = $1;
my $rvalue = $2;
$opt{$lvalue} = $rvalue;
}
}
close(IN);
}
sub logit {
if (defined($opt{"LOG_FILE"})) {
open(OUT, ">> $opt{LOG_FILE}") or die "Can't write to $opt{LOG_FILE}";
print OUT @_;
close(OUT);
}
}
sub doprint {
print @_;
logit @_;
}
sub dodie {
doprint "CRITICAL FAILURE... ", @_, "\n";
if ($opt{"REBOOT_ON_ERROR"}) {
doprint "REBOOTING\n";
`$opt{"POWER_CYCLE"}`;
} elsif ($opt{"POWEROFF_ON_ERROR"} && defined($opt{"POWER_OFF"})) {
doprint "POWERING OFF\n";
`$opt{"POWER_OFF"}`;
}
die @_;
}
sub fail {
if ($opt{"DIE_ON_FAILURE"}) {
dodie @_;
}
doprint "Failed: ", @_, "\n";
return 1;
}
sub run_command {
my ($command) = @_;
my $dolog = 0;
my $dord = 0;
my $pid;
doprint("$command ... ");
$pid = open(CMD, "$command 2>&1 |") or
(fail "unable to exec $command" and return 0);
if (defined($opt{"LOG_FILE"})) {
open(LOG, ">>$opt{LOG_FILE}") or
dodie "failed to write to log";
$dolog = 1;
}
if (defined($redirect)) {
open (RD, ">$redirect") or
dodie "failed to write to redirect $redirect";
$dord = 1;
}
while (<CMD>) {
print LOG if ($dolog);
print RD if ($dord);
}
waitpid($pid, 0);
my $failed = $?;
close(CMD);
close(LOG) if ($dolog);
close(RD) if ($dord);
if ($failed) {
doprint "FAILED!\n";
} else {
doprint "SUCCESS\n";
}
return !$failed;
}
sub get_grub_index {
return if (defined($grub_number));
doprint "Find grub menu ... ";
$grub_number = -1;
open(IN, "ssh $target cat /boot/grub/menu.lst |")
or die "unable to get menu.lst";
while (<IN>) {
if (/^\s*title\s+$opt{GRUB_MENU}\s*$/) {
$grub_number++;
last;
} elsif (/^\s*title\s/) {
$grub_number++;
}
}
close(IN);
die "Could not find '$opt{GRUB_MENU}' in /boot/grub/menu on $opt{MACHINE}"
if ($grub_number < 0);
doprint "$grub_number\n";
}
my $timeout = $opt{"TIMEOUT"};
sub wait_for_input
{
my ($fp, $time) = @_;
my $rin;
my $ready;
my $line;
my $ch;
if (!defined($time)) {
$time = $timeout;
}
$rin = '';
vec($rin, fileno($fp), 1) = 1;
$ready = select($rin, undef, undef, $time);
$line = "";
# try to read one char at a time
while (sysread $fp, $ch, 1) {
$line .= $ch;
last if ($ch eq "\n");
}
if (!length($line)) {
return undef;
}
return $line;
}
sub reboot_to {
run_command "ssh $target '(echo \"savedefault --default=$grub_number --once\" | grub --batch; reboot)'";
}
sub open_console {
my ($fp) = @_;
my $flags;
my $pid = open($fp, "$opt{CONSOLE}|") or
dodie "Can't open console $opt{CONSOLE}";
$flags = fcntl($fp, F_GETFL, 0) or
dodie "Can't get flags for the socket: $!\n";
$flags = fcntl($fp, F_SETFL, $flags | O_NONBLOCK) or
dodie "Can't set flags for the socket: $!\n";
return $pid;
}
sub close_console {
my ($fp, $pid) = @_;
doprint "kill child process $pid\n";
kill 2, $pid;
print "closing!\n";
close($fp);
}
sub monitor {
my $booted = 0;
my $bug = 0;
my $pid;
my $skip_call_trace = 0;
my $fp = \*IN;
my $loops;
$pid = open_console($fp);
my $line;
my $full_line = "";
doprint "Wait for monitor to settle down.\n";
# read the monitor and wait for the system to calm down
do {
$line = wait_for_input($fp, 5);
} while (defined($line));
reboot_to;
for (;;) {
if ($booted) {
$line = wait_for_input($fp, $opt{"BOOTED_TIMEOUT"});
} else {
$line = wait_for_input($fp);
}
last if (!defined($line));
doprint $line;
# we are not guaranteed to get a full line
$full_line .= $line;
if ($full_line =~ /$opt{"SUCCESS_LINE"}/) {
$booted = 1;
}
if ($full_line =~ /\[ backtrace testing \]/) {
$skip_call_trace = 1;
}
if ($full_line =~ /call trace:/i) {
$bug = 1 if (!$skip_call_trace);
}
if ($full_line =~ /\[ end of backtrace testing \]/) {
$skip_call_trace = 0;
}
if ($full_line =~ /Kernel panic -/) {
$bug = 1;
}
if ($line =~ /\n/) {
$full_line = "";
}
}
close_console($fp, $pid);
if (!$booted) {
return 0 if ($in_bisect);
fail "failed - never got a boot prompt.\n" and return 0;
}
if ($bug) {
return 0 if ($in_bisect);
fail "failed - got a bug report\n" and return 0;
}
return 1;
}
sub install {
run_command "scp $opt{OUTPUT_DIR}/$opt{BUILD_TARGET} $target:$opt{TARGET_IMAGE}" or
dodie "failed to copy image";
my $install_mods = 0;
# should we process modules?
$install_mods = 0;
open(IN, "$opt{OUTPUT_DIR}/.config") or dodie("Can't read config file");
while (<IN>) {
if (/CONFIG_MODULES(=y)?/) {
$install_mods = 1 if (defined($1));
last;
}
}
close(IN);
if (!$install_mods) {
doprint "No modules needed\n";
return;
}
run_command "$make INSTALL_MOD_PATH=$opt{TMP_DIR} modules_install" or
dodie "Failed to install modules";
my $modlib = "/lib/modules/$version";
my $modtar = "autotest-mods.tar.bz2";
run_command "ssh $target rm -rf $modlib" or
dodie "failed to remove old mods: $modlib";
# would be nice if scp -r did not follow symbolic links
run_command "cd $opt{TMP_DIR} && tar -cjf $modtar lib/modules/$version" or
dodie "making tarball";
run_command "scp $opt{TMP_DIR}/$modtar $target:/tmp" or
dodie "failed to copy modules";
unlink "$opt{TMP_DIR}/$modtar";
run_command "ssh $target '(cd / && tar xf /tmp/$modtar)'" or
dodie "failed to tar modules";
run_command "ssh $target rm -f /tmp/$modtar";
}
sub check_buildlog {
my ($patch) = @_;
my $buildlog = "$opt{TMP_DIR}/buildlog";
my @files = `git show $patch | diffstat -l`;
open(IN, "git show $patch |") or
dodie "failed to show $patch";
while (<IN>) {
if (m,^--- a/(.*),) {
chomp $1;
$files[$#files] = $1;
}
}
close(IN);
open(IN, $buildlog) or dodie "Can't open $buildlog";
while (<IN>) {
if (/^\s*(.*?):.*(warning|error)/) {
my $err = $1;
foreach my $file (@files) {
my $fullpath = "$opt{BUILD_DIR}/$file";
if ($file eq $err || $fullpath eq $err) {
fail "$file built with warnings" and return 0;
}
}
}
}
close(IN);
return 1;
}
sub build {
my ($type) = @_;
my $defconfig = "";
my $append = "";
if ($type =~ /^useconfig:(.*)/) {
run_command "cp $1 $opt{OUTPUT_DIR}/.config" or
dodie "could not copy $1 to .config";
$type = "oldconfig";
}
# old config can ask questions
if ($type eq "oldconfig") {
$append = "yes ''|";
# allow for empty configs
run_command "touch $opt{OUTPUT_DIR}/.config";
run_command "mv $opt{OUTPUT_DIR}/.config $opt{OUTPUT_DIR}/config_temp" or
dodie "moving .config";
if (!$noclean && !run_command "$make mrproper") {
dodie "make mrproper";
}
run_command "mv $opt{OUTPUT_DIR}/config_temp $opt{OUTPUT_DIR}/.config" or
dodie "moving config_temp";
} elsif (!$noclean) {
unlink "$opt{OUTPUT_DIR}/.config";
run_command "$make mrproper" or
dodie "make mrproper";
}
# add something to distinguish this build
open(OUT, "> $opt{OUTPUT_DIR}/localversion") or dodie("Can't make localversion file");
print OUT "$opt{LOCALVERSION}\n";
close(OUT);
if (defined($minconfig)) {
$defconfig = "KCONFIG_ALLCONFIG=$minconfig";
}
run_command "$defconfig $append $make $type" or
dodie "failed make config";
# patch check will examine the log
if ($in_patchcheck) {
$redirect = "$opt{TMP_DIR}/buildlog";
}
if (!run_command "$make $opt{BUILD_OPTIONS}") {
undef $redirect;
# bisect may need this to pass
return 0 if ($in_bisect);
fail "failed build" and return 0;
}
undef $redirect;
return 1;
}
sub reboot {
# try to reboot normally
if (!run_command "ssh $target reboot") {
# nope? power cycle it.
run_command "$opt{POWER_CYCLE}";
}
}
sub halt {
if (!run_command "ssh $target halt" or defined($opt{"POWER_OFF"})) {
# nope? the zap it!
run_command "$opt{POWER_OFF}";
}
}
sub success {
my ($i) = @_;
doprint "\n\n*******************************************\n";
doprint "*******************************************\n";
doprint "** SUCCESS!!!! **\n";
doprint "*******************************************\n";
doprint "*******************************************\n";
if ($i != $opt{"NUM_BUILDS"}) {
reboot;
doprint "Sleeping $opt{SLEEP_TIME} seconds\n";
sleep "$opt{SLEEP_TIME}";
}
}
sub get_version {
# get the release name
doprint "$make kernelrelease ... ";
$version = `$make kernelrelease | tail -1`;
chomp($version);
doprint "$version\n";
}
sub child_run_test {
my $failed;
$failed = !run_command $run_test;
exit $failed;
}
my $child_done;
sub child_finished {
$child_done = 1;
}
sub do_run_test {
my $child_pid;
my $child_exit;
my $pid;
my $line;
my $full_line;
my $bug = 0;
my $fp = \*IN;
$pid = open_console($fp);
# read the monitor and wait for the system to calm down
do {
$line = wait_for_input($fp, 1);
} while (defined($line));
$child_done = 0;
$SIG{CHLD} = qw(child_finished);
$child_pid = fork;
child_run_test if (!$child_pid);
$full_line = "";
do {
$line = wait_for_input($fp, 1);
if (defined($line)) {
# we are not guaranteed to get a full line
$full_line .= $line;
if ($full_line =~ /call trace:/i) {
$bug = 1;
}
if ($full_line =~ /Kernel panic -/) {
$bug = 1;
}
if ($line =~ /\n/) {
$full_line = "";
}
}
} while (!$child_done && !$bug);
if ($bug) {
doprint "Detected kernel crash!\n";
# kill the child with extreme prejudice
kill 9, $child_pid;
}
waitpid $child_pid, 0;
$child_exit = $?;
close_console($fp, $pid);
if ($bug || $child_exit) {
return 0 if $in_bisect;
fail "test failed" and return 0;
}
return 1;
}
sub run_bisect {
my ($type) = @_;
my $failed = 0;
my $result;
my $output;
my $ret;
if (defined($minconfig)) {
build "useconfig:$minconfig" or $failed = 1;
} else {
# ?? no config to use?
build "oldconfig" or $failed = 1;
}
if ($type ne "build") {
fail "Failed on build" if $failed;
# Now boot the box
get_grub_index;
get_version;
install;
monitor or $failed = 1;
if ($type ne "boot") {
fail "Failed on boot" if $failed;
do_run_test or $failed = 1;
}
}
if ($failed) {
$result = "bad";
# reboot the box to a good kernel
if ($type eq "boot") {
reboot;
doprint "sleep a little for reboot\n";
sleep $opt{"BISECT_SLEEP_TIME"};
}
} else {
$result = "good";
}
# Are we looking for where it worked, not failed?
if ($reverse_bisect) {
if ($failed) {
$result = "good";
} else {
$result = "bad";
}
}
doprint "git bisect $result ... ";
$output = `git bisect $result 2>&1`;
$ret = $?;
logit $output;
if ($ret) {
doprint "FAILED\n";
fail "Failed to git bisect";
}
doprint "SUCCESS\n";
if ($output =~ m/^(Bisecting: .*\(roughly \d+ steps?\))\s+\[([[:xdigit:]]+)\]/) {
doprint "$1 [$2]\n";
} elsif ($output =~ m/^([[:xdigit:]]+) is the first bad commit/) {
$bisect_bad = $1;
doprint "Found bad commit... $1\n";
return 0;
} else {
# we already logged it, just print it now.
print $output;
}
return 1;
}
sub bisect {
my ($i) = @_;
my $result;
die "BISECT_GOOD[$i] not defined\n" if (!defined($opt{"BISECT_GOOD[$i]"}));
die "BISECT_BAD[$i] not defined\n" if (!defined($opt{"BISECT_BAD[$i]"}));
die "BISECT_TYPE[$i] not defined\n" if (!defined($opt{"BISECT_TYPE[$i]"}));
my $good = $opt{"BISECT_GOOD[$i]"};
my $bad = $opt{"BISECT_BAD[$i]"};
my $type = $opt{"BISECT_TYPE[$i]"};
if (defined($opt{"BISECT_REVERSE[$i]"}) &&
$opt{"BISECT_REVERSE[$i]"} == 1) {
doprint "Performing a reverse bisect (bad is good, good is bad!)\n";
$reverse_bisect = 1;
} else {
$reverse_bisect = 0;
}
$in_bisect = 1;
run_command "git bisect start" or
fail "could not start bisect";
run_command "git bisect good $good" or
fail "could not set bisect good to $good";
run_command "git bisect bad $bad" or
fail "could not set bisect good to $bad";
# Can't have a test without having a test to run
if ($type eq "test" && !defined($run_test)) {
$type = "boot";
}
do {
$result = run_bisect $type;
} while ($result);
run_command "git bisect log" or
dodie "could not capture git bisect log";
run_command "git bisect reset" or
dodie "could not reset git bisect";
doprint "Bad commit was [$bisect_bad]\n";
$in_bisect = 0;
success $i;
}
sub patchcheck {
my ($i) = @_;
die "PATCHCHECK_START[$i] not defined\n"
if (!defined($opt{"PATCHCHECK_START[$i]"}));
die "PATCHCHECK_TYPE[$i] not defined\n"
if (!defined($opt{"PATCHCHECK_TYPE[$i]"}));
my $start = $opt{"PATCHCHECK_START[$i]"};
my $end = "HEAD";
if (defined($opt{"PATCHCHECK_END[$i]"})) {
$end = $opt{"PATCHCHECK_END[$i]"};
}
my $type = $opt{"PATCHCHECK_TYPE[$i]"};
# Can't have a test without having a test to run
if ($type eq "test" && !defined($run_test)) {
$type = "boot";
}
open (IN, "git log --pretty=oneline $end|") or
dodie "could not get git list";
my @list;
while (<IN>) {
chomp;
$list[$#list+1] = $_;
last if (/^$start/);
}
close(IN);
if ($list[$#list] !~ /^$start/) {
fail "SHA1 $start not found";
}
# go backwards in the list
@list = reverse @list;
my $save_clean = $noclean;
$in_patchcheck = 1;
foreach my $item (@list) {
my $sha1 = $item;
$sha1 =~ s/^([[:xdigit:]]+).*/$1/;
doprint "\nProcessing commit $item\n\n";
run_command "git checkout $sha1" or
die "Failed to checkout $sha1";
# only clean on the first and last patch
if ($item eq $list[0] ||
$item eq $list[$#list]) {
$noclean = $save_clean;
} else {
$noclean = 1;
}
if (defined($minconfig)) {
build "useconfig:$minconfig" or return 0;
} else {
# ?? no config to use?
build "oldconfig" or return 0;
}
check_buildlog $sha1 or return 0;
next if ($type eq "build");
get_grub_index;
get_version;
install;
monitor or return 0;
next if ($type eq "boot");
do_run_test or next;
}
$in_patchcheck = 0;
success $i;
return 1;
}
read_config $ARGV[0];
# mandatory configs
die "MACHINE not defined\n" if (!defined($opt{"MACHINE"}));
die "SSH_USER not defined\n" if (!defined($opt{"SSH_USER"}));
die "BUILD_DIR not defined\n" if (!defined($opt{"BUILD_DIR"}));
die "OUTPUT_DIR not defined\n" if (!defined($opt{"OUTPUT_DIR"}));
die "BUILD_TARGET not defined\n" if (!defined($opt{"BUILD_TARGET"}));
die "TARGET_IMAGE not defined\n" if (!defined($opt{"TARGET_IMAGE"}));
die "POWER_CYCLE not defined\n" if (!defined($opt{"POWER_CYCLE"}));
die "CONSOLE not defined\n" if (!defined($opt{"CONSOLE"}));
die "LOCALVERSION not defined\n" if (!defined($opt{"LOCALVERSION"}));
die "GRUB_MENU not defined\n" if (!defined($opt{"GRUB_MENU"}));
chdir $opt{"BUILD_DIR"} || die "can't change directory to $opt{BUILD_DIR}";
$target = "$opt{SSH_USER}\@$opt{MACHINE}";
if ($opt{"CLEAR_LOG"} && defined($opt{"LOG_FILE"})) {
unlink $opt{"LOG_FILE"};
}
doprint "\n\nSTARTING AUTOMATED TESTS\n\n";
foreach my $option (sort keys %opt) {
doprint "$option = $opt{$option}\n";
}
$make = "$opt{MAKE_CMD} O=$opt{OUTPUT_DIR}";
sub set_build_option {
my ($name, $i) = @_;
my $option = "$name\[$i\]";
if (defined($opt{$option})) {
return $opt{$option};
}
if (defined($opt{$name})) {
return $opt{$name};
}
return undef;
}
# First we need to do is the builds
for (my $i = 1; $i <= $opt{"NUM_BUILDS"}; $i++) {
my $type = "BUILD_TYPE[$i]";
if (!defined($opt{$type})) {
$opt{$type} = $opt{"DEFAULT_BUILD_TYPE"};
}
$noclean = set_build_option("BUILD_NOCLEAN", $i);
$minconfig = set_build_option("MIN_CONFIG", $i);
$run_test = set_build_option("TEST", $i);
$addconfig = set_build_option("ADD_CONFIG", $i);
doprint "\n\n";
doprint "RUNNING TEST $i of $opt{NUM_BUILDS} with option $opt{$type}\n\n";
if (!defined($minconfig)) {
$minconfig = $addconfig;
} elsif (defined($addconfig)) {
run_command "cat $addconfig $minconfig > $opt{TMP_DIR}/use_config" or
dodie "Failed to create temp config";
$minconfig = "$opt{TMP_DIR}/use_config";
}
my $checkout = $opt{"CHECKOUT[$i]"};
if (defined($checkout)) {
run_command "git checkout $checkout" or
die "failed to checkout $checkout";
}
if ($opt{$type} eq "bisect") {
bisect $i;
next;
} elsif ($opt{$type} eq "patchcheck") {
patchcheck $i;
next;
}
if ($opt{$type} ne "nobuild") {
build $opt{$type} or next;
}
get_grub_index;
get_version;
install;
monitor or next;
if (defined($run_test)) {
do_run_test or next;
}
success $i;
}
if ($opt{"POWEROFF_ON_SUCCESS"}) {
halt;
} elsif ($opt{"REBOOT_ON_SUCCESS"}) {
reboot;
}
exit 0;