From e8a8990cae6216279396bbd46ccdd41044dba9f3 Mon Sep 17 00:00:00 2001
From: Iain Lane <iain.lane@canonical.com>
Date: Tue, 21 Feb 2017 13:40:18 +0000
Subject: [PATCH] Generalise kernel panic code to be able to fail on more kinds
of error
systemd can fail with "Freezing execution." - make those real instead of
tmpfails.
---
worker/worker | 17 +++++++++++------
1 file changed, 11 insertions(+), 6 deletions(-)
diff --git a/worker/worker b/worker/worker
index 207e362..afb3f1f 100755
--- a/worker/worker
+++ b/worker/worker
@@ -34,6 +34,9 @@ running_test = False
status_exchange_name = 'teststatus.fanout'
amqp_con = None
+FAIL_STRINGS = ['Kernel panic - not syncing:',
+ 'Freezing execution.']
+
def term_handler(signum, frame):
'''SIGTERM handler, for clean exit after current test'''
@@ -395,7 +398,7 @@ def request(msg):
global running_test
running_test = True
start_time = time.time()
- num_kernel_panics = 0
+ num_failures = 0
for retry in range(3):
retry_start_time = time.time()
logging.info('Running %s', ' '.join(argv))
@@ -407,9 +410,11 @@ def request(msg):
try:
with open(os.path.join(out_dir, 'log')) as f:
contents = f.read()
- if 'Kernel panic - not syncing:' in contents:
- num_kernel_panics += 1
- logging.warning('Detected kernel panic - seen %d so far', num_kernel_panics)
+ fails = [s for s in FAIL_STRINGS if s in contents]
+ if fails:
+ num_failures += 1
+ logging.warning('Saw %s in log, which is a sign of a real (not tmp) failure - seen %d so far',
+ ' and '.join(fails), num_failures)
except IOError as e:
logging.error('Could not read log file: %s' % str(e))
logging.warning('Testbed failure, retrying in 5 minutes')
@@ -424,8 +429,8 @@ def request(msg):
else:
break
else:
- if num_kernel_panics >= 3:
- logging.warning('Three kernel panics in a row - considering this a failure rather than tmpfail')
+ if num_failures >= 3:
+ logging.warning('Three fails in a row - considering this a failure rather than tmpfail')
code = 4
else:
logging.error('Three tmpfails in a row, aborting worker. Log follows:')
--
2.11.0