smash stats and ignored message fix

bin/smash

@@ -26,6 +26,7 @@ del standardSetup
 
 global_exitcode_error = False
 
+
 def main():
    import os, os.path, sys
    import glob
@@ -181,7 +182,29 @@ def main():
 
    #
    ###########
-
+
+   class Stats:
+      def __init__(self):
+         self.failed = self.passed = self.ignored = self.total = 0
+
+      def add(self, returncode, ignored):
+         if ignored:
+            self.ignored += 1
+         elif returncode != 0:
+            self.failed += 1
+         else:
+            self.passed += 1
+
+         self.total += 1
+
+      def print_stats(self):
+         #logger.error("IN PRINT")
+         log_quiet("PASSED: %d" % self.passed)
+         log_quiet("FAILED: %d" % self.failed)
+         log_quiet("IGNORED: %d" % self.ignored)
+
+   stats = Stats()
+
    if args.dry_run:
       log_quiet('*** DRY RUN ***')
       smashbox.script.config_log(logging.INFO,hide_password=not args.debug)
@@ -204,6 +227,15 @@ def main():
 
        reporter.testcase_stop(p.returncode)
 
+       if config.get('_test_ignored', None):
+          test_ignored = 1
+          #logger.error('Not NONE: (%s)'%config.get('_test_ignored', None))   
+       else:
+          test_ignored = 0
+          #logger.error('IS NONE')
+
+       stats.add(p.returncode, test_ignored)
+
        if p.returncode != 0:
           if args.keep_going:
              logger.error('Non-zero exit code (%s)'%p.returncode)
@@ -219,6 +251,8 @@ def main():
              logFile = os.path.join(config.smashdir, logName)
              os.remove(logFile)
 
+       config._test_ignored = None
+
 
    from smashbox.utilities import  oc_webdav_url
 
@@ -317,11 +351,18 @@ def main():
    if args.dry_run:
       log_quiet('*** DRY RUN ***')
 
+   import smashbox.alarming
+   stats.print_stats()
+
+   if stats.failed > stats.failed/20.0 :
+      smashbox.alarming.send_alarm("smashbox failure","high rate of failed tests (>20%)",logger)
+
    if global_exitcode_error:
       sys.exit(global_exitcode_error)
 
    # FIXME: add finally clause with reporter.smashbox_stop()
 
+
 if __name__ == '__main__':
    main()
 

python/smashbox/alarming.py

@@ -0,0 +1,38 @@
+#alarms_sent = set()
+
+def send_alarm(what,msg,logger):
+
+    #if what in alarms_sent:
+    #    return
+
+    #alarms_sent.add(what)
+
+    # add signature
+    import socket
+
+    msg += ""
+
+    ALARM_EMAIL = "cernbox-admins@cern.ch"
+    #ALARM_EMAIL = "georgios.alexandropoulos@cern.ch"
+
+    try:
+        logger.warning("ALARM: sending alarm to %s: %s %s",ALARM_EMAIL,what,msg)
+
+        #if not config.ALARM_EMAIL:
+        #    logging.warning("sending alarms disabled (no ALARM_EMAIL)")
+        #    return
+
+        import smtplib
+        fromaddr='cernbox-admins@cern.ch'
+        toaddrs=[ALARM_EMAIL]
+        email = ("From: %s\r\nTo: %s\r\nSubject: smashbox fail alarm : %s\r\n\r\n" % (fromaddr, ", ".join(toaddrs), what))
+        email += what + ': ' + msg
+        server = smtplib.SMTP('localhost')
+        #server.set_debuglevel(1)
+        server.sendmail(fromaddr, toaddrs, email)
+        server.quit()
+    except Exception,x:
+        logger.critical("problem sending alarm: %s %s",what,str(x))
+        alarms_sent.remove(what)
+
+

python/smashbox/monitoring/kibana_monitoring.py

@@ -72,15 +72,15 @@ def test_finish(self):
         if not self.kibana_monitoring_host:
             return
 
-        if self._test_ignored is None: # we just want to track tests that we considered fixed
+        if self._test_ignored: # we just want to track tests that we considered fixed
+            self.test_results['ignoredFailures'] = 1 # if the test is ignored we put it in a separate counter
+        else:
             if(self.test_results['total_errors']>=1): # A subtest is considered failed with one or more errors
                 self.test_results['passed'] = 0
                 self.test_results['failed'] = 1
             else:
                 self.test_results['failed'] = 0
                 self.test_results['passed'] = 1
-        else:
-            self.test_results['ignoredFailures'] = 1 # if the test is ignored we put it in a separate counter
 
         json_results = self.get_json_results()
 

python/smashbox/utilities/__init__.py

@@ -901,8 +901,8 @@ def expect_does_not_exist(fn):
 
 ############ Helper functions to report/document the behaviour of the tests ############
 
-def do_not_report_as_failure(Issue=""):
-   config._test_ignored = Issue 
+def do_not_report_as_failure(reason="failure reporting disabled"):
+   config._test_ignored = reason
 
 ############ Smashbox Exceptions ############