op-test

#!/usr/bin/env python3
# IBM_PROLOG_BEGIN_TAG
# This is an automatically generated prolog.
#
# $Source: op-test-framework/ci/source/op_ci_bmc.py $
#
# OpenPOWER Automated Test Project
#
# Contributors Listed Below - COPYRIGHT 2015-2017
# [+] International Business Machines Corp.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied. See the License for the specific language governing
# permissions and limitations under the License.
#
# IBM_PROLOG_END_TAG
"""
op-test: run OpenPOWER test suite(s)
"""
from testcases import HelloWorld
import testcases
from testcases import PetitbootMMU
from testcases import RunHostTest
from testcases import OpTestRebootTimeout
from testcases import OpTestMtdPnorDriver
from testcases import OpTestMamboSim
from testcases import IpmiTorture
from testcases import InstallUpstreamKernel
from testcases import EMStress
from testcases import ConsoleBug150765
from testcases import BootTorture
from testcases import BMCResetTorture
from testcases import OpTestKernelArg
from testcases import EnergyScale_BaseLine
from testcases import CpuHotPlug
from testcases import IplParams
from testcases import TrustedBoot
from testcases import SecureBoot
from testcases import OpalSysfsTests
from testcases import gcov
from testcases import OpTestOpenCAPI
from testcases import OpTestCAPI
from testcases import InstallHostOS
from testcases import InstallRhel
from testcases import InstallUbuntu
from testcases import PciSlotLocCodes
from testcases import DeviceTreeWarnings
from testcases import DeviceTreeValidation
from testcases import SbePassThrough
from testcases import OpalGard
from testcases import OpTestPNOR
from testcases import testCronus
from testcases import testRestAPI
from testcases import Console
from testcases import fspTODCorruption
from testcases import OpalUtils
from testcases import OpTestPrdDaemon
from testcases import fspresetReload
from testcases import PowerNVDump
from testcases import EPOW
from testcases import DPO
from testcases import LightPathDiagnostics
from testcases import OpalErrorLog
from testcases import OpTestFlash
from testcases import KernelLog
from testcases import OpalMsglog
from testcases import SystemLogin
from testcases import OpTestDumps
from testcases import OpTestSystemBootSequence
from testcases import OpTestOCC
from testcases import OpTestOOBIPMI
from testcases import OpTestNVRAM
from testcases import OpTestInbandUsbInterface
from testcases import OpTestInbandIPMI
from testcases import OpTestIPMIReprovision
from testcases import OpTestIPMILockMode
from testcases import I2C
from testcases import OpTestHeartbeat
from testcases import OpTestHMIHandling
from testcases import OpTestFastReboot
from testcases import OpTestEnergyScale
from testcases import OpTestEEH
from testcases import AT24driver
from testcases import OpTestEM
from testcases import OpTestRTCdriver
from testcases import Petitbooti18n
from testcases import PetitbootDropbearServer
from testcases import BasicIPL
from testcases import FWTS
from testcases import OpTestPCI
from testcases import OpTestPrdDriver
from testcases import OpTestSensors
from testcases import OpTestSwitchEndianSyscall
from testcases import OpTestHostboot
from testcases import OpTestExample
import OpTestConfiguration
import sys
import time
import traceback
import os
import unittest
import re
import signal
import logging
import OpTestLogger
import importlib
# op-test is the parent logger
optestlog = logging.getLogger(OpTestLogger.optest_logger_glob.parent_logger)

OpTestConfiguration.conf = OpTestConfiguration.OpTestConfiguration()


def optest_handler(signum, frame):
    # CURRENTLY we do NOT have any artifacts needing cleanup
    # during the exposed signal handler window when the OpTestConfiguration
    # object is not bootstrapped
    # Future design consideration should provide parent/child
    # structure to monitor for signals if needing to close this exposure
    signal_dict = dict((key, value) for value, key in reversed(sorted(signal.__dict__.items()))
                       if value.startswith('SIG') and not value.startswith('SIG_'))
    if OpTestConfiguration.conf.signal_ready == True:
        OpTestConfiguration.conf.util.cleanup()
    traceback.print_stack()
    optestlog.error("OpTestSystem exiting from signal '{}' signum={}".format(
        signal_dict[signum], signum))
    os._exit(1)
    # do not use sys.exit, sys.exit in python throws an exception
    # to the stack frame executing at the time the signal is received
    # by the python interpreter, if the exception was caught during some other
    # try/except block, control will be given back to that block


signal.signal(signal.SIGHUP, optest_handler)
signal.signal(signal.SIGINT, optest_handler)
signal.signal(signal.SIGQUIT, optest_handler)
signal.signal(signal.SIGILL, optest_handler)
signal.signal(signal.SIGABRT, optest_handler)
signal.signal(signal.SIGFPE, optest_handler)
signal.signal(signal.SIGSEGV, optest_handler)
signal.signal(signal.SIGTERM, optest_handler)
signal.signal(signal.SIGUSR1, optest_handler)
signal.signal(signal.SIGUSR2, optest_handler)
args, remaining_args = OpTestConfiguration.conf.parse_args(sys.argv)
OpTestConfiguration.conf.objs()


class SystemAccessSuite():
    '''
    Tests all system interfaces
    '''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(SystemLogin.system_access_suite())

    def suite(self):
        return self.s


class StandbySuite():
    '''Machine at standby. Focused on BMC'''

    def __init__(self):
        self.s = unittest.TestSuite()
#        self.s.addTest(OpTestEnergyScale.standby_suite())

    def suite(self):
        return self.s


class SkirootSuite():
    '''Tests in Petitboot environment'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(BasicIPL.GotoPetitbootShell())
        # disabling DeviceTreeWarnings temporarily
#        self.s.addTest(DeviceTreeWarnings.Skiroot())
        self.s.addTest(unittest.TestLoader(
        ).loadTestsFromTestCase(IplParams.Skiroot))
        # disabling temporarily
#        self.s.addTest(SecureBoot.VerifyOPALSecureboot())
        self.s.addTest(TrustedBoot.VerifyOPALTrustedBoot())
        # disabling OpTestEM temporarily
#        self.s.addTest(OpTestEM.skiroot_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())
        self.s.addTest(OpTestRTCdriver.SkirootRTC())
        # disabling temporarily
#        self.s.addTest(AT24driver.SkirootAT24())
        self.s.addTest(I2C.BasicSkirootI2C())
        self.s.addTest(PciSlotLocCodes.SkirootDT())
        self.s.addTest(OpTestPCI.skiroot_suite())
        self.s.addTest(PetitbootDropbearServer.PetitbootDropbearServer())
        self.s.addTest(Petitbooti18n.Petitbooti18n())
        self.s.addTest(OpTestFastReboot.OpTestFastReboot())
        self.s.addTest(OpTestHeartbeat.HeartbeatSkiroot())
        self.s.addTest(OpTestNVRAM.SkirootNVRAM())
        self.s.addTest(Console.suite())
        self.s.addTest(OpTestPNOR.Skiroot())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationSkiroot())
        # disabling temporarily
#        self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(OpalSysfsTests.Skiroot))
        # disabling OpalMsglog temporarily
#        self.s.addTest(OpalMsglog.Skiroot())
        # disabling KernelLog temporarily
#        self.s.addTest(KernelLog.Skiroot())
        self.s.addTest(gcov.Skiroot())
        self.s.addTest(DPO.DPOSkiroot())
#        self.s.addTest(OpTestEnergyScale.runtime_suite())
        self.s.addTest(PetitbootMMU.PetitbootMMU())

    def suite(self):
        return self.s


class MamboSuite():
    '''Tests in Petitboot environment under Mambo'''

    def __init__(self):
        # All of the commented out suites should be eventually enabled
        self.s = unittest.TestSuite()
        self.s.addTest(BasicIPL.GotoPetitbootShell())
        self.s.addTest(DeviceTreeWarnings.Skiroot())
        # self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(IplParams.Skiroot))
        self.s.addTest(SecureBoot.VerifyOPALSecureboot())
        self.s.addTest(TrustedBoot.VerifyOPALTrustedBoot())
        # self.s.addTest(OpTestEM.skiroot_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())
        self.s.addTest(OpTestRTCdriver.SkirootRTC())
        # self.s.addTest(AT24driver.SkirootAT24())
        self.s.addTest(I2C.BasicSkirootI2C())
        self.s.addTest(PciSlotLocCodes.SkirootDT())
        self.s.addTest(OpTestPCI.skiroot_suite())
        self.s.addTest(PetitbootDropbearServer.PetitbootDropbearServer())
        self.s.addTest(Petitbooti18n.Petitbooti18n())
        self.s.addTest(OpTestFastReboot.OpTestFastReboot())
        self.s.addTest(OpTestHeartbeat.HeartbeatSkiroot())
        self.s.addTest(OpTestNVRAM.SkirootNVRAM())
        # self.s.addTest(Console.suite())
        self.s.addTest(OpTestPNOR.Skiroot())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationSkiroot())
        self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(
            OpalSysfsTests.Skiroot))
        self.s.addTest(OpalMsglog.Skiroot())
        self.s.addTest(KernelLog.Skiroot())
        self.s.addTest(DPO.DPOSkiroot())
#        self.s.addTest(OpTestEnergyScale.runtime_suite())
        self.s.addTest(PetitbootMMU.PetitbootMMU())

    def suite(self):
        return self.s


class HostSuite():
    '''Tests run in booted OS'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(SystemLogin.OOBHostLogin())
        # disabling temporarily
#        self.s.addTest(DeviceTreeWarnings.Host())
        self.s.addTest(OpTestOCC.basic_suite())
        self.s.addTest(unittest.TestLoader(
        ).loadTestsFromTestCase(IplParams.Host))
        self.s.addTest(OpTestPrdDriver.OpTestPrdDriver())
        self.s.addTest(PciSlotLocCodes.HostDT())
        self.s.addTest(OpTestPCI.host_suite())
        self.s.addTest(FWTS.FWTS())
        self.s.addTest(OpTestRTCdriver.BasicRTC())
        # disabling temporarily
#        self.s.addTest(OpTestEM.host_suite())
        self.s.addTest(AT24driver.AT24driver())
        self.s.addTest(I2C.BasicI2C())
        self.s.addTest(OpTestIPMILockMode.OpTestIPMILockMode())
        self.s.addTest(OpTestInbandIPMI.basic_suite())
        self.s.addTest(OpTestInbandUsbInterface.basic_suite())
        self.s.addTest(OpTestNVRAM.HostNVRAM())
        self.s.addTest(OpTestHeartbeat.HeartbeatHost())
        self.s.addTest(OpTestSensors.OpTestSensors())
        self.s.addTest(OpalErrorLog.BasicTest())
        self.s.addTest(OpTestPrdDaemon.OpTestPrdDaemon())
        self.s.addTest(SbePassThrough.SbePassThrough())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationHost())
        # disabling temporarily
#        self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(OpalSysfsTests.Host))
        # disabling temporarily
#        self.s.addTest(KernelLog.Host()) # We run before HMI to clear out HMI errors from kernel log
        self.s.addTest(OpTestHMIHandling.suite())
        self.s.addTest(OpTestPNOR.Host())
        # disabling temporarily
#        self.s.addTest(OpalMsglog.Host())
#        self.s.addTest(KernelLog.Host())
        self.s.addTest(OpTestOCC.OCC_RESET())
        self.s.addTest(gcov.Host())

    def suite(self):
        return self.s


class ExperimentalSuite():
    '''Tests that need further development'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestEEH.suite())
        # SwitchEndian is here as we need to resolve issue of running
        # kernel self-tests as part of op-test or not.
        self.s.addTest(OpTestSwitchEndianSyscall.OpTestSwitchEndianSyscall())

    def suite(self):
        return self.s


class BasicIPLSuite():
    '''Basic boot/reboot power on/off'''

    def suite(self):
        return BasicIPL.suite()


class SBSuite():
    '''Secure boot tests'''

    def suite(self):
        return SecureBoot.secureboot_suite()


class TBSuite():
    '''Trusted boot tests'''

    def suite(self):
        return TrustedBoot.trustedboot_suite()


class DefaultSuite():
    '''Basic regression tests'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(SkirootSuite().suite())
        if OpTestConfiguration.conf.args.host_scratch_disk:
            self.s.addTest(InstallHost().suite())
        self.s.addTest(HostSuite().suite())

    def suite(self):
        return self.s


class QemuSuite():
    '''Basic regression tests'''

    def __init__(self):
        # this is the same as Skiroot, but with currently
        # failing tests disabled
        self.s = unittest.TestSuite()
        self.s.addTest(BasicIPL.GotoPetitbootShell())
        # Currently we have some device tree warnings
        # self.s.addTest(DeviceTreeWarnings.Skiroot())
        # self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(IplParams.Skiroot))
        self.s.addTest(SecureBoot.VerifyOPALSecureboot())
        self.s.addTest(TrustedBoot.VerifyOPALTrustedBoot())
        self.s.addTest(OpTestEM.skiroot_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())
        self.s.addTest(OpTestRTCdriver.SkirootRTC())
        self.s.addTest(AT24driver.SkirootAT24())
        self.s.addTest(I2C.BasicSkirootI2C())
        # self.s.addTest(PciSlotLocCodes.SkirootDT())
        self.s.addTest(OpTestPCI.skiroot_suite())
        self.s.addTest(PetitbootDropbearServer.PetitbootDropbearServer())
        self.s.addTest(Petitbooti18n.Petitbooti18n())
        self.s.addTest(OpTestFastReboot.OpTestFastReboot())
        self.s.addTest(OpTestHeartbeat.HeartbeatSkiroot())
        self.s.addTest(OpTestNVRAM.SkirootNVRAM())
        self.s.addTest(Console.suite())
        self.s.addTest(OpTestPNOR.Skiroot())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationSkiroot())
        self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(
            OpalSysfsTests.Skiroot))
        self.s.addTest(OpalMsglog.Skiroot())
        self.s.addTest(KernelLog.Skiroot())
        self.s.addTest(DPO.DPOSkiroot())
        self.s.addTest(PetitbootMMU.PetitbootMMU())

    def suite(self):
        return self.s


class FullSuite():
    '''Every stable test'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(BasicIPLSuite().suite())
        self.s.addTest(DefaultSuite().suite())
        self.s.addTest(testRestAPI.runtime_suite())
        self.s.addTest(OpalGard.OpalGard())
        self.s.addTest(OpalUtils.OpalUtils())
        self.s.addTest(OpTestRTCdriver.HostRTC())
        self.s.addTest(OpTestInbandIPMI.full_suite())
        self.s.addTest(OpTestInbandUsbInterface.full_suite())
        self.s.addTest(I2C.FullI2C())
        self.s.addTest(OpTestSensors.OpTestSensors())
        self.s.addTest(LightPathDiagnostics.suite())
        self.s.addTest(OpalErrorLog.FullTest())
        self.s.addTest(OpTestSwitchEndianSyscall.OpTestSwitchEndianSyscall())
        self.s.addTest(OpalMsglog.Host())
        self.s.addTest(KernelLog.Host())

    def suite(self):
        return self.s


class OpTestEMHostSuite():
    '''Energy Management'''

    def __init__(self):
        self.s = unittest.TestSuite()

    def suite(self):
        self.s.addTest(OpTestEM.host_suite())
        self.s.addTest(EnergyScale_BaseLine.EnergyScale_BaseLine())
        self.s.addTest(unittest.TestLoader(
        ).loadTestsFromTestCase(OpalSysfsTests.Host))
        self.s.addTest(OpTestSensors.OpTestSensors())
        self.s.addTest(CpuHotPlug.CpuHotPlug())
        self.s.addTest(OCCSuite().suite())
        self.s.addTest(OpTestSensors.OpTestSensors())

        return self.s


class OpTestEMSuite():
    '''Energy Management'''

    def __init__(self):
        self.s = unittest.TestSuite()

    def suite(self):
        self.s.addTest(OpTestEMHostSuite().suite())
        self.s.addTest(OpTestEM.skiroot_suite())
        return self.s


class BasicPCISuite():
    '''Basic PCI Suite'''

    def __init__(self):
        self.s = unittest.TestSuite()

    def suite(self):
        self.s.addTest(OpTestPCI.skiroot_suite())
        self.s.addTest(OpTestPCI.host_suite())
        return self.s


class OpTestEEHSuite():
    '''PCI EEH error recovery'''

    def suite(self):
        return OpTestEEH.suite()


class HMISuite():
    '''HMI handling'''

    def suite(self):
        return OpTestHMIHandling.suite()


class ExperimentalHMISuite():
    def suite(self):
        return OpTestHMIHandling.experimental_suite()


class UnrecoverableHMISuite():
    def suite(self):
        return OpTestHMIHandling.unrecoverable_suite()


class OpTestEnergyScaleSuite():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestEnergyScale.standby_suite())
        self.s.addTest(OpTestEnergyScale.runtime_suite())

    def suite(self):
        return self.s


class BrokenReprovisionSuite():
    def suite(self):
        return OpTestIPMIReprovision.broken_suite()


class ExperimentalReprovisionSuite():
    def suite(self):
        return OpTestIPMIReprovision.experimental_suite()


class InbandIPMISuite():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestInbandIPMI.basic_suite())
        self.s.addTest(OpTestInbandIPMI.full_suite())
        self.s.addTest(OpTestInbandUsbInterface.basic_suite())
        self.s.addTest(OpTestInbandUsbInterface.full_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_basic_suite())
        self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())

    def suite(self):
        return self.s


class OutofbandIPMISuite():
    '''Out of band IPMI'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestOOBIPMI.basic_suite())
        self.s.addTest(OpTestOOBIPMI.standby_suite())
        self.s.addTest(OpTestOOBIPMI.runtime_suite())

    def suite(self):
        return self.s


class RestAPISuite():
    '''REST API(OpenBMC Specific Out of band management)'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(testRestAPI.runtime_suite())
        self.s.addTest(testRestAPI.host_off_suite())

    def suite(self):
        return self.s


class CronusSuite():
    '''Cronus'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(testCronus.runtime_suite())
        self.s.addTest(testCronus.host_off_suite())

    def suite(self):
        return self.s


class OCCSuite():
    '''OCC Test Suite'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestOCC.OCC_RESET())
        self.s.addTest(OpTestOCC.basic_suite())
        self.s.addTest(OpTestOCC.full_suite())
        self.s.addTest(OpTestOCC.OCCRESET_FSP())

    def suite(self):
        return self.s


class CAPISuite():
    '''CAPI Tests'''

    def suite(self):
        return OpTestCAPI.capi_test_suite()


class OpenCAPISuite():
    '''OpenCAPI Tests'''

    def suite(self):
        return OpTestOpenCAPI.opencapi_test_suite()


class SystemIPLSuite():
    '''System IPL's'''

    def suite(self):
        return OpTestSystemBootSequence.suite()


class CrashSuite():
    '''Crash Test Suite'''

    def suite(self):
        return PowerNVDump.crash_suite()


class FspOpalSuite():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestDumps.suite())
        self.s.addTest(OpalErrorLog.BasicTest())
        self.s.addTest(OpalErrorLog.FullTest())
        self.s.addTest(LightPathDiagnostics.extended_suite())
        self.s.addTest(fspresetReload.suite())
        self.s.addTest(EPOW.suite())
        self.s.addTest(OpTestOCC.OCCRESET_FSP())
        self.s.addTest(fspTODCorruption.suite())

    def suite(self):
        return self.s


class KnownBugs():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(Console.Console32k())
        self.s.addTest(Console.ControlC())

    def suite(self):
        return self.s


class FlashFirmware():
    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(testcases.OpTestFlash.FSPFWImageFLASH())
        self.s.addTest(testcases.OpTestFlash.BmcImageFlash())
        self.s.addTest(testcases.OpTestFlash.PNORFLASH())
        self.s.addTest(testcases.OpTestFlash.OpalLidsFLASH())

    def suite(self):
        return self.s


class InstallHost():
    def __init__(self):
        self.s = unittest.TestSuite()
        conf = OpTestConfiguration.conf
        target = conf.args.os_repo if conf.args.os_repo else conf.args.os_cdrom
        if conf.args.host_scratch_disk and target:
            if "ubuntu" in target.lower():
                self.s.addTest(InstallUbuntu.InstallUbuntu())
                optestlog.info('InstallUbuntu added to default suite')
            elif "rhel" or "fedora" in target.lower():
                self.s.addTest(InstallRhel.InstallRhel())
                optestlog.info('InstallRhel added to default suite')
            elif "hostos" in target.lower():
                self.s.addTest(InstallHostOS.InstallHostOS())
                optestlog.info('InstallHostOS added to default suite')
            else:
                optestlog.info(
                    "Could not parse OS name from '{}'".format(target))
                conf.util.cleanup()
                exit(-1)
        else:
            optestlog.info("Missing parameters for InstallHost suite, if this"
                           " is unexpected, check AES/Hostlocker for Attached Disk."
                           " This was triggered with host_scratch_disk={} target={}"
                           .format(conf.args.host_scratch_disk, target))
            conf.util.cleanup()
            exit(-1)

    def suite(self):
        return self.s


class OpPCIRegressionSuite():
    '''Tests covering PCI'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestPCI.skiroot_hardboot_suite())
        self.s.addTest(OpTestPCI.host_hardboot_suite())
        self.s.addTest(OpTestPCI.skiroot_softboot_suite())
        self.s.addTest(OpTestPCI.host_softboot_suite())

    def suite(self):
        return self.s


class OpTestHostbootSuite():
    '''Tests in OpTestHostboot'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestHostboot.skiroot_full_suite())
        self.s.addTest(OpTestHostboot.host_full_suite())

    def suite(self):
        return self.s


class OpTestExampleSuite():
    '''Tests in OpTestExample'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestExample.skiroot_full_suite())
        self.s.addTest(OpTestExample.host_full_suite())

    def suite(self):
        return self.s


class OpenBMCPalmettoSkirootSuite():
    '''Tests that pass for an OpenBMC Palmetto, restricted to Petitboot environment'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(BasicIPL.GotoPetitbootShell())
        # fails due to many DT warnings.
        # self.s.addTest(DeviceTreeWarnings.Skiroot())
        # fails
        # self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(IplParams.Skiroot))
        # fails CAPP verification
        # self.s.addTest(SecureBoot.VerifyOPALSecureboot())
        self.s.addTest(TrustedBoot.VerifyOPALTrustedBoot())
        self.s.addTest(OpTestEM.skiroot_suite())
        # No FRUs for Palmetto, other tests do work though.
        # self.s.addTest(OpTestInbandIPMI.skiroot_full_suite())
        # self.s.addTest(OpTestInbandUsbInterface.skiroot_full_suite())
        self.s.addTest(OpTestRTCdriver.SkirootRTC())
        # We're having some strange I2c issues, so disable these
        # self.s.addTest(AT24driver.SkirootAT24())
        # self.s.addTest(I2C.BasicSkirootI2C())
        self.s.addTest(PciSlotLocCodes.SkirootDT())
        self.s.addTest(OpTestPCI.skiroot_suite())
        self.s.addTest(PetitbootDropbearServer.PetitbootDropbearServer())
        self.s.addTest(Petitbooti18n.Petitbooti18n())
        self.s.addTest(OpTestFastReboot.OpTestFastReboot())
        self.s.addTest(OpTestHeartbeat.HeartbeatSkiroot())
        self.s.addTest(OpTestNVRAM.SkirootNVRAM())
        self.s.addTest(Console.suite())
        # We seem to be hitting a failure :/
        # self.s.addTest(OpTestPNOR.Skiroot())
        self.s.addTest(DeviceTreeValidation.DeviceTreeValidationSkiroot())
        self.s.addTest(unittest.TestLoader().loadTestsFromTestCase(
            OpalSysfsTests.Skiroot))
        # duplicate sensors and partition verification fails
        # self.s.addTest(OpalMsglog.Skiroot())
        self.s.addTest(KernelLog.Skiroot())
        self.s.addTest(DPO.DPOSkiroot())
#        self.s.addTest(OpTestEnergyScale.runtime_suite())

    def suite(self):
        return self.s


class OpTestPerCommitSuite():
    '''Simple Suite to run for CI on per-commit basis'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestExample.skiroot_full_suite())
        self.s.addTest(OpTestExample.host_full_suite())
        self.s.addTest(SystemLogin.system_access_suite())

    def suite(self):
        return self.s


class OpTestPullRequestSuite():
    '''Simple Suite to run for CI on pull-request basis'''

    def __init__(self):
        self.s = unittest.TestSuite()
        self.s.addTest(OpTestExample.skiroot_full_suite())
        self.s.addTest(OpTestExample.host_full_suite())
        self.s.addTest(SystemLogin.system_access_suite())

    def suite(self):
        return self.s


suites = {
    'system-access': SystemAccessSuite(),
    'skiroot': SkirootSuite(),
    'host': HostSuite(),
    'default': DefaultSuite(),
    'qemu': QemuSuite(),
    'mambo': MamboSuite(),
    'BasicIPL': BasicIPLSuite(),
    'BasicPCI': BasicPCISuite(),
    'secure-boot': SBSuite(),
    'trusted-boot': TBSuite(),
    'em':       OpTestEMSuite(),
    'em-host':  OpTestEMHostSuite(),
    'known-bugs': KnownBugs(),
    'experimental': ExperimentalSuite(),
    'experimental-eeh': OpTestEEHSuite(),
    'experimental-energyscale': OpTestEnergyScaleSuite(),
    'standby': StandbySuite(),
    'hmi': HMISuite(),
    'experimental-hmi': ExperimentalHMISuite(),
    'experimental-unrecoverable-hmi': UnrecoverableHMISuite(),
    'experimental-reprovision': ExperimentalReprovisionSuite(),
    'broken-reprovision': BrokenReprovisionSuite(),
    'full-inbandipmi': InbandIPMISuite(),
    'full-outofbandipmi': OutofbandIPMISuite(),
    'rest-api': RestAPISuite(),
    'cronus': CronusSuite(),
    'full-occ': OCCSuite(),
    'capi': CAPISuite(),
    'opencapi': OpenCAPISuite(),
    'crash-suite': CrashSuite(),
    'system-ipl': SystemIPLSuite(),
    'fsp-opal-suite':  FspOpalSuite(),
    'full': FullSuite(),
    'hostboot': OpTestHostbootSuite(),
    'example': OpTestExampleSuite(),
    'pci-regression': OpPCIRegressionSuite(),
    'palmetto-ci': OpenBMCPalmettoSkirootSuite(),
    'per-commit': OpTestPerCommitSuite(),
    'pull-request': OpTestPullRequestSuite(),
}

try:
    # Loop through the addons and load in suites defined there
    for opt in OpTestConfiguration.optAddons:
        suites = OpTestConfiguration.optAddons[opt].addSuites(suites)

    if OpTestConfiguration.conf.args.list_suites:
        print('{0:34}{1}'.format('Test Suite', 'Description'))
        print('{0:34}{1}'.format('----------', '-----------'))
        for key in suites:
            print('{0:34}{1}'.format(key, suites[key].__doc__))
        OpTestConfiguration.conf.util.cleanup()
        exit(0)

    importlib.reload(sys)

    t = unittest.TestSuite()

    if OpTestConfiguration.conf.args.run_suite:
        for suite in OpTestConfiguration.conf.args.run_suite:
            try:
                t.addTest(suites[suite].suite())
            except Exception as e:
                optestlog.error("We encountered a problem adding suite={},"
                                " see if its a valid suite name and retry"
                                .format(suite))
                raise e

    if OpTestConfiguration.conf.args.run:
        t.addTest(unittest.TestLoader().loadTestsFromNames(
            OpTestConfiguration.conf.args.run))

    if not OpTestConfiguration.conf.args.run_suite and not OpTestConfiguration.conf.args.run:
        if not OpTestConfiguration.conf.args.only_flash:
            t.addTest(suites['default'].suite())

    if OpTestConfiguration.conf.args.list_tests:
        print('{0:40}'.format('Test'))
        print('{0:40}'.format('----------'))

        def print_suite(s):
            for test in s:
                if isinstance(test, unittest.TestSuite):
                    print_suite(test)
                else:
                    test_id = test.id()
                    print('{0:40}'.format(test_id))

        print_suite(t)
        OpTestConfiguration.conf.util.cleanup()
        exit(0)

    xml_msg = ""
except Exception as e:
    traceback.print_exc()
    optestlog.error("Exit unexpectedly with Exception={}".format(e))
    OpTestConfiguration.conf.util.cleanup()
    exit(-1)


def run_tests(t, failfast):
    runner = unittest.TextTestRunner
    kwargs = {
        'verbosity': 2,
        'failfast': failfast,
    }
    try:
        import xmlrunner  # requires unittest-xml-reporting package
        runner = xmlrunner.XMLTestRunner
        kwargs.update({
            'output': OpTestConfiguration.conf.output,
            'outsuffix': OpTestConfiguration.conf.outsuffix,
            'stream': OpTestConfiguration.conf.logfile,
        })

        # xmlrunner pre v1.13 doesn't support failfast
        try:
            runner(failfast=True)
        except TypeError:
            if failfast:
                optestlog.info("Ignoring failfast: not supported by runner")
            del kwargs['failfast']
    except ImportError:
        pass

    return runner(**kwargs).run(t)


exit_code = 0
try:
    res = None
    optestlog.debug("op-test argv={}".format(sys.argv))

    if not OpTestConfiguration.conf.args.noflash:
        if any(s in OpTestConfiguration.conf.args.bmc_type for s in ("QEMU", "qemu")):
            optestlog.info("Not adding FlashFirmware suite for QEMU machine")
        else:
            res = run_tests(FlashFirmware().suite(),
                            failfast=OpTestConfiguration.conf.args.failfast)

    if OpTestConfiguration.conf.args.only_flash:
        if res is not None:
            optestlog.info('Exit with Result errors="{}" and failures="{}" from only_flash'.format(
                len(res.errors), len(res.failures)))
            OpTestConfiguration.conf.util.cleanup()
            exit_code = len(res.errors + res.failures)
            sys.exit(exit_code)
        else:
            OpTestConfiguration.conf.util.cleanup()
            sys.exit(exit_code)

    if not res or (res and not (res.errors or res.failures)):
        res = run_tests(t, failfast=OpTestConfiguration.conf.args.failfast)
    else:
        optestlog.error('Skipping main tests as flashing failed')
        OpTestConfiguration.conf.util.cleanup()
        exit_code = -1
        sys.exit(exit_code)

    # delay here to allow test results to dump first
    time.sleep(2)
    optestlog.info('Exit with Result errors="{}" and failures="{}"'.format(
        len(res.errors), len(res.failures)))

    OpTestConfiguration.conf.util.cleanup()
    if len(res.errors) == 0:
        optestlog.debug("Exit Results with No Errors")
    if len(res.failures) == 0:
        optestlog.debug("Exit Results with No Failures")
    for i, elem in enumerate(res.errors):
        optestlog.debug("Exit Results Error #{}={}".format(i+1, elem))
    for i, elem in enumerate(res.failures):
        optestlog.debug("Exit Results Failure #{}={}".format(i+1, elem))
    if res:
        exit_code = len(res.errors + res.failures)
    else:
        exit_code = -1  # somethings wrong
    sys.exit(exit_code)
except Exception as e:
    traceback.print_exc()
    optestlog.error("Exit unexpectedly with Exception={}".format(e))
    OpTestConfiguration.conf.util.cleanup()
    exit_code = -1
    sys.exit(exit_code)
finally:
    # Create a softlink to `latest` test results
    output = OpTestConfiguration.conf.logdir
    if not os.path.exists(output):
        optestlog.error("Test result folder not found")
        sys.exit(exit_code)
    basedir = os.path.abspath(os.path.join(output, os.pardir))
    latest = os.path.join(basedir, "latest")
    if os.path.exists(latest):
        if os.path.islink(latest):
            os.unlink(latest)
    try:
        os.symlink(output, latest)
    except OSError:
        optestlog.error('Unable to create test result latest softlink')
    sys.exit(exit_code)