Skip to content

Commit

Permalink
Merge branch 'openhwgroup:main' into main
Browse files Browse the repository at this point in the history
  • Loading branch information
@rosethompson
rosethompson authored Oct 29, 2024
2 parents 896bb21 + 55cffcb commit c93cadc
Show file tree
Hide file tree
Showing 19 changed files with 644 additions and 151 deletions.
7 changes: 7 additions & 0 deletions .github/workflows/install.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ jobs:
os: ubuntu-latest
image: null
riscv_path: /home/riscv
# Custom location user level installation
- name: custom-user-install
os: ubuntu-latest
image: null
user: true
riscv_path: $HOME/riscv-toolchain

# run on selected version of ubuntu or on ubuntu-latest with docker image
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -108,6 +114,7 @@ jobs:
fi
# Set environment variables for the rest of the job
- name: Set Environment Variables
if: always()
run: |
if [ ! -z ${{ matrix.riscv_path }} ]; then
sed -i 's,exit 1,export RISCV=${{ matrix.riscv_path }},g' setup.sh
Expand Down
2 changes: 1 addition & 1 deletion README.md
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
![Installation CI](https://github.com/jordancarlin/cvw/actions/workflows/install.yml/badge.svg?branch=main)
![Installation CI](https://github.com/openhwgroup/cvw/actions/workflows/install.yml/badge.svg?branch=main)

# core-v-wally

Expand Down
98 changes: 59 additions & 39 deletions bin/nightly_build.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,20 @@
class FolderManager:
"""A class for managing folders and repository cloning."""

def __init__(self):
def __init__(self, basedir):
"""
Initialize the FolderManager instance.
Args:
base_dir (str): The base directory where folders will be managed and repository will be cloned.
"""
env_extract_var = 'WALLY'
self.base_dir = os.environ.get(env_extract_var)
self.base_parent_dir = os.path.dirname(self.base_dir)
if os.environ.get(env_extract_var):
self.base_dir = os.environ.get(env_extract_var)
self.base_parent_dir = os.path.dirname(self.base_dir)
else:
self.base_dir = basedir
self.base_parent_dir = self.base_dir

# logger.info(f"Base directory: {self.base_dir}")
# logger.info(f"Parent Base directory: {self.base_parent_dir}")
Expand Down Expand Up @@ -313,7 +317,7 @@ def execute_makefile(self, makefile_path=None, target=None):
self.logger.error(f"Error making the tests. Target: {target}")
return False

def run_tests(self, test_type=None, test_name=None, test_extension=None):
def run_tests(self, test_type=None, test_name=None, test_extensions=None):
"""
Run a script through the terminal and save the output to a file.
Expand All @@ -329,12 +333,12 @@ def run_tests(self, test_type=None, test_name=None, test_extension=None):
output_file = self.log_dir.joinpath(f"{test_name}-output.log")
os.chdir(self.sim_dir)

if test_extension:
command = [test_type, test_name, test_extension]
self.logger.info(f"Command used to run tests: {test_type} {test_name} {test_extension}")
if test_extensions:
command = [test_type, test_name] + test_extensions
self.logger.info(f"Command used to run tests in directory {self.sim_dir}: {test_type} {test_name} {' '.join(test_extensions)}")
else:
command = [test_type, test_name]
self.logger.info(f"Command used to run tests: {test_type} {test_name}")
self.logger.info(f"Command used to run tests in directory {self.sim_dir}: {test_type} {test_name}")


# Execute the command using subprocess and save the output into a file
Expand All @@ -348,10 +352,10 @@ def run_tests(self, test_type=None, test_name=None, test_extension=None):
self.logger.error("There was an error in running the tests in the run_tests function: {e}")
# Check if the command executed successfuly
if result.returncode or result.returncode == 0:
self.logger.info(f"Test ran successfuly. Test type: {test_type}, test name: {test_name}, test extention: {test_extension}")
self.logger.info(f"Test ran successfuly. Test type: {test_type}, test name: {test_name}, test extension: {' '.join(test_extensions)}")
return True, output_file
else:
self.logger.error(f"Error making test. Test type: {test_type}, test name: {test_name}, test extention: {test_extension}")
self.logger.error(f"Error making test. Test type: {test_type}, test name: {test_name}, test extension: {' '.join(test_extensions)}")
return False, output_file


Expand Down Expand Up @@ -406,22 +410,30 @@ def clean_format_output(self, input_file, output_file=None):
# Remove ANSI escape codes
line = re.sub(r'\x1b\[[0-9;]*[mGK]', '', lines[index])

if "Success" in line:
if "Success" in line: # test succeeds
passed_configs.append(line.split(':')[0].strip())
elif "passed lint" in line:
passed_configs.append(line.split(' ')[0].strip())
passed_configs.append(f"Lint: {line.split(' ')[0].strip()}")
#passed_configs.append(line) # potentially use a space
elif "failed lint" in line:
failed_configs.append(line.split(' ')[0].strip(), "no log file")
failed_configs.append([f"Lint: {line.split(' ')[0].strip()}", "No Log File"])
#failed_configs.append(line)

elif "Failures detected in output" in line:
elif "Failures detected in output" in line: # Text explicitly fails
try:
config_name = line.split(':')[0].strip()
log_file = os.path.abspath("logs/"+config_name+".log")
log_file = os.path.abspath(os.path.join("logs", config_name, ".log"))
failed_configs.append((config_name, log_file))
except:
failed_configs.append((config_name, "Log file not found"))

elif "Timeout" in line: # Test times out
try:
config_name = line.split(':')[0].strip()
log_file = os.path.abspath("logs/"+config_name+".log")
failed_configs.append((f"Timeout: {config_name}", log_file))
except:
failed_configs.append((f"Timeout: {config_name}", "No Log File"))


index += 1
Expand Down Expand Up @@ -535,7 +547,7 @@ def combine_markdown_files(self, passed_tests, failed_tests, test_list, total_nu
md_file.write(f"\n**Total failed tests: {total_number_failures}**")
for (test_item, item) in zip(test_list, failed_tests):
md_file.write(f"\n\n### {test_item[1]} test")
md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {test_item[2]}\n\n")
md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {' '.join(test_item[2])}\n\n")
md_file.write(f"**Failed Tests:**\n")


Expand All @@ -558,7 +570,7 @@ def combine_markdown_files(self, passed_tests, failed_tests, test_list, total_nu
md_file.write(f"\n**Total successful tests: {total_number_success}**")
for (test_item, item) in zip(test_list, passed_tests):
md_file.write(f"\n\n### {test_item[1]} test")
md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {test_item[2]}\n\n")
md_file.write(f"\n**Command used:** {test_item[0]} {test_item[1]} {' '.join(test_item[2])}\n\n")
md_file.write(f"\n**Successful Tests:**\n")


Expand Down Expand Up @@ -619,7 +631,7 @@ def send_email(self, receiver_emails=None, subject="Nightly Regression Test"):

# check if there are any emails
if not receiver_emails:
self.logger.ERROR("No receiver emails provided.")
self.logger.error("No receiver emails provided.")
return

# grab the html file
Expand Down Expand Up @@ -660,7 +672,7 @@ def main():

parser.add_argument('--path',default = "nightly", help='specify the path for where the nightly repositories will be cloned ex: "nightly-runs')
parser.add_argument('--repository',default = "https://github.com/openhwgroup/cvw", help='specify which github repository you want to clone')
parser.add_argument('--target', default = "all", help='types of tests you can make are: all, wally-riscv-arch-test, no')
parser.add_argument('--target', default = "--jobs", help='types of tests you can make are: all, wally-riscv-arch-test, no')
parser.add_argument('--tests', default = "nightly", help='types of tests you can run are: nightly, test, test_lint')
parser.add_argument('--send_email',default = "", nargs="+", help='What emails to send test results to. Example: "[email1],[email2],..."')

Expand All @@ -682,7 +694,7 @@ def main():
log_file_path = log_path.joinpath("nightly_build.log")
previous_cvw_path = Path.home().joinpath(args.path,f"{yesterday}/cvw")
# creates the object
folder_manager = FolderManager()
folder_manager = FolderManager(basedir=args.path)

# setting the path on where to clone new repositories of cvw
folder_manager.create_folders([cvw_path, results_path, log_path])
Expand All @@ -691,14 +703,18 @@ def main():
folder_manager.clone_repository(cvw_path, args.repository)

# Define tests that we can run
if (args.tests == "nightly"):
test_list = [["python", "regression-wally", "--nightly --buildroot"]]
elif (args.tests == "test"):
test_list = [["python", "regression-wally", ""]]
elif (args.tests == "test_lint"):
test_list = [["bash", "lint-wally", "-nightly"]]
#
# flags are a list
if (args.tests == "all"):
test_list = [["python", "./regression-wally", ["--nightly", "--buildroot"]]]
elif (args.tests == "nightly"):
test_list = [["python", "./regression-wally", ["--nightly"]]]
elif (args.tests == "regression"):
test_list = [["python", "./regression-wally", []]]
elif (args.tests == "lint"):
test_list = [["bash", "./lint-wally", ["--nightly"]]]
else:
print(f"Error: Invalid test '"+args.test+"' specified")
print(f"Error: Invalid test {args.tests} specified")
raise SystemExit

#############################################
Expand Down Expand Up @@ -747,12 +763,12 @@ def main():

if args.target != "no":
test_runner.execute_makefile(target = args.target, makefile_path=test_runner.cvw)
if args.target == "all":
# Compile Linux for local testing
test_runner.set_env_var("RISCV",str(test_runner.cvw))
linux_path = test_runner.cvw / "linux"
test_runner.execute_makefile(target = "all_nosudo", makefile_path=linux_path)
test_runner.execute_makefile(target = "dumptvs_nosudo", makefile_path=linux_path)
# TODO: remove vestigial code if no longer wanted
# if args.target == "all":
# # Compile Linux for local testing
# test_runner.set_env_var("RISCV",str(test_runner.cvw))
# linux_path = test_runner.cvw / "linux"
# test_runner.execute_makefile(target = "all", makefile_path=linux_path)

#############################################
# RUN TESTS #
Expand All @@ -766,9 +782,9 @@ def main():
total_failures = []
total_success = []

for test_type, test_name, test_extension in test_list:
for test_type, test_name, test_extensions in test_list:

check, output_location = test_runner.run_tests(test_type=test_type, test_name=test_name, test_extension=test_extension)
check, output_location = test_runner.run_tests(test_type=test_type, test_name=test_name, test_extensions=test_extensions)
try:
if check: # this checks if the test actually ran successfuly
output_log_list.append(output_location)
Expand All @@ -778,7 +794,7 @@ def main():
passed, failed = test_runner.clean_format_output(input_file = output_location)
logger.info(f"{test_name} has been formatted to markdown")
except:
logger.ERROR(f"Error occured with formatting {test_name}")
logger.error(f"Error occured with formatting {test_name}")

logger.info(f"The # of failures are for {test_name}: {len(failed)}")
total_number_failures+= len(failed)
Expand All @@ -788,15 +804,19 @@ def main():
total_number_success += len(passed)
total_success.append(passed)
test_runner.rewrite_to_markdown(test_name, passed, failed)

newlinechar = "\n"
logger.info(f"Failed tests: \n{newlinechar.join([x[0] for x in failed])}")

except Exception as e:
logger.error("There was an error in running the tests: {e}")
logger.error(f"There was an error in running the tests: {e}")

logger.info(f"The total sucesses for all tests ran are: {total_number_success}")
logger.info(f"The total failures for all tests ran are: {total_number_failures}")

# Copy actual test logs from sim/questa, sim/verilator
test_runner.copy_sim_logs([test_runner.cvw / "sim/questa/logs", test_runner.cvw / "sim/verilator/logs"])
# Copy actual test logs from sim/questa, sim/verilator, sim/vcs
if not args.tests == "test_lint":
test_runner.copy_sim_logs([test_runner.cvw / "sim/questa/logs", test_runner.cvw / "sim/verilator/logs", test_runner.cvw / "sim/vcs/logs"])

#############################################
# FORMAT TESTS #
Expand Down
3 changes: 2 additions & 1 deletion bin/regression-wally
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,7 +507,7 @@ def main():
TIMEOUT_DUR = 20*60 # seconds
os.system('rm -f questa/cov/*.ucdb')
elif args.fcov:
TIMEOUT_DUR = 2*60
TIMEOUT_DUR = 8*60
os.system('rm -f questa/fcov_ucdb/* questa/fcov_logs/* questa/fcov/*')
elif args.buildroot:
TIMEOUT_DUR = 60*1440 # 1 day
Expand All @@ -534,6 +534,7 @@ def main():
try:
num_fail+=result.get(timeout=TIMEOUT_DUR)
except TimeoutError:
pool.terminate()
num_fail+=1
print(f"{bcolors.FAIL}%s: Timeout - runtime exceeded %d seconds{bcolors.ENDC}" % (config.cmd, TIMEOUT_DUR))

Expand Down
Loading

0 comments on commit c93cadc

Please sign in to comment.