From 6458d65135ad776236fb3321154d42a6eb6360fa Mon Sep 17 00:00:00 2001 From: ahmedsamyh Date: Wed, 19 Mar 2025 23:09:00 +0500 Subject: [PATCH] Implemented new modular test system. --- test.py | 55 +++++++++++++++--------- test.sh | 8 ++++ tests/.arena.build.code.expected | 1 + tests/.arena.build.err.expected | 0 tests/.arena.build.in.expected | 0 tests/.arena.build.out.expected | 0 tests/.arena.code.expected | 1 + tests/.arena.err.expected | 0 tests/.arena.in.expected | 0 tests/.arena.out.expected | 0 tests/.dynamic_array.build.code.expected | 1 + tests/.dynamic_array.build.err.expected | 0 tests/.dynamic_array.build.in.expected | 0 tests/.dynamic_array.build.out.expected | 0 tests/.dynamic_array.code.expected | 1 + tests/.dynamic_array.err.expected | 1 + tests/.dynamic_array.in.expected | 0 tests/.dynamic_array.out.expected | 51 ++++++++++++++++++++++ tests/.logging.build.code.expected | 1 + tests/.logging.build.err.expected | 0 tests/.logging.build.in.expected | 0 tests/.logging.build.out.expected | 0 tests/.logging.code.expected | 1 + tests/.logging.err.expected | 1 + tests/.logging.in.expected | 0 tests/.logging.out.expected | 2 + tests/.shift_args.build.code.expected | 1 + tests/.shift_args.build.err.expected | 0 tests/.shift_args.build.in.expected | 0 tests/.shift_args.build.out.expected | 0 tests/.shift_args.code.expected | 1 + tests/.shift_args.err.expected | 0 tests/.shift_args.in.expected | 0 tests/.shift_args.out.expected | 4 ++ tests/.slurp_file.build.code.expected | 1 + tests/.slurp_file.build.err.expected | 0 tests/.slurp_file.build.in.expected | 0 tests/.slurp_file.build.out.expected | 0 tests/.slurp_file.code.expected | 1 + tests/.slurp_file.err.expected | 0 tests/.slurp_file.in.expected | 0 tests/.slurp_file.out.expected | 15 +++++++ tests/.string_view.build.code.expected | 1 + tests/.string_view.build.err.expected | 0 tests/.string_view.build.in.expected | 0 tests/.string_view.build.out.expected | 0 tests/.string_view.code.expected | 1 + tests/.string_view.err.expected | 0 tests/.string_view.in.expected | 0 tests/.string_view.out.expected | 6 +++ 50 files changed, 134 insertions(+), 21 deletions(-) create mode 100644 test.sh create mode 100644 tests/.arena.build.code.expected create mode 100644 tests/.arena.build.err.expected create mode 100644 tests/.arena.build.in.expected create mode 100644 tests/.arena.build.out.expected create mode 100644 tests/.arena.code.expected create mode 100644 tests/.arena.err.expected create mode 100644 tests/.arena.in.expected create mode 100644 tests/.arena.out.expected create mode 100644 tests/.dynamic_array.build.code.expected create mode 100644 tests/.dynamic_array.build.err.expected create mode 100644 tests/.dynamic_array.build.in.expected create mode 100644 tests/.dynamic_array.build.out.expected create mode 100644 tests/.dynamic_array.code.expected create mode 100644 tests/.dynamic_array.err.expected create mode 100644 tests/.dynamic_array.in.expected create mode 100644 tests/.dynamic_array.out.expected create mode 100644 tests/.logging.build.code.expected create mode 100644 tests/.logging.build.err.expected create mode 100644 tests/.logging.build.in.expected create mode 100644 tests/.logging.build.out.expected create mode 100644 tests/.logging.code.expected create mode 100644 tests/.logging.err.expected create mode 100644 tests/.logging.in.expected create mode 100644 tests/.logging.out.expected create mode 100644 tests/.shift_args.build.code.expected create mode 100644 tests/.shift_args.build.err.expected create mode 100644 tests/.shift_args.build.in.expected create mode 100644 tests/.shift_args.build.out.expected create mode 100644 tests/.shift_args.code.expected create mode 100644 tests/.shift_args.err.expected create mode 100644 tests/.shift_args.in.expected create mode 100644 tests/.shift_args.out.expected create mode 100644 tests/.slurp_file.build.code.expected create mode 100644 tests/.slurp_file.build.err.expected create mode 100644 tests/.slurp_file.build.in.expected create mode 100644 tests/.slurp_file.build.out.expected create mode 100644 tests/.slurp_file.code.expected create mode 100644 tests/.slurp_file.err.expected create mode 100644 tests/.slurp_file.in.expected create mode 100644 tests/.slurp_file.out.expected create mode 100644 tests/.string_view.build.code.expected create mode 100644 tests/.string_view.build.err.expected create mode 100644 tests/.string_view.build.in.expected create mode 100644 tests/.string_view.build.out.expected create mode 100644 tests/.string_view.code.expected create mode 100644 tests/.string_view.err.expected create mode 100644 tests/.string_view.in.expected create mode 100644 tests/.string_view.out.expected diff --git a/test.py b/test.py index cf9159e..ca4588c 100644 --- a/test.py +++ b/test.py @@ -18,22 +18,22 @@ RUN_CMD="NOT SET" # The name of the directory all the tests live in. (Will chdir to this) TESTS_DIR="NOT SET" # Suffix of the source files the build command iw (if there is a . prefix, it will be removed -SRC_FILE_SUFFIX="NOT SET" +SRC_SUFFIX="NOT SET" def get_env_variables(): - global BUILD_CMD, RUN_CMD, TESTS_DIR, SRC_FILE_SUFFIX + global BUILD_CMD, RUN_CMD, TESTS_DIR, SRC_SUFFIX BUILD_CMD = os.environ["BUILD_CMD"] if "BUILD_CMD" in os.environ else "NOT SET" RUN_CMD = os.environ["RUN_CMD"] if "RUN_CMD" in os.environ else "NOT SET" TESTS_DIR = os.environ["TESTS_DIR"] if "TESTS_DIR" in os.environ else "NOT SET" - SRC_FILE_SUFFIX = os.environ["SRC_FILE_SUFFIX"] if "SRC_FILE_SUFFIX" in os.environ else "NOT SET" - SRC_FILE_SUFFIX = SRC_FILE_SUFFIX.removeprefix(".") - # SRC_FILE_SUFFIX = os.environ["SRC_FILE_SUFFIX"] if "SRC_FILE_SUFFIXsuffix" in os.environ else "NOT SET" + SRC_SUFFIX = os.environ["SRC_SUFFIX"] if "SRC_SUFFIX" in os.environ else "NOT SET" + SRC_SUFFIX = SRC_SUFFIX.removeprefix(".") + # SRC_SUFFIX = os.environ["SRC_SUFFIX"] if "SRC_SUFFIXsuffix" in os.environ else "NOT SET" def get_cmd_substituted(cmd, tests, current_test): assert current_test in tests, "The test that you passed is not in the tests map!" cmd = cmd.replace("{test_name}", tests[current_test].name) - cmd = cmd.replace("{src_suffix}", SRC_FILE_SUFFIX) + cmd = cmd.replace("{src_suffix}", SRC_SUFFIX) return cmd @@ -117,6 +117,13 @@ class Test: build_input_array.pop(i) return build_input_array + def get_stdin_list(self): + input_array = self.stdin.split(sep=' ') + for i in range(len(input_array)-1, -1, -1): + if len(input_array[i]) <= 0: + input_array.pop(i) + return input_array + def usage(program: str): print(f"Usage: {program} [flags]") @@ -188,10 +195,8 @@ def main(): get_env_variables() - check_crucial_envvar(BUILD_CMD, "BUILD_CMD") - check_crucial_envvar(RUN_CMD, "RUN_CMD") check_crucial_envvar(TESTS_DIR, "TESTS_DIR") - check_crucial_envvar(SRC_FILE_SUFFIX, "SRC_FILE_SUFFIX") + check_crucial_envvar(SRC_SUFFIX, "SRC_SUFFIX") os.chdir(TESTS_DIR) vlog(verbose_output, f"[INFO] Changed cwd to {os.getcwd()}") @@ -199,15 +204,15 @@ def main(): tests = {} for e in sorted(os.listdir(os.getcwd())): - if not e.endswith("." + SRC_FILE_SUFFIX): continue - base_name = e.removesuffix("." + SRC_FILE_SUFFIX) + if not e.endswith("." + SRC_SUFFIX): continue + base_name = e.removesuffix("." + SRC_SUFFIX) if not tests.get(base_name): tests[base_name] = Test(base_name) - print(f"BUILD_CMD: {BUILD_CMD}") - print(f"RUN_CMD: {RUN_CMD}") - print(f"TESTS_DIR: {TESTS_DIR}") - print(f"SRC_FILE_SUFFIX: {SRC_FILE_SUFFIX}") + # print(f"BUILD_CMD: {BUILD_CMD}") + # print(f"RUN_CMD: {RUN_CMD}") + # print(f"TESTS_DIR: {TESTS_DIR}") + # print(f"SRC_SUFFIX: {SRC_SUFFIX}") for subcmd in subcmds: total_tests_count = len(tests) @@ -218,6 +223,7 @@ def main(): hhelp() exit(0) elif subcmd == "build": + check_crucial_envvar(BUILD_CMD, "BUILD_CMD") print(f'----- [BUILD] -----') for test_name in tests: print(f'+ Building {test_name} [{current_test_id+1}/{total_tests_count}]...') @@ -231,7 +237,7 @@ def main(): if stop_on_error: exit(1) continue - cmd = [COMPILER, f"{test_name}{SUFFIX}"] + cmd = shlex.split(get_cmd_substituted(BUILD_CMD, tests, test_name)) build_stdin_list = test.get_build_stdin_list() if len(build_stdin_list) > 0: cmd.extend(build_stdin_list) vlog(verbose_output, f"[CMD] {cmd}") @@ -268,7 +274,7 @@ def main(): print(f"Build {passing_tests_count}/{total_tests_count} tests") elif subcmd == "run": - assert False, "'run' Subcommand is not fully tested!" + check_crucial_envvar(RUN_CMD, "RUN_CMD") print(f'----- [RUN] -----') for test_name in tests: print(f'+ Running {test_name} [{current_test_id+1}/{total_tests_count}]...') @@ -277,7 +283,7 @@ def main(): res = None try: - cmd = [f"./{test_name}"] + cmd = shlex.split(get_cmd_substituted(RUN_CMD, tests, test_name)) vlog(verbose_output, f"[CMD] {cmd}") res = subprocess.run(cmd, capture_output = True, text = True) except Exception as e: @@ -300,12 +306,16 @@ def main(): print(f"PASSED {passing_tests_count}/{total_tests_count}") elif subcmd == "record": - assert False, "'record' Subcommand is not fully tested!" + check_crucial_envvar(RUN_CMD, "RUN_CMD") print(f'----- [RECORD] -----') for test_name in tests: print(f"+ Recording expected behaviour for '{test_name}'...") test = tests[test_name] + cmd = shlex.split(get_cmd_substituted(RUN_CMD, tests, test_name)) + stdin_list = test.get_stdin_list() + if len(stdin_list) > 0: cmd.extend(stdin_list) + vlog(verbose_output, f"[CMD] {cmd}") res = subprocess.run([f"./{test_name}"], capture_output = True, text = True) print(f"stdout: {res.stdout}") @@ -324,6 +334,7 @@ def main(): else: print('[SKIP]') elif subcmd == "record_build": + check_crucial_envvar(BUILD_CMD, "BUILD_CMD") print(f'----- [RECORD_BUILD] -----') for test_name in tests: print(f"+ Recording expected build behaviour for '{test_name}'...") @@ -334,11 +345,13 @@ def main(): ans = input("Do you want to change the build_input? [y/N]") if ans.lower() == 'y': test.build_stdin = input("What is the input passed? ") + else: + print("[SKIPPING]...") + continue else: test.build_stdin = input("What is the input passed? ") - - cmd = [COMPILER, f"{test_name}{SUFFIX}"] + cmd = shlex.split(get_cmd_substituted(BUILD_CMD, tests, test_name)) build_stdin_list = test.get_build_stdin_list() if len(build_stdin_list) > 0: cmd.extend(build_stdin_list) vlog(verbose_output, f"[CMD] {cmd}") diff --git a/test.sh b/test.sh new file mode 100644 index 0000000..0c08457 --- /dev/null +++ b/test.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +export SRC_SUFFIX=.c +export TESTS_DIR=./tests +export BUILD_CMD="gcc -o {test_name} {test_name}.{src_suffix}" +export RUN_CMD="./{test_name}" + +./test.py $* diff --git a/tests/.arena.build.code.expected b/tests/.arena.build.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.arena.build.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.arena.build.err.expected b/tests/.arena.build.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.arena.build.in.expected b/tests/.arena.build.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.arena.build.out.expected b/tests/.arena.build.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.arena.code.expected b/tests/.arena.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.arena.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.arena.err.expected b/tests/.arena.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.arena.in.expected b/tests/.arena.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.arena.out.expected b/tests/.arena.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.dynamic_array.build.code.expected b/tests/.dynamic_array.build.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.dynamic_array.build.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.dynamic_array.build.err.expected b/tests/.dynamic_array.build.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.dynamic_array.build.in.expected b/tests/.dynamic_array.build.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.dynamic_array.build.out.expected b/tests/.dynamic_array.build.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.dynamic_array.code.expected b/tests/.dynamic_array.code.expected new file mode 100644 index 0000000..56a6051 --- /dev/null +++ b/tests/.dynamic_array.code.expected @@ -0,0 +1 @@ +1 \ No newline at end of file diff --git a/tests/.dynamic_array.err.expected b/tests/.dynamic_array.err.expected new file mode 100644 index 0000000..4a65ef8 --- /dev/null +++ b/tests/.dynamic_array.err.expected @@ -0,0 +1 @@ +dynamic_array.c:27:0 [ASSERTION FAILED] a == 6: da_shift did not return correct value \ No newline at end of file diff --git a/tests/.dynamic_array.in.expected b/tests/.dynamic_array.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.dynamic_array.out.expected b/tests/.dynamic_array.out.expected new file mode 100644 index 0000000..c242289 --- /dev/null +++ b/tests/.dynamic_array.out.expected @@ -0,0 +1,51 @@ +[INFO] Even numbers: +[INFO] 0 +[INFO] 2 +[INFO] 4 +[INFO] 6 +[INFO] 8 +[INFO] 10 +[INFO] 12 +[INFO] 14 +[INFO] 16 +[INFO] 18 +[INFO] 20 +[INFO] 22 +[INFO] 24 +[INFO] 26 +[INFO] 28 +[INFO] 30 +[INFO] 32 +[INFO] 34 +[INFO] 36 +[INFO] 38 +[INFO] 40 +[INFO] 42 +[INFO] 44 +[INFO] 46 +[INFO] 48 +[INFO] 50 +[INFO] 52 +[INFO] 54 +[INFO] 56 +[INFO] 58 +[INFO] 60 +[INFO] 62 +[INFO] 64 +[INFO] 66 +[INFO] 68 +[INFO] 70 +[INFO] 72 +[INFO] 74 +[INFO] 76 +[INFO] 78 +[INFO] 80 +[INFO] 82 +[INFO] 84 +[INFO] 86 +[INFO] 88 +[INFO] 90 +[INFO] 92 +[INFO] 94 +[INFO] 96 +[INFO] 98 diff --git a/tests/.logging.build.code.expected b/tests/.logging.build.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.logging.build.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.logging.build.err.expected b/tests/.logging.build.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.logging.build.in.expected b/tests/.logging.build.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.logging.build.out.expected b/tests/.logging.build.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.logging.code.expected b/tests/.logging.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.logging.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.logging.err.expected b/tests/.logging.err.expected new file mode 100644 index 0000000..420d108 --- /dev/null +++ b/tests/.logging.err.expected @@ -0,0 +1 @@ +[ERROR] This one is an error log diff --git a/tests/.logging.in.expected b/tests/.logging.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.logging.out.expected b/tests/.logging.out.expected new file mode 100644 index 0000000..e3a0d36 --- /dev/null +++ b/tests/.logging.out.expected @@ -0,0 +1,2 @@ +[INFO] This is an info log +[WARNING] And this is a warning log diff --git a/tests/.shift_args.build.code.expected b/tests/.shift_args.build.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.shift_args.build.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.shift_args.build.err.expected b/tests/.shift_args.build.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.shift_args.build.in.expected b/tests/.shift_args.build.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.shift_args.build.out.expected b/tests/.shift_args.build.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.shift_args.code.expected b/tests/.shift_args.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.shift_args.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.shift_args.err.expected b/tests/.shift_args.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.shift_args.in.expected b/tests/.shift_args.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.shift_args.out.expected b/tests/.shift_args.out.expected new file mode 100644 index 0000000..3809763 --- /dev/null +++ b/tests/.shift_args.out.expected @@ -0,0 +1,4 @@ +program: ./shift_args +After: +argc: 0 +argv: (null) diff --git a/tests/.slurp_file.build.code.expected b/tests/.slurp_file.build.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.slurp_file.build.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.slurp_file.build.err.expected b/tests/.slurp_file.build.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.slurp_file.build.in.expected b/tests/.slurp_file.build.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.slurp_file.build.out.expected b/tests/.slurp_file.build.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.slurp_file.code.expected b/tests/.slurp_file.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.slurp_file.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.slurp_file.err.expected b/tests/.slurp_file.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.slurp_file.in.expected b/tests/.slurp_file.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.slurp_file.out.expected b/tests/.slurp_file.out.expected new file mode 100644 index 0000000..1483648 --- /dev/null +++ b/tests/.slurp_file.out.expected @@ -0,0 +1,15 @@ +[INFO] buff: '#define COMMONLIB_IMPLEMENTATION +#include "../commonlib.h" + +int main(void) { + bool success = false; + const char *buff = c_slurp_file(__FILE__, &success); + + if (!success) { return 1; } + + c_log_info("buff: '%s'", buff); + + // We don't care about leaking memory since we are just exiting right away! + return 0; +} +' diff --git a/tests/.string_view.build.code.expected b/tests/.string_view.build.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.string_view.build.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.string_view.build.err.expected b/tests/.string_view.build.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.string_view.build.in.expected b/tests/.string_view.build.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.string_view.build.out.expected b/tests/.string_view.build.out.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.string_view.code.expected b/tests/.string_view.code.expected new file mode 100644 index 0000000..c227083 --- /dev/null +++ b/tests/.string_view.code.expected @@ -0,0 +1 @@ +0 \ No newline at end of file diff --git a/tests/.string_view.err.expected b/tests/.string_view.err.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.string_view.in.expected b/tests/.string_view.in.expected new file mode 100644 index 0000000..e69de29 diff --git a/tests/.string_view.out.expected b/tests/.string_view.out.expected new file mode 100644 index 0000000..6c289a1 --- /dev/null +++ b/tests/.string_view.out.expected @@ -0,0 +1,6 @@ +[INFO] main: 'World!' +[INFO] a: 'Hello,' +[INFO] b: 'World!' +[INFO] mc: '/*Blah Blah +Blah Blah2 +THis is the end' -- 2.39.5