1 2import sys 3import argparse 4import yaml 5import os 6from datetime import date 7import time 8 9github_yaml = "" 10workspace = "" 11nemu_home = "" 12am_home = "" 13head_sha = "" 14wave_home = "" 15perf_home = "" 16set_env = "" 17sh_path = "" 18 19def remove_empty(list_str): 20 list_new = [] 21 for s in list_str: 22 if not (s == ""): 23 list_new.append(s) 24 return list_new 25 26def split_cmd(cmd): 27 c = cmd.replace("\\\n", " ") 28 c = remove_empty(c.split("\n")) 29 cs = [] 30 for ci in c: 31 ci = ci.replace("\\\n", " ") 32 ci = ci.replace("\n\n", "\n") 33 ci = ci.replace(" ", " ") 34 if ci[-1] == "\n": 35 ci = ci[:-1] 36 cs.append(ci) 37 return cs 38 39def parse_yaml(yaml_file): 40 file_data = open(yaml_file, "r", encoding="utf-8").read() 41 yaml_data = yaml.load(file_data, Loader=yaml.CLoader) 42 return yaml_data 43 44def show(test_info): 45 name = test_info["name"] 46 print(name) 47 48def run_test(test_info, numa, run_mode): 49 name = test_info["name"] 50 coe_key = "continue-on-error" 51 to_key = "timeout-minutes" 52 s_key = "steps" 53 continue_on_error = True if (coe_key not in test_info.keys()) else (False if (test_info[coe_key] == "false") else True) 54 timeout_minutes = 9999 if (to_key not in test_info.keys()) else int(test_info[to_key]) 55 if s_key not in test_info.keys(): 56 print(name, " ", s_key, " not found in yaml, skip") 57 sys.exit() 58 steps_raw = test_info[s_key] 59 # print("Steps") 60 # print(steps) 61 steps = {} 62 for s in steps_raw: 63 if "name" in s.keys(): 64 steps[s["name"]] = s["run"] 65 # print(steps) 66 67 replace_list = [ 68 ["--numa", "--numa" if numa else ""], 69 ["$GITHUB_WORKSPACE", f"{workspace}"], 70 ["$HEAD_SHA", f"{head_sha}"], 71 ["$PERF_HOME", f"{perf_home}"], 72 ["$WAVE_HOME", f"{wave_home}"], 73 ["$AM_HOME", f"{am_home}"] 74 ] 75 76 steps.pop("set env") 77 78 for s in steps.keys(): 79 for r in replace_list: 80 steps[s] = steps[s].replace(r[0], r[1]) 81 82 cmd = [] 83 for s in steps.keys(): 84 cs = split_cmd(steps[s]) 85 cmd = cmd + cs 86 87 if run_mode: 88 for c in cmd: 89 f_c = set_env + " " + c 90 print(f"[CMD] {f_c}", flush=True) 91 os.system(f_c) 92 else: 93 if (sh_path is None): 94 print("sh_path is None") 95 sys.exit() 96 elif (not os.path.exists(sh_path)): 97 os.mkdir(sh_path) 98 99 100 sh_file_name = os.path.join(sh_path, "_".join(name.strip().replace("-", "").split())+".sh") 101 with open(sh_file_name, "w") as tmp_sh: 102 tmp_sh.write(f"mkdir -p {wave_home}\n") 103 tmp_sh.write(f"mkdir -p {perf_home}\n") 104 for c in cmd: 105 print(c) 106 tmp_sh.write(c+"\n") 107 108if __name__ == "__main__": 109 # Usage: 110 # 1. run ci test 111 # python3 scripts/local_ci.py --xs-path $(pwd) --run 112 # More Params: 113 # --pick-test MC: only run 'EMU - MC' 114 115 # 2. print ci test name 116 # python3 scripts/local_ci.py --xs-path $(pwd) --show-test 117 # This can also use --pick-test 118 119 # 3. print ci test command into splited sh files. Run the sh manualy. 120 # python3 scripts/local_ci.py --xs-path $(pwd) --sh-path /nfs/home/zhangzifei/work/xs-master/ci-sh 121 # just remove --run 122 123 # Other Params: 124 # --numa: use numa ctrl, require eypc 125 # --head-sha: magic word, default is today's date 126 # --nemu-home/--am-home: don't know if it is used 127 128 parser = argparse.ArgumentParser(description="run ci locally") 129 parser.add_argument("--xs-path", type=str, help="XiangShan, NOOP_HOME") 130 parser.add_argument("--nemu-home", type=str, help="NEMU_HOME") 131 parser.add_argument("--am-home", type=str, help="AM_HOME") 132 parser.add_argument("--sh-path", type=str, help="ci's sh file path") 133 parser.add_argument("--head-sha", type=str, help="magic word") 134 parser.add_argument("--run", action='store_true', help="run test, not gen sh") 135 parser.add_argument("--numa", action='store_true', help="epyc numactl") 136 parser.add_argument("--show-test", action="store_true", help="print test case") 137 parser.add_argument("--pick-test", type=str, help="pick only one test") 138 139 args = parser.parse_args() 140 141 print(args) 142 143 workspace = os.getenv("NOOP_HOME") if (args.xs_path is None) else args.xs_path 144 head_sha = date.today().strftime(r"%Y%m%d") if (args.head_sha is None) else args.head_sha 145 wave_home = os.path.join(workspace, "wave", head_sha) 146 perf_home = os.path.join(workspace, "perf", head_sha) 147 github_yaml = os.path.join(workspace, ".github/workflows/emu.yml") 148 nemu_home = os.getenv("NEMU_HOME") if (args.nemu_home is None) else args.nemu_home 149 am_home = os.getenv("AM_HOME") if (args.am_home is None) else args.am_home 150 set_env = f"NEMU_HOME={nemu_home} NOOP_HOME={workspace} WAVE_HOME={wave_home} PERF_HOME={perf_home} AM_HOME={am_home}" 151 sh_path = f"{workspace}/ci-sh" if (args.sh_path is None) else args.sh_path 152 153 print("workspace(NOOP_HOME): ", workspace) 154 print("head_sha: ", head_sha) 155 print("wave_home: ", wave_home) 156 print("perf_home: ", perf_home) 157 print("github_yaml: ", github_yaml) 158 print("nemu_home: ", nemu_home) 159 print("am_home: ", am_home) 160 print("set_env: ", set_env) 161 print("sh_path", sh_path) 162 163 input("Press Enter to continue") 164 165 ci_tests = parse_yaml(github_yaml)["jobs"] 166 167 if (args.show_test): 168 for test in ci_tests.keys(): 169 show(ci_tests[test]) 170 else: 171 for test in ci_tests.keys(): 172 if args.pick_test is not None: 173 if (args.pick_test in ci_tests[test]["name"]): 174 run_test(ci_tests[test], args.numa, args.run) 175 else: 176 run_test(ci_tests[test], args.numa, args.run)