diff options
Diffstat (limited to 'salis.py')
| -rwxr-xr-x | salis.py | 745 |
1 files changed, 300 insertions, 445 deletions
@@ -1,42 +1,35 @@ #!/usr/bin/env -S PYTHONDONTWRITEBYTECODE=1 python -import contextlib -import ctypes +import colorlog +import enum import json import os import random +import runpy import shutil -import sqlite3 +import socket import subprocess import sys -import urllib.parse -import urllib.request from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, ArgumentTypeError, RawTextHelpFormatter -from http.server import ThreadingHTTPServer, SimpleHTTPRequestHandler from tempfile import TemporaryDirectory +from types import SimpleNamespace # ------------------------------------------------------------------------------ -# Parse CLI arguments +# Argument parser # ------------------------------------------------------------------------------ description = "Salis: Simple A-Life Simulator" prog = sys.argv[0] -epilog = f"Use '-h' to list arguments for each command.\nExample: '{prog} bench -h'" +epilog = f"Use '-h' to show command arguments; e.g. '{prog} new -h'" -main_parser = ArgumentParser( - description=description, - epilog=epilog, - formatter_class=RawTextHelpFormatter, - prog=prog, -) +parser = ArgumentParser(description=description, epilog=epilog, formatter_class=RawTextHelpFormatter, prog=prog) +sub_parsers = parser.add_subparsers(dest="command", required=True) +formatter_class = lambda prog: ArgumentDefaultsHelpFormatter(max_help_position=32, prog=prog) -parsers = main_parser.add_subparsers(dest="command", required=True) -formatter_class = lambda prog: ArgumentDefaultsHelpFormatter(prog, max_help_position=32) - -bench = parsers.add_parser("bench", formatter_class=formatter_class, help="run benchmark") -load = parsers.add_parser("load", formatter_class=formatter_class, help="load saved simulation") -new = parsers.add_parser("new", formatter_class=formatter_class, help="create new simulation") -serve = parsers.add_parser("serve", formatter_class=formatter_class, help="run data server") +new = sub_parsers.add_parser("new", formatter_class=formatter_class, help="create new simulation") +load = sub_parsers.add_parser("load", formatter_class=formatter_class, help="load saved simulation") +server = sub_parsers.add_parser("server", formatter_class=formatter_class, help="run data server") +client = sub_parsers.add_parser("client", formatter_class=formatter_class, help="run data client") architectures = os.listdir("./arch") uis = os.listdir("./ui") @@ -46,526 +39,388 @@ def seed(i): if ival < -1: raise ArgumentTypeError("invalid seed value") return ival -def ipos(i): +def pos(i): ival = int(i, 0) if ival < 0: raise ArgumentTypeError("value must be positive integer") return ival -def inat(i): +def nat(i): ival = int(i, 0) if ival < 1: raise ArgumentTypeError("value must be greater than zero") return ival -def iport(i): +def port(i): ival = int(i, 0) if not 0 <= ival <= 65535: raise ArgumentTypeError("value must be valid port number") return ival -option_keys = ["short", "long", "metavar", "help", "default", "required", "type", "parsers"] -option_list = [ - ["A", "anc", "ANC", "ancestor file name without extension, to be compiled on all cores (ANC points to 'anc/{arch}/{ANC}.asm')", None, True, str, [bench, new]], - ["a", "arch", architectures, "VM architecture", "dummy", False, str, [bench, new]], - ["b", "steps", "N", "number of steps to run in benchmark", 0x1000000, False, ipos, [bench]], - ["C", "clones", "N", "number of ancestor clones on each core", 1, False, inat, [bench, new]], - ["c", "cores", "N", "number of simulator cores", 2, False, inat, [bench, new]], - ["d", "data-push-pow", "POW", "data aggregation interval exponent (interval == 2^{POW} >= {sync-pow}); a value of 0 disables data aggregation (requires 'sqlite')", 28, False, ipos, [new]], - ["f", "force", None, "overwrite existing simulation of given name", False, False, bool, [new]], - ["F", "muta-flip", None, "cosmic rays flip bits instead of randomizing whole bytes", False, False, bool, [bench, new]], - ["g", "compiler", "CC", "C compiler to use", "gcc", False, str, [bench, load, new, serve]], - ["G", "compiler-flags", "FLAGS", "base set of flags to pass to C compiler", "-Wall -Wextra -Werror -pedantic", False, str, [bench, load, new, serve]], - ["M", "muta-pow", "POW", "mutator range exponent (range == 2^{POW})", 32, False, ipos, [bench, new]], - ["m", "mvec-pow", "POW", "memory vector size exponent (size == 2^{POW})", 20, False, ipos, [bench, new]], - ["n", "name", "NAME", "name of new or loaded simulation", "def.sim", False, str, [load, new, serve]], - ["o", "optimized", None, "build with optimizations", False, False, bool, [bench, load, new, serve]], - ["P", "port", "PORT", "port number for data server", 8080, False, iport, [serve]], - ["p", "pre-cmd", "CMD", "shell command to wrap call to executable (e.g. gdb, time, valgrind, etc.)", None, False, str, [bench, load, new]], - ["s", "seed", "SEED", "seed value for new simulation; a value of 0 disables cosmic rays; a value of -1 creates a random seed", 0, False, seed, [bench, new]], - ["T", "keep-temp-dir", None, "delete temporary directory on exit", False, False, bool, [bench, load, new, serve]], - ["t", "thread-gap", "N", "memory gap between cores in bytes (may help reduce cache misses)", 0x100, False, inat, [bench, load, new, serve]], - ["u", "ui", uis, "user interface", "curses", False, str, [load, new]], - ["U", "update", None, "update vendors (call 'git commit' afterwards to track updated files, if any)", False, False, bool, [serve]], - ["x", "no-compress", None, "do not compress save files (useful if 'zlib' is unavailable)", True, False, bool, [new]], - ["y", "sync-pow", "POW", "core sync interval exponent (interval == 2^{POW})", 20, False, ipos, [bench, new]], - ["z", "auto-save-pow", "POW", "auto-save interval exponent (interval == 2^{POW})", 36, False, ipos, [new]], -] - -options = list(map(lambda option: dict(zip(option_keys, option)), option_list)) -parser_map = ((parser, option) for option in options for parser in option["parsers"]) - -for parser, option in parser_map: - arg_kwargs = {} - - def push_same(key): arg_kwargs[key] = option[key] - def push_diff(tgt_key, src_key): arg_kwargs[tgt_key] = option[src_key] - def push_val(key, val): arg_kwargs[key] = val - - push_same("help") - push_same("required") - - if option["metavar"] is None: - push_val("action", "store_true") - else: - push_same("default") - push_same("type") +fmt_id = lambda val: val +fmt_hex = lambda val: hex(int(val, 0)) if type(val) == str else hex(val) - if type(option["metavar"]) is list: push_diff("choices", "metavar") - if type(option["metavar"]) is str: push_same("metavar") +options = { + (("A", "anc"), (new,), fmt_id): {"metavar": "ANC", "help": "ancestor file name without extension; to be compiled on all cores; ANC points to 'anc/{arch}/{ANC}.asm'", "required": True, "type": str}, + (("a", "arch"), (new,), fmt_id): {"choices": architectures, "help": "VM architecture", "default": "dummy", "required": False, "type": str}, + (("C", "clones"), (new,), fmt_id): {"metavar": "N", "help": "number of ancestor clones on each core", "default": 1, "required": False, "type": nat}, + (("c", "cores"), (new,), fmt_id): {"metavar": "N", "help": "number of simulator cores", "default": 2, "required": False, "type": nat}, + (("d", "data-push-pow"), (new,), fmt_id): {"metavar": "POW", "help": "data aggregation interval exponent; interval = 2^{POW} >= {sync-pow}; a value of 0 disables data aggregation; requires 'sqlite' and 'zlib'", "default": 28, "required": False, "type": pos}, + (("F", "muta-flip"), (new,), fmt_id): {"action": "store_true", "help": "cosmic rays flip bits instead of randomizing whole bytes", "required": False}, + (("f", "force"), (new,), fmt_id): {"action": "store_true", "help": "overwrite existing simulation of given name", "required": False}, + (("G", "compiler-flags"), (new, load, server, client), fmt_id): {"metavar": "FLAGS", "help": "base set of flags to pass to C compiler", "default": "-Wall -Wextra -Werror -pedantic", "required": False, "type": str}, + (("g", "compiler"), (new, load, server, client), fmt_id): {"metavar": "CC", "help": "C compiler to use", "default": "gcc", "required": False, "type": str}, + (("H", "home"), (new, load, server), fmt_id): {"metavar": "PATH", "help": "salis home directory", "default": os.path.join(os.environ["HOME"], ".salis"), "required": False, "type": str}, + (("M", "muta-pow"), (new,), fmt_id): {"metavar": "POW", "help": "mutator range exponent; each step a cosmic ray hits addr, where addr = rand_uint64() %% 2^{POW}; lower values of POW mean higher mutation rates", "default": 32, "required": False, "type": pos}, + (("i", "ip"), (client,), fmt_id): {"metavar": "IP", "help": "ip address of server", "default": "127.0.0.1", "required": False, "type": str}, + (("m", "mvec-pow"), (new,), fmt_id): {"metavar": "POW", "help": "memory core size exponent; size = 2^{POW}", "default": 20, "required": False, "type": pos}, + (("n", "name"), (new, load, server), fmt_id): {"metavar": "NAME", "help": "name of new or loaded simulation", "default": "def.sim", "required": False, "type": str}, + (("o", "optimized"), (new, load, server, client), fmt_id): {"action": "store_true", "help": "build with optimizations", "required": False}, + (("P", "port"), (server, client), fmt_id): {"metavar": "PORT", "help": "port number for data server", "default": 8080, "required": False, "type": port}, + (("p", "pre-cmd"), (new, load, server, client), fmt_id): {"metavar": "CMD", "help": "shell command with which to wrap call to executable; e.g. gdb, time, valgrind, etc.", "required": False, "type": str}, + (("s", "seed"), (new,), fmt_hex): {"metavar": "SEED", "help": "seed value for new simulation; a value of 0 disables cosmic rays; a value of -1 creates a random seed", "default": 0, "required": False, "type": seed}, + (("T", "keep-temp-dir"), (new, load, server, client), fmt_id): {"action": "store_true", "help": "keep temporary directory on exit", "required": False}, + (("t", "thread-gap"), (new, load), fmt_hex): {"metavar": "N", "help": "memory gap between core elements in bytes; may help reduce cache misses", "default": 0x100, "required": False, "type": nat}, + (("u", "ui"), (new, load), fmt_id): {"choices": uis, "help": "user interface", "default": "curses", "required": False, "type": str}, + (("x", "no-compress"), (new,), fmt_id): {"action": "store_true", "help": "do not compress save files; useful if 'zlib' is unavailable", "required": False}, + (("y", "sync-pow"), (new,), fmt_id): {"metavar": "POW", "help": "core sync interval exponent; sync events occur every N steps, where N = 2^{POW}", "default": 20, "required": False, "type": pos}, + (("z", "auto-save-pow"), (new,), fmt_id): {"metavar": "POW", "help": "auto-save interval exponent; auto-saves occur every N steps, where N = 2^{POW}", "default": 36, "required": False, "type": pos}, +} - parser.add_argument(f"-{option["short"]}", f"--{option["long"]}", **arg_kwargs) +for (name, sub_parsers, _), kwargs in options.items(): + for sub_parser in sub_parsers: + sub_parser.add_argument(f"-{name[0]}", f"--{name[1]}", **kwargs) -args = main_parser.parse_args() +args = parser.parse_args() # ------------------------------------------------------------------------------ -# Bootstrap logging system +# Logging # ------------------------------------------------------------------------------ -tempdir = TemporaryDirectory(prefix="salis_", delete=not args.keep_temp_dir) -logger_flags = set() -logger_includes = set() -logger_defines = set() - -logger_flags.update({*args.compiler_flags.split(), "-shared", "-fPIC"}) -logger_includes.update({"assert.h", "stdarg.h", "stdbool.h", "stdio.h", "stdlib.h", "time.h", "unistd.h"}) - -if args.optimized: - logger_flags.add("-O3") - logger_defines.add("-DNDEBUG") -else: - logger_flags.add("-ggdb") +handler = colorlog.StreamHandler() +handler.setFormatter(colorlog.ColoredFormatter("%(log_color)s%(asctime)s %(process)07d [%(levelname).4s] %(reset)s%(message)s")) -logger_so = os.path.join(tempdir.name, "log.so") -logger_build_cmd = [args.compiler, "core/logger.c", "-o", logger_so] -logger_build_cmd.extend(logger_flags) -logger_build_cmd.extend(sum(map(lambda include: ["-include", include], logger_includes), [])) -logger_build_cmd.extend(logger_defines) -subprocess.run(logger_build_cmd, check=True) - -logger_dll = ctypes.CDLL(logger_so) -def info(msg): logger_dll.log_info(msg.encode()) -def warn(msg): logger_dll.log_warn(msg.encode()) -def erro(msg): logger_dll.log_erro(msg.encode()) and sys.exit(1) +log = colorlog.getLogger() +log.setLevel(colorlog.INFO) +log.addHandler(handler) # ------------------------------------------------------------------------------ -# Load configuration +# Build class # ------------------------------------------------------------------------------ -info(description) -info(f"Called '{prog} {args.command}' with the following options: {vars(args)}") -info(f"Using temporary directory: {tempdir.name}") -info(f"Using logging dl: {logger_so}") -info(f"Built with command: {logger_build_cmd}") +class Build: + def __init__(self, path, library=False): + self.tempdir = TemporaryDirectory(prefix="salis_", delete=not args.keep_temp_dir) + log.info(f"Generated temporary directory for C builds at: {self.tempdir.name}") -if args.command in ["load", "new", "serve"]: - sim_dir = os.path.join(os.environ["HOME"], ".salis", args.name) - sim_opts = os.path.join(sim_dir, "opts.py") - sim_path = os.path.join(sim_dir, args.name) + self.name = os.path.splitext(os.path.basename(path))[0] + self.binfile = os.path.join(self.tempdir.name, f"{self.name}{".so" if library else ""}") + self.argsfile = os.path.join(self.tempdir.name, f"{self.name}.arg") -if args.command in ["load", "serve"]: - if not os.path.isdir(sim_dir): - erro(f"No simulation found named: {args.name}") + self.flags = {*args.compiler_flags.split(), *({"-shared", "-fPIC"} if library else set()), *({"-O3"} if args.optimized else {"-ggdb"})} + self.defines = {"-DNDEBUG"} if args.optimized else set() + self.links = set() - sys.path.append(sim_dir) - import opts + self.build_cmd = [args.compiler, f"@{self.argsfile}", path, "-o", self.binfile] + log.info(f"Build class initialized for {"library" if library else "executable"}: {path}") + log.info(f"Compiler flags stored at: {self.argsfile}") - opt_vars = {opt: getattr(opts, opt) for opt in dir(opts) if not opt.startswith("__")} + def build(self): + fmt_nl = lambda line: f"{line}\n" + fmt_define = lambda line: f"{line.replace(" ", "\\ ").replace("\"", "\\\"").replace("'", "\\'")}\n" - for key, val in opt_vars.items(): - setattr(args, key, val) + with open(self.argsfile, "w") as f: + f.writelines(map(fmt_nl, sorted(self.flags))) + f.writelines(map(fmt_define, sorted(self.defines))) + f.writelines(map(fmt_nl, sorted(self.links))) - info(f"Sourced configuration from: '{sim_opts}': {opt_vars}") + log.info(f"Running build command: '{self.build_cmd}'") + subprocess.run(self.build_cmd, check=True) -if args.command in ["new"]: - if args.data_push_pow and args.data_push_pow < args.sync_pow: - erro("Data push power must be equal or greater than thread sync power") + def exec(self): + run_cmd = args.pre_cmd.split() if args.pre_cmd else [] + run_cmd.append(self.binfile) - if os.path.isdir(sim_dir) and args.force: - warn(f"Force flag used! Wiping old simulation at: {sim_dir}") - shutil.rmtree(sim_dir) + log.info(f"Running binary with command: '{" ".join(run_cmd)}'") + proc = subprocess.Popen(run_cmd, stdout=sys.stdout, stderr=sys.stderr) - if os.path.isdir(sim_dir): - erro(f"Simulation directory found at: {sim_dir}") + # When using signals (e.g. SIGTERM), they must be sent to the entire process group + # to make sure both the simulator and the interpreter get shut down. + try: + proc.wait() + except KeyboardInterrupt: + proc.terminate() + proc.wait() - if args.seed == -1: - args.seed = random.getrandbits(64) - info(f"Using random seed: {args.seed}") - - info(f"Creating new simulation directory at: {sim_dir}") - info(f"Creating configuration file at: {sim_opts}") - - os.mkdir(sim_dir) + code = proc.returncode - opts = ( - option["long"].replace("-", "_") - for option in options - if new in option["parsers"] and load not in option["parsers"] - ) - - with open(sim_opts, "w") as file: - for opt in opts: - file.write(f"{opt} = {repr(eval(f"args.{opt}"))}\n") + if code != 0: raise RuntimeError(f"Binary returned code: {code}") # ------------------------------------------------------------------------------ -# Load architecture variables +# Options dict formatter # ------------------------------------------------------------------------------ -arch_path = os.path.join("arch", args.arch) -info(f"Loading architecture variables from: {os.path.join(arch_path, "arch_vars.py")}") -sys.path.append(arch_path) -from arch_vars import ArchVars -arch_vars = ArchVars(args) +fmt_h2u = lambda field: field.replace("-", "_") +fmt_u2h = lambda field: field.replace("_", "-") -# ------------------------------------------------------------------------------ -# Launch data server -# ------------------------------------------------------------------------------ -if args.command in ["serve"]: - if (args.update): - vendors = { - "plotly.min.js": "https://cdnjs.cloudflare.com/ajax/libs/plotly.js/3.1.1/plotly.min.js", - "vue3-sfc-loader": "https://cdn.jsdelivr.net/npm/vue3-sfc-loader", - "vue@latest": "https://unpkg.com/vue@latest", - } +def fmt_field(field, val): + for ((_, name), _, fmt), _ in options.items(): + if name == fmt_u2h(field): + return fmt(val) - info(f"Updating vendors: {vendors}") + return fmt_id(val) - for file, url in vendors.items(): - with urllib.request.urlopen(url) as response: - with open(os.path.join("data/vendor", file), "wb") as f: - f.write(response.read()) +def fmt_opts(opts): + opts_out = {} - # Build SQLite event-array render extension - sqlx_flags = set() - sqlx_defines = set() - sqlx_links = set() + for field, val in opts.items(): + opts_out[field] = fmt_field(field, val) - sqlx_flags.update({*args.compiler_flags.split(), "-shared", "-fPIC", "-Icore"}) - sqlx_defines.add(f"-DMVEC_SIZE={2 ** args.mvec_pow}ul") - sqlx_defines.add(f"-DTHREAD_GAP={args.thread_gap}") + return json.dumps(opts_out, indent=2) - if arch_vars.mvec_loop: sqlx_defines.add("-DMVEC_LOOP") - - if args.optimized: - sqlx_flags.add("-O3") - sqlx_defines.add("-DNDEBUG") - else: - sqlx_flags.add("-ggdb") - - sqlx_links.add("-lz") - - sqlx_so = os.path.join(tempdir.name, "render.so") - info(f"Building salis SQLite extension at: {sqlx_so}") +# ------------------------------------------------------------------------------ +# Source configuration +# ------------------------------------------------------------------------------ +log.info(description) +log.info(f"Called '{prog} {args.command}' with the following options: {fmt_opts(vars(args))}") +ns = SimpleNamespace() - sqlx_build_cmd = [args.compiler, "core/render.c", "-o", sqlx_so] - sqlx_build_cmd.extend(sqlx_flags) - sqlx_build_cmd.extend(sqlx_defines) - sqlx_build_cmd.extend(sqlx_links) +class Request(enum.Enum): + REQUEST_NAME = "n" + REQUEST_OPTS = "o" + REQUEST_HASH = "h" - info(f"Using build command: {sqlx_build_cmd}") - subprocess.run(sqlx_build_cmd, check=True) +def gen_sim_paths(): + ns.sim_dir = os.path.join(args.home, args.name) + ns.sim_opts = os.path.join(ns.sim_dir, "opts.json") + ns.sim_path = os.path.join(ns.sim_dir, args.name) - # Generate configuration so front-end knows how to render the plots. - # Each architecture may also provide its own set of plots, which will be merged with the - # default dictionary below. - plots = { - "General": { - "cycl": { - "table": "general", - "type": "lines", - "cols": [f"cycl_{i}" for i in range(args.cores)], - }, - "mall": { - "table": "general", - "type": "lines", - "cols": [f"mall_{i}" for i in range(args.cores)], - }, - "pnum": { - "table": "general", - "type": "lines", - "cols": [f"pnum_{i}" for i in range(args.cores)], - }, - "ppop": { - "table": "general", - "type": "lines", - "cols": [f"{pref}_{i}" for i in range(args.cores) for pref in ["pfst", "plst"]], - }, - "ambs": { - "table": "general", - "type": "lines", - "cols": [f"{pref}_{i}" for pref in ["amb0", "amb1"] for i in range(args.cores)], - }, - "eevs": { - "table": "general", - "type": "lines", - "cols": [f"{pref}_{i}" for pref in ["emb0", "emb1", "eliv", "edea"] for i in range(args.cores)], - }, - }, - } +def source_from_opts_file(): + if not os.path.isdir(ns.sim_dir): + raise RuntimeError(f"No simulation found named {args.name}") - heatmaps = { - "Events": { - f"aev_{i}": { - "table": f"aev_{i}", - } for i in range(args.cores) - } | { - f"eev_{i}": { - "table": f"eev_{i}", - } for i in range(args.cores) - } | { - f"bev_{i}": { - "table": f"bev_{i}", - } for i in range(args.cores) - }, - } + with open(ns.sim_opts, "r") as f: + opts = json.loads(f.read()) - for key in arch_vars.plots: - plots[key] = (plots[key] if key in plots else {}) | arch_vars.plots[key] + for key, val in opts.items(): + setattr(args, key, val) - for key in arch_vars.heatmaps: - heatmaps[key] = (heatmaps[key] if key in heatmaps else {}) | arch_vars.heatmaps[key] + log.info(f"Sourced configuration from '{ns.sim_opts}': {fmt_opts(opts)}") - info(f"Generated plot configuration: {plots}") - info(f"Generated heatmap configuration: {heatmaps}") +def request_from_server(request): + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as client: + client.connect((args.ip, args.port)) + client.sendall(request.value.encode()) + return json.load(client.makefile(mode="r")) - # NOTE: this server implementation is very minimal and has no built-in security. - # Please do not put this on the internet! Only run the data server within secure - # networks that you own. - sim_db = os.path.join(sim_dir, f"{args.name}.sqlite3") +# New simulation +if args.command == "new": + gen_sim_paths() - class Handler(SimpleHTTPRequestHandler): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, directory="data") + if 0 < args.data_push_pow < args.sync_pow: + raise RuntimeError("Data push power must be equal or greater than core sync power") - def log_message(_, format, *args): - info(format % args) + if os.path.isdir(ns.sim_dir) and args.force: + log.warning(f"Force flag used! Wiping old simulation at: {ns.sim_dir}") + shutil.rmtree(ns.sim_dir) - def log_error(_, format, *args): - warn(format % args) + if os.path.isdir(ns.sim_dir): + raise RuntimeError(f"Simulation directory found at: {ns.sim_dir}; use --force flag to remove it") - def send_200_and_mime_type(self, mime_type): - self.send_response(200) - self.send_header("Content-type", mime_type) - self.end_headers() + if args.seed == -1: + args.seed = random.getrandbits(64) + log.info(f"Using random seed: {hex(args.seed)}") - def send_as_json(self, obj): - self.send_200_and_mime_type("application/json") - self.wfile.write(json.dumps(obj).encode("utf-8")) + log.info(f"Creating new simulation directory at: {ns.sim_dir}") + log.info(f"Creating configuration file at: {ns.sim_opts}") + os.mkdir(ns.sim_dir) - def send_file_as(self, path, mime_type): - self.send_200_and_mime_type(mime_type) - with open(path, "r") as f: self.wfile.write(f.read().encode("utf-8")) + opts = {fmt_h2u(name[1]): getattr(args, fmt_h2u(name[1])) for (name, sub_parsers, _), kwargs in options.items() if new in sub_parsers and load not in sub_parsers} - @staticmethod - @contextlib.contextmanager - def sqlite_connect(): - db_con = sqlite3.connect(sim_db, timeout=600) - db_con.row_factory = sqlite3.Row - db_con.enable_load_extension(True) - db_con.execute("PRAGMA journal_mode=wal;") - db_con.execute(f"SELECT load_extension('{sqlx_so}');") - try: yield db_con - finally: db_con.close() + with open(ns.sim_opts, "w") as f: + f.write(f"{fmt_opts(opts)}\n") - def do_GET(self): - bits = urllib.parse.urlparse(self.path) +# Loaded simulation +if args.command == "load": + gen_sim_paths() + source_from_opts_file() - if bits.path == "/": return self.send_file_as("data/index.html", "text/html") - if bits.path.split("/")[1] in ["js", "vendor", "vue"]: return self.send_file_as("data" + bits.path, "text/javascript") - if bits.path == "/opts": return self.send_as_json(opt_vars | {"mvec_loop": arch_vars.mvec_loop, "name": args.name}) - if bits.path == "/plots": return self.send_as_json(plots) - if bits.path == "/heatmaps": return self.send_as_json(heatmaps) +# Data server +if args.command == "server": + gen_sim_paths() + source_from_opts_file() - if bits.path == "/data": - http_query = urllib.parse.parse_qs(bits.query) - table = http_query["table"][0] - rows = http_query["rows"][0] - nth = http_query["nth"][0] - axis = http_query["axis"][0] - low = http_query["low"][0] - high = http_query["high"][0] - is_eva = http_query["is_eva"][0] +# Data client +if args.command == "client": + # Pull basic information from the server + # Required to build the client with correct flags + args.name = request_from_server(Request.REQUEST_NAME)["name"] + log.info(f"Sourced simulation name from '{args.ip}:{args.port}': {args.name}") - if is_eva == "true": - left = http_query["left"][0] - pixels = http_query["pixels"][0] - pixel_pow = http_query["pixel_pow"][0] - selects = ", ".join([f"cycl_{i}" for i in range(args.cores)]) + f", eva_render({left}, {pixels}, {pixel_pow}, evts) as eva_render, step" - else: - selects = "*" + opts = request_from_server(Request.REQUEST_OPTS) - sql_query = f"SELECT * FROM (SELECT rowid, {selects} FROM {table} WHERE {axis} >= {low} AND {axis} <= {high} AND rowid % {nth} == 0 ORDER BY {axis} DESC LIMIT {rows}) ORDER BY {axis} ASC;" - with Handler.sqlite_connect() as db_con: sql_list = [dict(row) for row in db_con.execute(sql_query).fetchall()] - return self.send_as_json(sql_list) + for key, val in opts.items(): + setattr(args, key, val) - if bits.path == "/high": - http_query = urllib.parse.parse_qs(bits.query) - axis = http_query["axis"][0] - sql_query = f"SELECT {axis} as high FROM general ORDER BY {axis} DESC LIMIT 1;" - with Handler.sqlite_connect() as db_con: sql_dict = dict(db_con.execute(sql_query).fetchone()) - return self.send_as_json(sql_dict) + log.info(f"Sourced configuration from '{args.ip}:{args.port}': {fmt_opts(opts)}") - self.log_error(f"Unsupported endpoint: {bits.path}") - self.send_response(400) - self.end_headers() + # Server and client should be on the same hash + server_hash = request_from_server(Request.REQUEST_HASH)["hash"] + client_hash = subprocess.check_output(["git", "rev-parse", "HEAD"]).decode("utf-8").strip() - info("Launching data server") - server = ThreadingHTTPServer(("", args.port), Handler) + if server_hash != client_hash: + raise RuntimeError(f"Server hash '{server_hash}' and client hash '{client_hash}' don't match") - try: - server.serve_forever() - except KeyboardInterrupt: - pass + log.info(f"Confirmed server and client are running against the same git hash: {server_hash}") - info("Shutting down data server") - sys.exit(0) +# ------------------------------------------------------------------------------ +# Architecture variables +# ------------------------------------------------------------------------------ +arch_path = os.path.join("arch", args.arch, "vars.py") +log.info(f"Loading architecture variables from: {arch_path}") +arch_vars = runpy.run_path(arch_path, init_globals=globals()) # ------------------------------------------------------------------------------ -# Compile ancestor organism +# Assembler # ------------------------------------------------------------------------------ -if args.command in ["bench", "new"] and args.anc is not None: - anc_path = os.path.join("anc", args.arch, f"{args.anc}.asm") +anc_path = os.path.join("anc", args.arch, f"{args.anc}.asm") - if not os.path.isfile(anc_path): - erro(f"Could not find ancestor file: {anc_path}") +if not os.path.isfile(anc_path): + raise RuntimeError(f"Could not find ancestor file: {anc_path}") - with open(anc_path, "r") as file: - lines = file.read().splitlines() +with open(anc_path, "r") as f: + lines = f.read().splitlines() - lines = filter(lambda line: not line.startswith(";"), lines) - lines = filter(lambda line: not line.isspace(), lines) - lines = filter(lambda line: line, lines) - lines = map(lambda line: line.split(), lines) +lines = filter(lambda line: not line.startswith(";"), lines) +lines = filter(lambda line: not line.isspace(), lines) +lines = filter(lambda line: line, lines) +lines = map(lambda line: line.split(), lines) - anc_bytes = [] +anc_bytes = [] - for line in lines: - found = False +for line in lines: + found = False - for byte, tup in enumerate(arch_vars.inst_set): - if line == tup[0]: - anc_bytes.append(byte) - found = True - break + for byte, tup in enumerate(arch_vars["inst_set"]): + if line == tup[0]: + anc_bytes.append(byte) + found = True + break - if not found: - erro(f"Unrecognized instruction in ancestor file: {line}") + if not found: + raise RuntimeError(f"Unrecognized instruction in ancestor file: {line}") - anc_bytes_repr = ",".join(map(str, anc_bytes)) - info(f"Compiled ancestor file '{anc_path}' into byte array: {{{anc_bytes_repr}}}") +anc_repr = f"{{{",".join(map(str, anc_bytes))}}}" +log.info(f"Compiled ancestor file '{anc_path}' into byte array: {anc_repr}") # ------------------------------------------------------------------------------ -# Populate compiler flags +# Compiler flags # ------------------------------------------------------------------------------ -flags = set() -includes = set() -defines = set() -links = set() +def pop_ui_vars(): + ns.ui_path = os.path.join("ui", args.ui, "vars.py") + ns.ui_vars = runpy.run_path(ns.ui_path, init_globals=globals()) + log.info(f"Sourced UI variables from: {ns.ui_path}") -flags.update({*args.compiler_flags.split(), f"-Iarch/{args.arch}", "-Icore"}) + ns.b.flags.add(f"-Iui/{args.ui}") + ns.b.flags.update({*ns.ui_vars["flags"]}) + ns.b.defines.update({*ns.ui_vars["defines"]}) + ns.b.links.update({*ns.ui_vars["links"]}) -defines.add(f"-DARCH=\"{args.arch}\"") -defines.add(f"-DCOMMAND_{args.command.upper()}") -defines.add(f"-DCORES={args.cores}") -defines.add(f"-DMUTA_RANGE={2 ** args.muta_pow}ul") -defines.add(f"-DMVEC_SIZE={2 ** args.mvec_pow}ul") -defines.add(f"-DSEED={args.seed}ul") -defines.add(f"-DSYNC_INTERVAL={2 ** args.sync_pow}ul") -defines.add(f"-DTHREAD_GAP={args.thread_gap}") +def pop_data_push_vars(): + ns.sim_db = os.path.join(ns.sim_dir, f"{args.name}.sqlite3") -defines.add(f"-DCORE_FIELDS={" ".join(f"CORE_FIELD({", ".join(field)})" for field in arch_vars.core_fields)}") -defines.add(f"-DCORE_DATA_FIELDS={" ".join(f"CORE_DATA_FIELD({", ".join(field)})" for field in arch_vars.core_data_fields)}") -defines.add(f"-DPROC_FIELDS={" ".join(f"PROC_FIELD({", ".join(field)})" for field in arch_vars.proc_fields)}") -defines.add(f"-DINST_SET={" ".join(f"INST({index}, {"_".join(inst[0])}, \"{" ".join(inst[0])}\", L'{inst[1]}')" for index, inst in enumerate(arch_vars.inst_set))}") -defines.add(f"-DCORE_FIELD_COUNT={len(arch_vars.core_fields)}") -defines.add(f"-DPROC_FIELD_COUNT={len(arch_vars.proc_fields)}") -defines.add(f"-DINST_COUNT={len(arch_vars.inst_set)}") -defines.add(f"-DFOR_CORES={" ".join(f"FOR_CORE({i})" for i in range(args.cores))}") + if args.data_push_pow: + ns.b.defines.add("-DDATA_PUSH") + ns.b.defines.add(f"-DDATA_PUSH_INTERVAL={2 ** args.data_push_pow}ul") + ns.b.defines.add(f"-DDATA_PUSH_PATH=\"{ns.sim_db}\"") + ns.b.links.add("-lsqlite3") + ns.b.links.add("-lz") + log.info(f"Data will be aggregated at: {ns.sim_db}") + else: + log.warning("Data aggregation disabled") -if args.muta_flip: defines.add("-DMUTA_FLIP") -if arch_vars.mvec_loop: defines.add("-DMVEC_LOOP") + if not args.no_compress: + ns.b.defines.add("-DCOMPRESS") + ns.b.links.add("-lz") + log.info("Save file compression enabled") + else: + log.warning("Save file compression disabled") -if args.optimized: - flags.add("-O3") - defines.add("-DNDEBUG") -else: - flags.add("-ggdb") +def pop_sim_path_vars(): + ns.b.defines.add(f"-DSIM_OPTS=\"{ns.sim_opts}\"") + ns.b.defines.add(f"-DSIM_PATH=\"{ns.sim_path}\"") -if args.command in ["bench"]: - includes.add("stdio.h") - defines.add(f"-DSTEPS={args.steps}ul") +def pop_net_vars(): + ns.b.defines.add(f"-DPORT={args.port}") + ns.b.defines.add(f"-DREQUEST_NAME='{Request.REQUEST_NAME.value}'") + ns.b.defines.add(f"-DREQUEST_OPTS='{Request.REQUEST_OPTS.value}'") + ns.b.defines.add(f"-DREQUEST_HASH='{Request.REQUEST_HASH.value}'") + ns.b.links.add("-ljson-c") -if args.command in ["bench", "new"]: - defines.add(f"-DCLONES={args.clones}") +def pop_general(): + ns.b.flags.add(f"-Iarch/{args.arch}") + ns.b.flags.add("-Icore") - if args.anc is not None: - defines.add(f"-DANC_BYTES={{{anc_bytes_repr}}}") - defines.add(f"-DANC_SIZE={len(anc_bytes)}") + if args.muta_flip: + ns.b.defines.add("-DMUTA_FLIP") -if args.command in ["load", "new"]: - ui_path = os.path.join("ui", args.ui) - info(f"Loading UI variables from: {os.path.join(ui_path, "ui_vars.py")}") - sys.path.append(ui_path) - from ui_vars import UIVars - ui_vars = UIVars(args) + if arch_vars["mvec_loop"]: + ns.b.defines.add("-DMVEC_LOOP") - flags.add(f"-Iui/{args.ui}") - flags.update(ui_vars.flags) - includes.update(ui_vars.includes) - defines.update(ui_vars.defines) - defines.add(f"-DAUTOSAVE_INTERVAL={2 ** args.auto_save_pow}ul") - defines.add(f"-DAUTOSAVE_NAME_LEN={len(sim_path) + 20}") - defines.add(f"-DNAME=\"{args.name}\"") - defines.add(f"-DSIM_PATH=\"{sim_path}\"") - links.update(ui_vars.links) + ns.b.defines.add(f"-DANC=\"{args.anc}\"") + ns.b.defines.add(f"-DANC_BYTES={anc_repr}") + ns.b.defines.add(f"-DANC_SIZE={len(anc_bytes)}") + ns.b.defines.add(f"-DARCH=\"{args.arch}\"") + ns.b.defines.add(f"-DAUTOSAVE_INTERVAL={2 ** args.auto_save_pow}ul") + ns.b.defines.add(f"-DCLONES={args.clones}") + ns.b.defines.add(f"-DCOMMAND_{args.command.upper()}") + ns.b.defines.add(f"-DCORE_DATA_FIELDS={" ".join(f"CORE_DATA_FIELD({", ".join(field)})" for field in arch_vars["core_data_fields"])}") + ns.b.defines.add(f"-DCORE_FIELD_COUNT={len(arch_vars["core_fields"])}") + ns.b.defines.add(f"-DCORE_FIELDS={" ".join(f"CORE_FIELD({", ".join(field)})" for field in arch_vars["core_fields"])}") + ns.b.defines.add(f"-DCORES={args.cores}") + ns.b.defines.add(f"-DFOR_CORES={" ".join(f"FOR_CORE({i})" for i in range(args.cores))}") + ns.b.defines.add(f"-DINST_COUNT={len(arch_vars["inst_set"])}") + ns.b.defines.add(f"-DINST_SET={" ".join(f"INST({index}, {"_".join(inst[0])}, \"{" ".join(inst[0])}\", L'{inst[1]}')" for index, inst in enumerate(arch_vars["inst_set"]))}") + ns.b.defines.add(f"-DMUTA_RANGE={2 ** args.muta_pow}ul") + ns.b.defines.add(f"-DMVEC_SIZE={2 ** args.mvec_pow}ul") + ns.b.defines.add(f"-DNAME=\"{args.name}\"") + ns.b.defines.add(f"-DPROC_FIELD_COUNT={len(arch_vars["proc_fields"])}") + ns.b.defines.add(f"-DPROC_FIELDS={" ".join(f"PROC_FIELD({", ".join(field)})" for field in arch_vars["proc_fields"])}") + ns.b.defines.add(f"-DSEED={args.seed}ul") + ns.b.defines.add(f"-DSYNC_INTERVAL={2 ** args.sync_pow}ul") - if args.data_push_pow: - includes.update({"sqlite3.h", "zlib.h"}) - data_push_path = os.path.join(sim_dir, f"{args.name}.sqlite3") - defines.add(f"-DDATA_PUSH_INTERVAL={2 ** args.data_push_pow}ul") - defines.add(f"-DDATA_PUSH_PATH=\"{data_push_path}\"") - links.update({"-lsqlite3", "-lz"}) - info(f"Data will be aggregated at: {data_push_path}") - else: - warn("Data aggregation disabled") +# Populate for new +if args.command == "new": + ns.b = Build("core/salis.c") + pop_ui_vars() + pop_data_push_vars() + pop_sim_path_vars() + pop_general() - if not args.no_compress: - includes.add("zlib.h") - defines.add("-DCOMPRESS") - links.add("-lz") - info("Save file compression enabled") - else: - warn("Save file compression disabled") + ns.b.defines.add(f"-DAUTOSAVE_NAME_LEN={len(ns.sim_path) + 20}") + ns.b.defines.add(f"-DTHREAD_GAP={args.thread_gap}") -# ------------------------------------------------------------------------------ -# Build executable -# ------------------------------------------------------------------------------ -salis_bin = os.path.join(tempdir.name, "salis_bin") -info(f"Building salis binary at: {salis_bin}") +# Populate for load +if args.command == "load": + ns.b = Build("core/salis.c") + pop_ui_vars() + pop_data_push_vars() + pop_sim_path_vars() + pop_general() + + ns.b.defines.add(f"-DAUTOSAVE_NAME_LEN={len(ns.sim_path) + 20}") + ns.b.defines.add(f"-DTHREAD_GAP={args.thread_gap}") -build_cmd = [args.compiler, "core/salis.c", "-o", salis_bin] -build_cmd.extend(flags) -build_cmd.extend(sum(map(lambda include: ["-include", include], includes), [])) -build_cmd.extend(defines) -build_cmd.extend(links) +# Populate for server +if args.command == "server": + ns.b = Build("data/server.c") + pop_sim_path_vars() + pop_net_vars() + pop_general() -info(f"Using build command: {build_cmd}") -subprocess.run(build_cmd, check=True) +# Populate for client +if args.command == "client": + ns.b = Build("data/client.c") + pop_net_vars() + pop_general() + ns.b.defines.add(f"-DIP={args.ip}") # ------------------------------------------------------------------------------ -# Run salis binary +# Build and launch executable # ------------------------------------------------------------------------------ -info("Running salis binary...") - -run_cmd = [args.pre_cmd] if args.pre_cmd else [] -run_cmd.append(salis_bin) - -info(f"Using run command: {" ".join(run_cmd)}") -salis_sp = subprocess.Popen(run_cmd, stdout=sys.stdout, stderr=sys.stderr) - -# When using signals (e.g. SIGTERM), they must be sent to the entire process group -# to make sure both the simulator and the interpreter get shut down. -try: - salis_sp.wait() -except KeyboardInterrupt: - salis_sp.terminate() - salis_sp.wait() - -code = salis_sp.returncode - -if code != 0: - erro(f"Salis binary returned code: {code}") +ns.b.build() +ns.b.exec() |