// file    : core/salis.c
// project : Salis-VM
// author  : Paul Oliver <contact@pauloliver.dev>

// index:
// [section] includes
// [section] macros
// [section] globals
// [section] memory vector functions
// [section] mutator functions
// [section] process functions
// [section] core functions
// [section] salis functions

// ----------------------------------------------------------------------------
// [section] includes
// ----------------------------------------------------------------------------
#include <assert.h>
#include <sqlite3.h>
#include <stdatomic.h>
#include <stdbool.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <threads.h>
#include <zlib.h>

#include "arch.h"
#include "arch_inst.h"
#include "arch_spec.h"
#include "compress.h"
#include "logger.h"
#include "salis.h"
#include "sql.h"

// ----------------------------------------------------------------------------
// [section] macros
// ----------------------------------------------------------------------------
#define INST_MASK 0x7f
#define IPCM_FLAG 0x80
#define MALL_FLAG 0x80

#define STEP_GROUP_TIME_MIN_MS 0x400
#define STEP_GROUP_TIME_MAX_MS 0x800

// ----------------------------------------------------------------------------
// [section] globals
// ----------------------------------------------------------------------------
struct Core g_cores[CORES];
uint64_t g_step;
static uint64_t g_sync;
static const struct Proc g_null_proc;

static char g_asav_pbuf[AUTOSAVE_NAME_LEN];
static char g_evas_pbuf[EVA_SAVE_NAME_LEN];

static atomic_bool g_running;
static thrd_t g_thrd;

// ----------------------------------------------------------------------------
// [section] memory vector functions
// ----------------------------------------------------------------------------
#if defined(ARCH_SPEC_MVEC_LOOP)
uint64_t mvec_loop(uint64_t addr) {
    return addr % MVEC_SIZE;
}
#endif

bool mvec_is_alloc(const struct Core *core, uint64_t addr) {
    assert(core);

#if defined(ARCH_SPEC_MVEC_LOOP)
    return core->mvec[mvec_loop(addr)] & MALL_FLAG ? true : false;
#else
    if (addr < MVEC_SIZE) {
        return core->mvec[addr] & MALL_FLAG ? true : false;
    } else {
        return true;
    }
#endif
}

void mvec_alloc(struct Core *core, uint64_t addr) {
    assert(core);
    assert(!mvec_is_alloc(core, addr));

#if defined(ARCH_SPEC_MVEC_LOOP)
    uint64_t loop_addr = mvec_loop(addr);
#else
    assert(addr < MVEC_SIZE);
    uint64_t loop_addr = addr;
#endif

    core->mvec[loop_addr] |= MALL_FLAG;
    core->aeva.data[loop_addr]++;
    core->mall++;
}


void mvec_free(struct Core *core, uint64_t addr) {
    assert(core);
    assert(mvec_is_alloc(core, addr));

#if defined(ARCH_SPEC_MVEC_LOOP)
    uint64_t loop_addr = mvec_loop(addr);
#else
    assert(addr < MVEC_SIZE);
    uint64_t loop_addr = addr;
#endif

    core->mvec[loop_addr] ^= MALL_FLAG;
    core->aeva.data[loop_addr]++;
    core->mall--;
}

uint8_t mvec_get_byte(const struct Core *core, uint64_t addr) {
    assert(core);

#if defined(ARCH_SPEC_MVEC_LOOP)
    return core->mvec[mvec_loop(addr)];
#else
    if (addr < MVEC_SIZE) {
        return core->mvec[addr];
    } else {
        return 0;
    }
#endif
}

uint8_t mvec_get_inst(const struct Core *core, uint64_t addr) {
    assert(core);

#if defined(ARCH_SPEC_MVEC_LOOP)
    return core->mvec[mvec_loop(addr)] & INST_MASK;
#else
    if (addr < MVEC_SIZE) {
        return core->mvec[addr] & INST_MASK;
    } else {
        return 0;
    }
#endif
}

void mvec_set_inst(struct Core *core, uint64_t addr, uint8_t inst) {
    assert(core);
    assert(inst < ARCH_INST_CAP);

#if defined(ARCH_SPEC_MVEC_LOOP)
    core->mvec[mvec_loop(addr)] &= MALL_FLAG;
    core->mvec[mvec_loop(addr)] |= inst;
#else
    assert(addr < MVEC_SIZE);
    core->mvec[addr] &= MALL_FLAG;
    core->mvec[addr] |= inst;
#endif
}

#if defined(ARCH_SPEC_MUTA_FLIP)
void mvec_flip_bit(struct Core *core, uint64_t addr, int bit) {
    assert(core);
    assert(bit < 8);
    core->mvec[addr] ^= (1 << bit) & INST_MASK;
}
#endif

bool mvec_proc_is_live(const struct Core *core, uint64_t pix) {
    assert(core);
    return pix >= core->pfst && pix <= core->plst;
}

bool mvec_is_in_mb0_of_proc(const struct Core *core, uint64_t addr, uint64_t pix) {
    assert(core);
    assert(mvec_proc_is_live(core, pix));
    uint64_t mb0a = arch_proc_get_mb0_addr(core, pix);
    uint64_t mb0s = arch_proc_get_mb0_size(core, pix);
    return ((addr - mb0a) % MVEC_SIZE) < mb0s;
}

bool mvec_is_in_mb1_of_proc(const struct Core *core, uint64_t addr, uint64_t pix) {
    assert(core);
    assert(mvec_proc_is_live(core, pix));
    uint64_t mb1a = arch_proc_get_mb1_addr(core, pix);
    uint64_t mb1s = arch_proc_get_mb1_size(core, pix);
    return ((addr - mb1a) % MVEC_SIZE) < mb1s;
}

bool mvec_is_proc_owner(const struct Core *core, uint64_t addr, uint64_t pix) {
    assert(core);
    assert(mvec_proc_is_live(core, pix));
    return mvec_is_in_mb0_of_proc(core, addr, pix) || mvec_is_in_mb1_of_proc(core, addr, pix);
}

uint64_t mvec_get_owner(const struct Core *core, uint64_t addr) {
    assert(core);
#if !defined(ARCH_SPEC_MVEC_LOOP)
    assert(addr < MVEC_SIZE);
#endif
    assert(mvec_is_alloc(core, addr));

    for (uint64_t pix = core->pfst; pix <= core->plst; pix++) {
        if (mvec_is_proc_owner(core, addr, pix)) {
            return pix;
        }
    }

    assert(false);
    return -1;
}

// ----------------------------------------------------------------------------
// [section] mutator functions
// ----------------------------------------------------------------------------
// The following implements SplitMix64 PRNG for cosmic-ray events:
// https://en.wikipedia.org/wiki/Xorshift
#if SEED != 0
#if defined(COMMAND_NEW)
static uint64_t muta_smix(uint64_t *seed) {
    assert(seed);
    uint64_t next = (*seed += 0x9e3779b97f4a7c15);
    next = (next ^ (next >> 30)) * 0xbf58476d1ce4e5b9;
    next = (next ^ (next >> 27)) * 0x94d049bb133111eb;
    return next ^ (next >> 31);
}
#endif

static uint64_t muta_ro64(uint64_t x, int k) {
    return (x << k) | (x >> (64 - k));
}

uint64_t muta_next(struct Core *core) {
    assert(core);
    uint64_t r = muta_ro64(core->muta[1] * 5, 7) * 9;
    uint64_t t = core->muta[1] << 17;
    core->muta[2] ^= core->muta[0];
    core->muta[3] ^= core->muta[1];
    core->muta[1] ^= core->muta[2];
    core->muta[0] ^= core->muta[3];
    core->muta[2] ^= t;
    core->muta[3] = muta_ro64(core->muta[3], 45);
    return r;
}

void muta_cosmic_ray(struct Core *core) {
    assert(core);
    uint64_t a = muta_next(core) % MUTA_RANGE;
    uint64_t b = muta_next(core);

    if (a < MVEC_SIZE) {
#if defined(ARCH_SPEC_MUTA_FLIP)
        mvec_flip_bit(core, a, (int)(b % 8));
#else
        mvec_set_inst(core, a, b & INST_MASK);
#endif
    }
}
#endif

// ----------------------------------------------------------------------------
// [section] process functions
// ----------------------------------------------------------------------------
void proc_new(struct Core *core, const struct Proc *proc) {
    assert(core);
    assert(proc);

    if (core->pnum == core->pcap) {
        // Reallocate circular array
        uint64_t new_pcap = core->pcap * 2;
        struct Proc *new_pvec = calloc(new_pcap, sizeof(struct Proc));

        for (uint64_t pix = core->pfst; pix <= core->plst; pix++) {
            uint64_t iold = pix % core->pcap;
            uint64_t inew = pix % new_pcap;
            memcpy(&new_pvec[inew], &core->pvec[iold], sizeof(struct Proc));
        }

        free(core->pvec);
        core->pcap = new_pcap;
        core->pvec = new_pvec;
    }

    core->pnum++;
    core->plst++;
    memcpy(&core->pvec[core->plst % core->pcap], proc, sizeof(struct Proc));

    // Register process birth event
    uint64_t child_addr = arch_proc_get_mb0_addr(core, core->plst);
    uint64_t child_size = arch_proc_get_mb0_size(core, core->plst);

    for (uint64_t i = 0; i < child_size; i++) {
        uint64_t addr = child_addr + i;
#if defined(ARCH_SPEC_MVEC_LOOP)
        core->beva.data[mvec_loop(addr)]++;
#else
        core->beva.data[addr]++;
#endif
    }
}

void proc_kill(struct Core *core) {
    assert(core);
    assert(core->pnum > 1);
    arch_on_proc_kill(core);
    core->pcur++;
    core->pfst++;
    core->pnum--;
}

const struct Proc *proc_get(const struct Core *core, uint64_t pix) {
    assert(core);

    if (mvec_proc_is_live(core, pix)) {
        return &core->pvec[pix % core->pcap];
    } else {
        return &g_null_proc;
    }
}

struct Proc *proc_fetch(struct Core *core, uint64_t pix) {
    assert(core);
    assert(mvec_proc_is_live(core, pix));
    return &core->pvec[pix % core->pcap];
}

// ----------------------------------------------------------------------------
// [section] core functions
// ----------------------------------------------------------------------------
static void core_save(const struct Core *core, FILE *f) {
    assert(core);
    assert(f);

    fwrite(&core->step, sizeof(uint64_t), 1, f);
    fwrite(&core->cycl, sizeof(uint64_t), 1, f);
    fwrite(&core->mall, sizeof(uint64_t), 1, f);
    fwrite(core->muta, sizeof(uint64_t), 4, f);
    fwrite(&core->pnum, sizeof(uint64_t), 1, f);
    fwrite(&core->pcap, sizeof(uint64_t), 1, f);
    fwrite(&core->pfst, sizeof(uint64_t), 1, f);
    fwrite(&core->plst, sizeof(uint64_t), 1, f);
    fwrite(&core->pcur, sizeof(uint64_t), 1, f);
    fwrite(&core->psli, sizeof(uint64_t), 1, f);
    fwrite(&core->ivpt, sizeof(uint64_t), 1, f);
    fwrite(core->ivav, sizeof(uint64_t), SYNC_INTERVAL, f);
    fwrite(core->iviv, sizeof(uint8_t), SYNC_INTERVAL, f);

    fwrite(core->mvec, sizeof(uint8_t), MVEC_SIZE, f);
    fwrite(core->pvec, sizeof(struct Proc), core->pcap, f);

    fwrite(&core->amb0, sizeof(double), 1, f);
    fwrite(&core->amb1, sizeof(double), 1, f);
    fwrite(&core->emb0, sizeof(uint64_t), 1, f);
    fwrite(&core->emb1, sizeof(uint64_t), 1, f);
    fwrite(&core->eliv, sizeof(uint64_t), 1, f);
    fwrite(&core->edea, sizeof(uint64_t), 1, f);

    fwrite(core->aeva.data, sizeof(uint64_t), MVEC_SIZE, f);
    fwrite(core->eeva.data, sizeof(uint64_t), MVEC_SIZE, f);
    fwrite(core->beva.data, sizeof(uint64_t), MVEC_SIZE, f);

    arch_core_save(core, f);
}

void core_event_array_init(struct EventArray *eva) {
    assert(eva);
    eva->params.size = sizeof(uint64_t) * MVEC_SIZE;
    eva->params.in = (Bytef *)eva->data;
    eva->params.out = (Bytef *)eva->comp;
}

#if defined(COMMAND_NEW)
static void core_assemble_ancestor(struct Core *core) {
    assert(core);

#if defined(ARCH_SPEC_MVEC_LOOP)
    uint64_t addr = SALIS_UINT64_HALF;
#else
    uint64_t addr = 0;
#endif

    uint8_t anc_bytes[] = ANC_BYTES;

    for (uint64_t i = 0; i < ANC_SIZE; i++, addr++) {
        for (uint64_t j = 0; j < CLONES; j++) {
            uint64_t addr_clone = addr + (MVEC_SIZE / CLONES) * j;
            mvec_alloc(core, addr_clone);
            mvec_set_inst(core, addr_clone, anc_bytes[i]);
        }
    }
}

static void core_init(struct Core *core, uint64_t *seed) {
    assert(core);
    assert(seed);

#if SEED != 0
    assert(*seed);
    core->muta[0] = muta_smix(seed);
    core->muta[1] = muta_smix(seed);
    core->muta[2] = muta_smix(seed);
    core->muta[3] = muta_smix(seed);
#else
    (void)seed;
#endif

    core->pnum = CLONES;
    core->pcap = CLONES;
    core->plst = CLONES - 1;
    core->psli = arch_proc_get_slice(core, core->pcur);
    core->ivav = calloc(SYNC_INTERVAL, sizeof(uint64_t));
    core->iviv = calloc(SYNC_INTERVAL, sizeof(uint8_t));
    core->pvec = calloc(core->pcap, sizeof(struct Proc));
    assert(core->pvec);

    core_event_array_init(&core->aeva);
    core_event_array_init(&core->eeva);
    core_event_array_init(&core->beva);

    core_assemble_ancestor(core);
    arch_core_init(core);
}
#endif

#if defined(COMMAND_LOAD)
static void core_load(struct Core *core, FILE *f) {
    assert(core);
    assert(f);

    fread(&core->step, sizeof(uint64_t), 1, f);
    fread(&core->cycl, sizeof(uint64_t), 1, f);
    fread(&core->mall, sizeof(uint64_t), 1, f);
    fread(core->muta, sizeof(uint64_t), 4, f);
    fread(&core->pnum, sizeof(uint64_t), 1, f);
    fread(&core->pcap, sizeof(uint64_t), 1, f);
    fread(&core->pfst, sizeof(uint64_t), 1, f);
    fread(&core->plst, sizeof(uint64_t), 1, f);
    fread(&core->pcur, sizeof(uint64_t), 1, f);
    fread(&core->psli, sizeof(uint64_t), 1, f);
    fread(&core->ivpt, sizeof(uint64_t), 1, f);

    core->ivav = calloc(SYNC_INTERVAL, sizeof(uint64_t));
    core->iviv = calloc(SYNC_INTERVAL, sizeof(uint8_t));
    core->pvec = calloc(core->pcap, sizeof(struct Proc));
    assert(core->ivav);
    assert(core->iviv);
    assert(core->pvec);

    fread(core->ivav, sizeof(uint64_t), SYNC_INTERVAL, f);
    fread(core->iviv, sizeof(uint8_t), SYNC_INTERVAL, f);
    fread(core->mvec, sizeof(uint8_t), MVEC_SIZE, f);
    fread(core->pvec, sizeof(struct Proc), core->pcap, f);

    fread(&core->amb0, sizeof(double), 1, f);
    fread(&core->amb1, sizeof(double), 1, f);
    fread(&core->emb0, sizeof(uint64_t), 1, f);
    fread(&core->emb1, sizeof(uint64_t), 1, f);
    fread(&core->eliv, sizeof(uint64_t), 1, f);
    fread(&core->edea, sizeof(uint64_t), 1, f);

    fread(core->aeva.data, sizeof(uint64_t), MVEC_SIZE, f);
    fread(core->eeva.data, sizeof(uint64_t), MVEC_SIZE, f);
    fread(core->beva.data, sizeof(uint64_t), MVEC_SIZE, f);

    core_event_array_init(&core->aeva);
    core_event_array_init(&core->eeva);
    core_event_array_init(&core->beva);

    arch_core_load(core, f);
}
#endif

static void core_pull_ipcm(struct Core *core) {
    assert(core);
    assert(core->ivpt < SYNC_INTERVAL);
    uint8_t *iinst = &core->iviv[core->ivpt];
    uint64_t *iaddr = &core->ivav[core->ivpt];

    if ((*iinst & IPCM_FLAG) != 0) {
        mvec_set_inst(core, *iaddr, *iinst & INST_MASK);
        *iinst = 0;
        *iaddr = 0;
    }

    assert(*iinst == 0);
    assert(*iaddr == 0);
}

void core_push_ipcm(struct Core *core, uint8_t inst, uint64_t addr) {
    assert(core);
    assert(core->ivpt < SYNC_INTERVAL);
    assert((inst & IPCM_FLAG) == 0);
    uint8_t *iinst = &core->iviv[core->ivpt];
    uint64_t *iaddr = &core->ivav[core->ivpt];
    assert(*iinst == 0);
    assert(*iaddr == 0);
    *iinst = inst | IPCM_FLAG;
    *iaddr = addr;
}

static void core_step(struct Core *core) {
    assert(core);
    assert(core->psli);
    assert(mvec_proc_is_live(core, core->pcur));

    core_pull_ipcm(core);
    core->ivpt++;
    core->ivpt %= SYNC_INTERVAL;

    // Register execution event
    uint64_t pcur_ip = arch_proc_get_ip_addr(core, core->pcur);

    if (mvec_is_in_mb0_of_proc(core, pcur_ip, core->pcur)) {
        core->emb0++;
    } else if (mvec_is_in_mb1_of_proc(core, pcur_ip, core->pcur)) {
        core->emb1++;
    } else if (mvec_is_alloc(core, pcur_ip)) {
        core->eliv++;
    } else {
        assert(!mvec_is_in_mb0_of_proc(core, pcur_ip, core->pcur));
        assert(!mvec_is_in_mb1_of_proc(core, pcur_ip, core->pcur));
        assert(!mvec_is_alloc(core, pcur_ip));
        core->edea++;
    }

#if defined(ARCH_SPEC_MVEC_LOOP)
    core->eeva.data[mvec_loop(pcur_ip)]++;
#else
    if (pcur_ip < MVEC_SIZE) {
        core->eeva.data[pcur_ip]++;
    }
#endif

    // Step process and decrement time slice counter
    arch_on_proc_step(core, core->pcur);
    core->step++;
    core->psli--;

    if (core->psli) {
        return;
    }

    // Time slice of process has ended; start time slice of next process
    bool cycle_ended = core->pcur == core->plst;
    core->pcur = cycle_ended ? core->pfst : core->pcur + 1;
    core->psli = arch_proc_get_slice(core, core->pcur);

    if (cycle_ended) {
        // Run the reaper at the end of every cycle
        // Kill old processes until at least 50% of memory is not allocated
        while (core->mall > MVEC_SIZE / 2 && core->pnum > 1) {
            proc_kill(core);
        }

#if SEED != 0
        muta_cosmic_ray(core);
#endif

        core->cycl++;
    }
}

static int core_thread(struct Core *core) {
    assert(core);
    assert(core->step == g_step);
    uint64_t step_group_size = 1;

    // Minimize calls to atomic_load_explicit by grouping calls
    // to core_step. Check state of atomic variable once per period time_delta_ms,
    // where STEP_GROUP_TIME_MIN_MS < time_delta_ms < STEP_GROUP_TIME_MAX_MS.
    while (atomic_load_explicit(&g_running, memory_order_relaxed)) {
        struct timespec time_start;
        clock_gettime(CLOCK_MONOTONIC_COARSE, &time_start);

        for (uint64_t i = 0; i < step_group_size; i++) {
            core_step(core);

            if (core->step % SYNC_INTERVAL == 0) {
                log_trace("Exiting core thread at step %#lx: sync-interval reached", core->step);
                log_trace("Step group size: %#lx", step_group_size);
                return 0;
            }
        }

        struct timespec time_end;
        clock_gettime(CLOCK_MONOTONIC_COARSE, &time_end);
        int time_delta_ms = ((time_end.tv_sec - time_start.tv_sec) * 1000) + ((time_end.tv_nsec - time_start.tv_nsec) / 1000000);

        if (time_delta_ms < STEP_GROUP_TIME_MIN_MS) {
            step_group_size *= 2;
        } else if (time_delta_ms > STEP_GROUP_TIME_MAX_MS && step_group_size > 1) {
            step_group_size /= 2;
        }
    }

    log_trace("Exiting core thread at step %#lx: simulation paused", core->step);
    log_trace("Step group size: %#lx", step_group_size);
    return 0;
}

#if !defined(NDEBUG)
static void core_validate(struct Core *core) {
    assert(core);

    // Validate core fields
    assert(core->step == g_step);
    assert(core->cycl <= g_step);
    assert(core->pnum <= core->pcap);
    assert(core->pnum == core->plst + 1 - core->pfst);
    assert(core->plst >= core->pfst);
    assert(core->pcur >= core->pfst && core->pcur <= core->plst);
    assert(core->psli != 0);

    // Validate memory allocation
    uint64_t mall = 0;

    for (uint64_t i = 0; i < MVEC_SIZE; i++) {
        mall += mvec_is_alloc(core, i) ? 1 : 0;
    }

    assert(core->mall == mall);

    // Validate IPCM state
    for (uint64_t i = 0; i < SYNC_INTERVAL; i++) {
        uint8_t iinst = core->iviv[i];

        if ((iinst & IPCM_FLAG) == 0) {
            uint64_t iaddr = core->ivav[i];
            assert(iinst == 0);
            assert(iaddr == 0);
        }
    }

    assert(core->ivpt == g_step % SYNC_INTERVAL);

    // Validate VM architecture specifics
    arch_validate_core(core);
}
#endif

static void core_free(struct Core *core) {
    assert(core);
    assert(core->ivav);
    assert(core->iviv);
    assert(core->pvec);

    arch_core_free(core);

    free(core->pvec);
    free(core->iviv);
    free(core->ivav);

    core->pvec = NULL;
    core->iviv = NULL;
    core->ivav = NULL;
}

// ----------------------------------------------------------------------------
// [section] salis functions
// ----------------------------------------------------------------------------
static void salis_sync(void) {
    assert(g_step % SYNC_INTERVAL == 0);
    log_trace("Synchronizing cores @step: %#lx; @sync: %#lx", g_step, g_sync);

#if !defined(NDEBUG)
    for (int i = 0; i < CORES; i++) {
        assert(g_cores[i].ivpt == 0);
    }
#endif

    uint8_t *iviv0 = g_cores[0].iviv;
    uint64_t *ivav0 = g_cores[0].ivav;

    for (int i = 1; i < CORES; i++) {
        g_cores[i - 1].iviv = g_cores[i].iviv;
        g_cores[i - 1].ivav = g_cores[i].ivav;
    }

    g_cores[CORES - 1].iviv = iviv0;
    g_cores[CORES - 1].ivav = ivav0;

    for (int i = 0; i < CORES; i++) {
        g_cores[i].ivpt = 0;
    }

    g_sync++;
}

static void salis_save(const char *path) {
    assert(path);
    log_info("Saving simulation state to '%s' on step %#lx", path, g_step);

    size_t size = 0;
    char *in = NULL;
    FILE *f = open_memstream(&in, &size);
    assert(f);

    for (int i = 0; i < CORES; i++) {
        core_save(&g_cores[i], f);
    }

    fwrite(&g_step, sizeof(uint64_t), 1, f);
    fwrite(&g_sync, sizeof(uint64_t), 1, f);
    fclose(f);

    char *out = malloc(size);
    assert(size);
    assert(out);

    struct DeflateParams params = {
        .size = size,
        .in = (Bytef *)in,
        .out = (Bytef *)out,
    };

    comp_deflate(&params);
    FILE *fx = fopen(path, "wb");
    assert(fx);
    fwrite(&size, sizeof(size_t), 1, fx);
    fwrite(out, sizeof(char), params.strm.total_out, fx);
    fclose(fx);
    comp_deflate_end(&params);

    free(in);
    free(out);
}

static void salis_auto_save(void) {
    assert(g_step % AUTOSAVE_INTERVAL == 0);
    int rem = snprintf(g_asav_pbuf, AUTOSAVE_NAME_LEN, "%s-%016lx", SIM_PATH, g_step);
    assert(rem >= 0);
    assert(rem < AUTOSAVE_NAME_LEN);
    (void)rem;
    salis_save(g_asav_pbuf);
}

#if defined(COMMAND_NEW)
static void salis_push_data_header(void) {
    assert(g_sim_db);
    log_info("Creating core table in SQLite database");

    // Empty blob columns help the data-server stay agnostic of DB schema.
    // These represent EVA data that will be appended by data server.
    sql_exec(
        NULL, NULL,
        "create table core ("
#define FOR_CORE(i) \
        "step_" #i " int not null, " \
        "cycl_" #i " int not null, " \
        "mall_" #i " int not null, " \
        "pnum_" #i " int not null, " \
        "pfst_" #i " int not null, " \
        "plst_" #i " int not null, " \
        "amb0_" #i " real not null, " \
        "amb1_" #i " real not null, " \
        "emb0_" #i " int not null, " \
        "emb1_" #i " int not null, " \
        "eliv_" #i " int not null, " \
        "edea_" #i " int not null, " \
        "aev" SALIS_EVENT_ARRAY_SIZE_COL_MARKER #i " int not null, aev_" #i " blob, " \
        "eev" SALIS_EVENT_ARRAY_SIZE_COL_MARKER #i " int not null, eev_" #i " blob, " \
        "bev" SALIS_EVENT_ARRAY_SIZE_COL_MARKER #i " int not null, bev_" #i " blob, "
        FOR_CORES
#undef FOR_CORE
        "step int not null"
        ");"
    );

    arch_push_data_header();
}
#endif

static int salis_measure_ambs_thread(struct Core *core) {
    assert(core);
    core->amb0 = 0.;
    core->amb1 = 0.;

    for (uint64_t j = core->pfst; j < core->plst; j++) {
        core->amb0 += (double)arch_proc_get_mb0_size(core, j);
        core->amb1 += (double)arch_proc_get_mb1_size(core, j);
    }

    core->amb0 /= core->pnum;
    core->amb1 /= core->pnum;
    return 0;
}

static void salis_prepare_data_line(void) {
    assert(g_sim_db);

    // Measure average memory block sizes and compress event-arrays in parallel
    for (int i = 0; i < CORES; i++) {
        struct Core *core = &g_cores[i];
        thrd_create(&core->amb_thrd, (thrd_start_t)salis_measure_ambs_thread, core);
        thrd_create(&core->aeva.thrd, (thrd_start_t)comp_deflate, &core->aeva.params);
        thrd_create(&core->eeva.thrd, (thrd_start_t)comp_deflate, &core->eeva.params);
        thrd_create(&core->beva.thrd, (thrd_start_t)comp_deflate, &core->beva.params);
    }

    for (int i = 0; i < CORES; i++) {
        struct Core *core = &g_cores[i];
        thrd_join(core->amb_thrd, NULL);
        thrd_join(core->aeva.thrd, NULL);
        thrd_join(core->eeva.thrd, NULL);
        thrd_join(core->beva.thrd, NULL);
        core->aeva.blob_size = core->aeva.params.strm.total_out;
        core->eeva.blob_size = core->eeva.params.strm.total_out;
        core->beva.blob_size = core->beva.params.strm.total_out;
    }

    arch_prepare_data_line();
}

static void salis_push_data_line(void) {
    assert(g_sim_db);
    assert(g_step % DATA_PUSH_INTERVAL == 0);
    salis_prepare_data_line();
    log_info("Pushing row to core table in SQLite database");
    sql_exec(
        NULL, NULL,
        "insert into core ("
#define FOR_CORE(i) \
        "step_" #i ", " \
        "cycl_" #i ", " \
        "mall_" #i ", " \
        "pnum_" #i ", " \
        "pfst_" #i ", " \
        "plst_" #i ", " \
        "amb0_" #i ", " \
        "amb1_" #i ", " \
        "emb0_" #i ", " \
        "emb1_" #i ", " \
        "eliv_" #i ", " \
        "edea_" #i ", " \
        "aev" SALIS_EVENT_ARRAY_SIZE_COL_MARKER #i ", " \
        "eev" SALIS_EVENT_ARRAY_SIZE_COL_MARKER #i ", " \
        "bev" SALIS_EVENT_ARRAY_SIZE_COL_MARKER #i ", "
        FOR_CORES
#undef FOR_CORE
        "step"
        ") values ("
#define FOR_CORE(i) \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%f, " \
        "%f, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, " \
        "%ld, "
        FOR_CORES
#undef FOR_CORE
        "%ld"
        ");",
#define EVENT_ARRAY(core, index, ev) \
        g_cores[core].ev##_blob_size,
#define FOR_CORE(i) \
        g_cores[i].step, \
        g_cores[i].cycl, \
        g_cores[i].mall, \
        g_cores[i].pnum, \
        g_cores[i].pfst, \
        g_cores[i].plst, \
        g_cores[i].amb0, \
        g_cores[i].amb1, \
        g_cores[i].emb0, \
        g_cores[i].emb1, \
        g_cores[i].eliv, \
        g_cores[i].edea, \
        g_cores[i].aeva.blob_size, \
        g_cores[i].eeva.blob_size, \
        g_cores[i].beva.blob_size,
        FOR_CORES
#undef FOR_CORE
        g_step
    );

    // Store EVA data
    int rem = snprintf(g_evas_pbuf, EVA_SAVE_NAME_LEN, "%s/evas-%016lx", SIM_EVAS, g_step);
    assert(rem >= 0);
    assert(rem < EVA_SAVE_NAME_LEN);
    (void)rem;

    log_info("Saving event-array data to file: %s", g_evas_pbuf);
    FILE *eva_file = fopen(g_evas_pbuf, "wb");
    assert(eva_file);

    for (int i = 0; i < CORES; i++) {
        struct Core *core = &g_cores[i];
        fwrite(core->aeva.comp, sizeof(Bytef), core->aeva.blob_size, eva_file);
        fwrite(core->eeva.comp, sizeof(Bytef), core->eeva.blob_size, eva_file);
        fwrite(core->beva.comp, sizeof(Bytef), core->beva.blob_size, eva_file);
        comp_deflate_end(&core->aeva.params);
        comp_deflate_end(&core->eeva.params);
        comp_deflate_end(&core->beva.params);
    }

    arch_push_data_line(eva_file);
    fclose(eva_file);
}

#if defined(COMMAND_NEW)
void salis_init(void) {
    log_info("Creating a new simulation");

    uint64_t seed = SEED;

    for (int i = 0; i < CORES; i++) {
        core_init(&g_cores[i], &seed);
    }

    sql_open();
    salis_push_data_header();
    salis_push_data_line();
    salis_auto_save();
}
#endif

#if defined(COMMAND_LOAD)
void salis_load(void) {
    log_info("Loading simulation from: %s", SIM_PATH);
    FILE *fx = fopen(SIM_PATH, "rb");
    assert(fx);
    fseek(fx, 0, SEEK_END);
    size_t x_size = ftell(fx) - sizeof(size_t);
    char *in = malloc(x_size);
    rewind(fx);
    assert(x_size);
    assert(in);

    size_t size = 0;
    fread(&size, sizeof(size_t), 1, fx);
    fread(in, 1, x_size, fx);
    fclose(fx);
    assert(size);
    char *out = malloc(size);
    assert(out);

    struct InflateParams params = {
        .avail_in = x_size,
        .size = size,
        .in = (Bytef *)in,
        .out = (Bytef *)out,
    };

    comp_inflate(&params);
    comp_inflate_end(&params);
    FILE *f = fmemopen(out, size, "rb");
    assert(f);

    for (int i = 0; i < CORES; i++) {
        core_load(&g_cores[i], f);
    }

    fread(&g_step, sizeof(uint64_t), 1, f);
    fread(&g_sync, sizeof(uint64_t), 1, f);
    fclose(f);
    free(in);
    free(out);

    sql_open();
}
#endif

static void salis_check_intervals(void) {
    if (g_step % SYNC_INTERVAL == 0) {
        salis_sync();
    }

    if (g_step % AUTOSAVE_INTERVAL == 0) {
        salis_auto_save();
    }

    if (g_step % DATA_PUSH_INTERVAL == 0) {
        salis_push_data_line();
    }
}

#if !defined(NDEBUG)
static void salis_validate(void) {
    log_trace("Validating simulation state");
    assert(g_step / SYNC_INTERVAL == g_sync);

    for (int i = 0; i < CORES; i++) {
        core_validate(&g_cores[i]);
    }
}
#endif

static int salis_thread(void *data) {
    assert(!data);
    assert(atomic_load_explicit(&g_running, memory_order_relaxed));
    (void)data;

    struct timespec time_start;
    clock_gettime(CLOCK_MONOTONIC_COARSE, &time_start);
    uint64_t step_start = g_step;

    while (atomic_load_explicit(&g_running, memory_order_relaxed)) {
        log_trace("Launching core threads");

        for (int i = 0; i < CORES; i++) {
            struct Core *core = &g_cores[i];
            thrd_create(&core->thrd, (thrd_start_t)core_thread, core);
        }

        for (int i = 0; i < CORES; i++) {
            struct Core *core = &g_cores[i];
            thrd_join(core->thrd, NULL);
        }

        // Align step-count on all cores
        for (int i = 0; i < CORES; i++) {
            struct Core *core = &g_cores[i];

            if (core->step > g_step) {
                g_step = core->step;
            }
        }

        for (int i = 0; i < CORES; i++) {
            struct Core *core = &g_cores[i];

            while (core->step < g_step) {
                core_step(core);
            }
        }

        salis_check_intervals();

#if !defined(NDEBUG)
        salis_validate();
#endif

        // Log simulation steps/s
        struct timespec time_end;
        clock_gettime(CLOCK_MONOTONIC_COARSE, &time_end);
        int time_delta = time_end.tv_sec - time_start.tv_sec;

        if (time_delta > 0) {
            uint64_t step_end = g_step;
            uint64_t step_delta = step_end - step_start;
            time_start = time_end;
            step_start = step_end;
            log_info("Running simulation @%#018lx steps/s", step_delta);
        }
    }

    return 0;
}

void salis_start(void) {
    assert(!atomic_load_explicit(&g_running, memory_order_relaxed));
    log_trace("Launching main thread");
    atomic_store_explicit(&g_running, true, memory_order_relaxed);
    thrd_create(&g_thrd, (thrd_start_t)salis_thread, NULL);
}

void salis_join(void) {
    assert(atomic_load_explicit(&g_running, memory_order_relaxed));
    log_trace("Waiting for main thread");
    thrd_join(g_thrd, NULL);
}

void salis_signal_stop(void) {
    assert(atomic_load_explicit(&g_running, memory_order_relaxed));
    log_trace("Signaling main thread to stop");
    atomic_store_explicit(&g_running, false, memory_order_relaxed);
}

void salis_step(uint64_t steps) {
    assert(!atomic_load_explicit(&g_running, memory_order_relaxed));
    log_trace("Stepping simulation %lu times", steps);

    for (uint64_t i = 0; i < steps; i++) {
        for (int i = 0; i < CORES; i++) {
            core_step(&g_cores[i]);
        }

        g_step++;

        salis_check_intervals();
    }

#if !defined(NDEBUG)
    salis_validate();
#endif
}

void salis_free(void) {
    assert(!atomic_load_explicit(&g_running, memory_order_relaxed));
    log_info("Freeing simulation resources");
    salis_save(SIM_PATH);
    sql_close();

    for (int i = 0; i < CORES; i++) {
        core_free(&g_cores[i]);
    }
}