% === DSP settings ===
dsp_options = struct();
dsp_options.mode = "run_id";
dsp_options.recipe = @mpi_recipe_dev;
dsp_options.append_to_db = false;
dsp_options.start_occurence = 1;
dsp_options.max_occurences = 15;
dsp_options.debug_plots = false;

dsp_options.database_type = "mysql";

dsp_options.dataBase = "labor";
dsp_options.storage_path = "W:\labdata\ECOC Silas\ecoc_2025";

dsp_options.server = "192.168.178.192";
dsp_options.port = 3306;
dsp_options.user = "silas";
dsp_options.password = "silas";

db = DBHandler("dataBase", [dsp_options.dataBase], ...
    "type", dsp_options.database_type, ...
    "server", dsp_options.server, ...
    "user", dsp_options.user, "password", dsp_options.password);

scriptDir = fileparts(mfilename("fullpath"));

whpam4_loaded = load(fullfile(scriptDir, "pam_4_results.mat"));
whpam6_loaded = load(fullfile(scriptDir, "pam_6_results.mat"));
whpam8_loaded = load(fullfile(scriptDir, "pam_8_results.mat"));

whpam4 = whpam4_loaded.obj;
whpam6 = whpam6_loaded.obj;
whpam8 = whpam8_loaded.obj;

whList = {whpam4, whpam6, whpam8};
algorithmStorageNames = fieldnames(whpam4.sto);

%% Build a lossless run_id warehouse and cache run metadata

pamformats = [4, 6, 8];
baudrates = [112e9, 96e9, 72e9];
runMetaTable = queryRunMetadata(db, whList, pamformats, baudrates);

wh_run_id_combined = mergeDataStoragesByUnion(whList);
wh_run_id_combined = addRunMetadataStorages(wh_run_id_combined, runMetaTable);

runIdSavePath = fullfile(scriptDir, "combined_by_run_id_results.mat");
save(runIdSavePath, "wh_run_id_combined", "runMetaTable", "algorithmStorageNames", '-v7.3');

fprintf("Saved run_id-combined warehouse:\n  %s\n", runIdSavePath);
wh_run_id_combined.showInfo;

%% Build a grouped config warehouse for analysis views

configFields = ["pam_level", "symbolrate", "interference_path_length", "sir"];
wh_config_combined = buildConfigWarehouse( ...
    wh_run_id_combined, runMetaTable, algorithmStorageNames, configFields);

configSavePath = fullfile(scriptDir, "combined_by_config_results.mat");
save(configSavePath, "wh_config_combined", "runMetaTable", "algorithmStorageNames", '-v7.3');

fprintf("Saved config-grouped warehouse:\n  %s\n", configSavePath);
wh_config_combined.showInfo;

%% BER over SIR at fixed path length

plot_options = struct();
plot_options.path_length = 1000;
plot_options.use_boundedlines = true;
plot_options.boundedline_alpha = 0.10;
plot_options.polyfit_order_max = 3;
plot_options.fec_ber_threshold = 2e-2;
plot_options.figure_base = 7100;
plot_options.xlim = [15 35];

plotBerOverSirForPath(wh_config_combined, algorithmStorageNames, plot_options);

%% Local functions

function runMetaTable = queryRunMetadata(db, whList, pamformats, baudrates)
metaTables = {};
metadataFields = db.getTableFieldNames('Runs');

for whIdx = 1:numel(whList)
    curWh = whList{whIdx};
    curRunIds = curWh.parameter.run_id.values(:);

    fp = QueryFilter();
    fp.where('Runs', 'run_id', 'GREATER_EQUAL', 3153);
    fp.where('Runs', 'symbolrate', 'EQUALS', baudrates(whIdx));
    fp.where('Runs', 'fiber_length', 'EQUALS', 0);
    fp.where('Runs', 'db_mode', 'EQUALS', '"no_db"');
    fp.where('Runs', 'pam_level', 'EQUALS', pamformats(whIdx));

    [dataTable, ~] = db.queryDB(fp, metadataFields);
    dataTable.run_id = numericColumn(dataTable.run_id);
    dataTable = dataTable(ismember(dataTable.run_id, curRunIds), :);

    missingRunIds = setdiff(curRunIds, dataTable.run_id);
    if ~isempty(missingRunIds)
        warning("recombine_warehouses:MissingMetadata", ...
            "%d run_id(s) from PAM %.0f warehouse were not found in the metadata query.", ...
            numel(missingRunIds), pamformats(whIdx));
    end

    metaTables{end+1} = dataTable; %#ok<AGROW>
end

runMetaTable = vertcat(metaTables{:});
[~, uniqueIdx] = unique(numericColumn(runMetaTable.run_id), "stable");
runMetaTable = runMetaTable(uniqueIdx, :);

numericFields = ["run_id", "pam_level", "symbolrate", "interference_path_length", "sir"];
for fieldIdx = 1:numel(numericFields)
    fieldName = numericFields(fieldIdx);
    if ismember(fieldName, string(runMetaTable.Properties.VariableNames))
        runMetaTable.(char(fieldName)) = numericColumn(runMetaTable.(char(fieldName)));
    end
end

[~, sortIdx] = sort(runMetaTable.run_id);
runMetaTable = runMetaTable(sortIdx, :);
end

function x = numericColumn(x)
if isnumeric(x)
    x = double(x);
elseif iscell(x)
    x = str2double(string(x));
elseif isstring(x) || ischar(x) || iscategorical(x)
    x = str2double(string(x));
else
    x = double(x);
end
x = x(:);
end

function whMerged = mergeDataStoragesByUnion(whList)
templateWh = whList{1};
paramNames = cellstr(templateWh.fn);

for whIdx = 2:numel(whList)
    if ~isequal(sort(cellstr(whList{whIdx}.fn)), sort(paramNames))
        error("recombine_warehouses:ParameterMismatch", ...
            "All input warehouses must use the same parameter names.");
    end
end

mergedParams = struct();
for paramIdx = 1:numel(paramNames)
    paramName = paramNames{paramIdx};
    values = [];

    for whIdx = 1:numel(whList)
        newValues = whList{whIdx}.parameter.(paramName).values(:).';
        values = [values, newValues]; %#ok<AGROW>
    end

    values = unique(values, "stable");
    if isnumeric(values)
        values = sort(values);
    end
    mergedParams.(paramName) = values;
end

whMerged = DataStorage(mergedParams);

for whIdx = 1:numel(whList)
    curWh = whList{whIdx};
    curStorageNames = fieldnames(curWh.sto);

    for storageIdx = 1:numel(curStorageNames)
        storageName = curStorageNames{storageIdx};
        if ~isfield(whMerged.sto, storageName)
            whMerged.addStorage(storageName);
        end

        for linIdx = 1:numel(curWh.sto.(storageName))
            storedValue = curWh.sto.(storageName){linIdx};
            if isempty(storedValue)
                continue
            end

            targetLinIdx = mapLinearIndex(curWh, whMerged, linIdx);
            existingValue = whMerged.sto.(storageName){targetLinIdx};
            whMerged.sto.(storageName){targetLinIdx} = mergeStoredValue(existingValue, storedValue);
        end
    end
end
end

function wh = addRunMetadataStorages(wh, runMetaTable)
metadataStorageNames = ["meta_pam_level", "meta_symbolrate", ...
    "meta_interference_path_length", "meta_sir"];
metadataTableFields = ["pam_level", "symbolrate", "interference_path_length", "sir"];

for fieldIdx = 1:numel(metadataStorageNames)
    if ~isfield(wh.sto, metadataStorageNames(fieldIdx))
        wh.addStorage(char(metadataStorageNames(fieldIdx)));
    end
end

for linIdx = 1:wh.getLastLinIndice()
    physStruct = linearIndexToStruct(wh, linIdx);
    if ~isfield(physStruct, "run_id")
        continue
    end

    metaIdx = find(runMetaTable.run_id == physStruct.run_id, 1);
    if isempty(metaIdx)
        continue
    end

    for fieldIdx = 1:numel(metadataStorageNames)
        wh.sto.(char(metadataStorageNames(fieldIdx))){linIdx} = ...
            runMetaTable.(char(metadataTableFields(fieldIdx)))(metaIdx);
    end
end
end

function whConfig = buildConfigWarehouse(whRun, runMetaTable, algorithmStorageNames, configFields)
runParamNames = cellstr(whRun.fn);
extraParamNames = setdiff(string(runParamNames), "run_id", "stable");

configParams = struct();
for fieldIdx = 1:numel(configFields)
    fieldName = configFields(fieldIdx);
    values = unique(runMetaTable.(char(fieldName))(:).', "stable");
    if isnumeric(values)
        values = sort(values);
    end
    configParams.(char(fieldName)) = values;
end

for paramIdx = 1:numel(extraParamNames)
    paramName = extraParamNames(paramIdx);
    configParams.(char(paramName)) = whRun.parameter.(char(paramName)).values;
end

whConfig = DataStorage(configParams);
for storageIdx = 1:numel(algorithmStorageNames)
    whConfig.addStorage(algorithmStorageNames{storageIdx});
end
whConfig.addStorage("run_id");

for storageIdx = 1:numel(algorithmStorageNames)
    storageName = algorithmStorageNames{storageIdx};

    for linIdx = 1:numel(whRun.sto.(storageName))
        storedValue = whRun.sto.(storageName){linIdx};
        if isempty(storedValue)
            continue
        end

        sourcePhys = linearIndexToStruct(whRun, linIdx);
        metaIdx = find(runMetaTable.run_id == sourcePhys.run_id, 1);
        if isempty(metaIdx)
            continue
        end

        targetArgs = configTargetArgs(whConfig, sourcePhys, runMetaTable, metaIdx, configFields);
        targetLinIdx = whConfig.getIndicesByPhys(targetArgs);

        existingValue = whConfig.sto.(storageName){targetLinIdx};
        whConfig.sto.(storageName){targetLinIdx} = mergeStoredValue(existingValue, storedValue);
    end
end

for linIdx = 1:whRun.getLastLinIndice()
    sourcePhys = linearIndexToStruct(whRun, linIdx);
    metaIdx = find(runMetaTable.run_id == sourcePhys.run_id, 1);
    if isempty(metaIdx)
        continue
    end

    targetArgs = configTargetArgs(whConfig, sourcePhys, runMetaTable, metaIdx, configFields);
    targetLinIdx = whConfig.getIndicesByPhys(targetArgs);
    whConfig.sto.run_id{targetLinIdx} = mergeUniqueNumeric( ...
        whConfig.sto.run_id{targetLinIdx}, sourcePhys.run_id);
end
end

function targetArgs = configTargetArgs(whConfig, sourcePhys, runMetaTable, metaIdx, configFields)
targetArgs = cell(1, numel(whConfig.fn));

for paramIdx = 1:numel(whConfig.fn)
    paramName = string(whConfig.fn(paramIdx));
    if any(configFields == paramName)
        targetArgs{paramIdx} = runMetaTable.(char(paramName))(metaIdx);
    else
        targetArgs{paramIdx} = sourcePhys.(char(paramName));
    end
end
end

function targetLinIdx = mapLinearIndex(sourceWh, targetWh, sourceLinIdx)
sourcePhys = linearIndexToStruct(sourceWh, sourceLinIdx);
targetArgs = cell(1, numel(targetWh.fn));

for paramIdx = 1:numel(targetWh.fn)
    paramName = char(targetWh.fn(paramIdx));
    targetArgs{paramIdx} = sourcePhys.(paramName);
end

targetLinIdx = targetWh.getIndicesByPhys(targetArgs);
end

function physStruct = linearIndexToStruct(wh, linIdx)
[physValues, physNames] = wh.getPhysIndicesByLinIndex(linIdx);
physStruct = struct();

for paramIdx = 1:numel(physNames)
    physStruct.(char(physNames{paramIdx})) = physValues{paramIdx};
end
end

function out = mergeStoredValue(existingValue, newValue)
if isempty(existingValue)
    out = newValue;
    return
end

existingPackages = asPackageCell(existingValue);
newPackages = asPackageCell(newValue);
out = [existingPackages(:); newPackages(:)].';
end

function packages = asPackageCell(value)
if isempty(value)
    packages = {};
elseif iscell(value)
    packages = value(:).';
else
    packages = {value};
end
end

function out = mergeUniqueNumeric(existingValue, newValue)
if isempty(existingValue)
    out = newValue;
else
    out = unique([existingValue(:); newValue(:)].', "stable");
end
end

function plotBerOverSirForPath(whConfig, algorithmStorageNames, plotOptions)
paramNames = string(whConfig.fn);

if ~any(paramNames == "pam_level") || ~any(paramNames == "interference_path_length") || ~any(paramNames == "sir")
    error("recombine_warehouses:MissingPlotParameters", ...
        "The config warehouse must contain pam_level, interference_path_length, and sir parameters.");
end

pamLevels = whConfig.parameter.pam_level.values;
hasBlockUpdate = any(paramNames == "block_update");
if hasBlockUpdate
    blockUpdates = whConfig.parameter.block_update.values;
else
    blockUpdates = NaN;
end

if exist("linspecer", "file")
    curveColors = linspecer(max(numel(algorithmStorageNames), 1));
else
    curveColors = lines(max(numel(algorithmStorageNames), 1));
end

figureIdx = plotOptions.figure_base;
for blockIdx = 1:numel(blockUpdates)
    for pamIdx = 1:numel(pamLevels)
        pamLevel = pamLevels(pamIdx);
        figureIdx = figureIdx + 1;
        figure(figureIdx); clf; hold on

        for storageIdx = 1:numel(algorithmStorageNames)
            storageName = algorithmStorageNames{storageIdx};
            curveColor = curveColors(storageIdx, :);

            filter = struct();
            filter.pam_level = pamLevel;
            filter.interference_path_length = plotOptions.path_length;
            if hasBlockUpdate
                filter.block_update = blockUpdates(blockIdx);
            end

            [sirValues, berGroups] = collectBerBySir(whConfig, storageName, filter);
            if isempty(sirValues)
                continue
            end

            [sirValues, sortIdx] = sort(sirValues);
            berGroups = berGroups(sortIdx);

            berAvg = nan(1, numel(sirValues));
            berMin = nan(1, numel(sirValues));
            berMax = nan(1, numel(sirValues));

            for sirIdx = 1:numel(sirValues)
                rawBer = berGroups{sirIdx};
                rawBer = rawBer(isfinite(rawBer));
                if isempty(rawBer)
                    continue
                end

                scatter(repmat(sirValues(sirIdx), size(rawBer)), rawBer, ...
                    12, ...
                    "Marker", ".", ...
                    "MarkerEdgeColor", curveColor, ...
                    "MarkerFaceColor", curveColor, ...
                    "HandleVisibility", "off");

                berAvg(sirIdx) = mean(rawBer, "omitnan");
                berMin(sirIdx) = min(rawBer);
                berMax(sirIdx) = max(rawBer);
            end

            validAvg = isfinite(sirValues) & isfinite(berAvg);
            if ~any(validAvg)
                continue
            end

            if plotOptions.use_boundedlines && exist("boundedline", "file")
                yLower = max(berAvg - berMin, 0);
                yUpper = max(berMax - berAvg, 0);
                yBounds = [yLower(:), yUpper(:)];

                [hl, hp] = boundedline(sirValues(:), berAvg(:), yBounds, ...
                    "alpha", "transparency", plotOptions.boundedline_alpha, ...
                    "cmap", curveColor, ...
                    "nan", "fill", ...
                    "orientation", "vert");
                set(hl, "LineStyle", "none", "Marker", "none", "HandleVisibility", "off");
                set(hp, "HandleVisibility", "off", "LineStyle", "none");
            end

            scatter(sirValues(validAvg), berAvg(validAvg), ...
                54, ...
                "Marker", "o", ...
                "MarkerEdgeColor", curveColor, ...
                "MarkerFaceColor", "none", ...
                "LineWidth", 1.2, ...
                "DisplayName", storageName);

            fitMask = validAvg & berAvg > 0;
            if nnz(fitMask) >= 2
                fitOrder = min(plotOptions.polyfit_order_max, nnz(fitMask) - 1);
                fitCoeff = polyfit(sirValues(fitMask), log10(berAvg(fitMask)), fitOrder);
                xFit = linspace(min(sirValues(fitMask)), max(sirValues(fitMask)), 300);
                yFit = 10 .^ polyval(fitCoeff, xFit);

                plot(xFit, yFit, ...
                    "LineWidth", 1.2, ...
                    "LineStyle", "--", ...
                    "Marker", "none", ...
                    "Color", curveColor, ...
                    "HandleVisibility", "off");
            end
        end

        if hasBlockUpdate
            title(sprintf("BER over SIR, PAM %.0f, path %.0f m, block update = %g", ...
                pamLevel, plotOptions.path_length, blockUpdates(blockIdx)));
        else
            title(sprintf("BER over SIR, PAM %.0f, path %.0f m", ...
                pamLevel, plotOptions.path_length));
        end
        xlabel("SIR (dB)");
        ylabel("BER");
        xlim(plotOptions.xlim);
        grid on
        box on

        if exist("beautifyBERplot", "file")
            beautifyBERplot("logscale", true, "setcolors", false, "setmarkers", false);
        else
            set(gca, "YScale", "log");
        end
        legend("Location", "best", "Interpreter", "none");
    end
end
end

function [sirValues, berGroups] = collectBerBySir(whConfig, storageName, filter)
sirValues = [];
berGroups = {};

for linIdx = 1:numel(whConfig.sto.(storageName))
    storedValue = whConfig.sto.(storageName){linIdx};
    if isempty(storedValue)
        continue
    end

    physStruct = linearIndexToStruct(whConfig, linIdx);
    if ~matchesFilter(physStruct, filter)
        continue
    end

    berValues = extractBerValues(storedValue);
    if isempty(berValues)
        continue
    end

    sirValue = physStruct.sir;
    sirIdx = find(sirValues == sirValue, 1);
    if isempty(sirIdx)
        sirValues(end+1) = sirValue; %#ok<AGROW>
        berGroups{end+1} = berValues; %#ok<AGROW>
    else
        berGroups{sirIdx} = [berGroups{sirIdx}, berValues]; %#ok<AGROW>
    end
end
end

function tf = matchesFilter(physStruct, filter)
tf = true;
filterFields = fieldnames(filter);

for fieldIdx = 1:numel(filterFields)
    fieldName = filterFields{fieldIdx};
    if ~isfield(physStruct, fieldName) || physStruct.(fieldName) ~= filter.(fieldName)
        tf = false;
        return
    end
end
end

function berValues = extractBerValues(value)
berValues = [];

if isstruct(value)
    if isfield(value, "metrics") && isfield(value.metrics, "BER")
        berValues(end+1) = value.metrics.BER;
    end
elseif iscell(value)
    for valueIdx = 1:numel(value)
        berValues = [berValues, extractBerValues(value{valueIdx})]; %#ok<AGROW>
    end
end
end