database_type   = 'mysql';
dataBase   = 'labor_highspeed';%'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor\silas_labor_newdsp_newstructure.db';
db = DBHandler("dataBase", [dataBase], "type", database_type);

fp = QueryFilter();
% fp.where('Runs', 'run_id','EQUALS', 987);
M = 6;
fp.where('Runs', 'pam_level','EQUALS', M);
% fp.where('Runs', 'bitrate','LESS_THAN', 310e9);
fp.where('Runs', 'fiber_length','EQUALS', 2);
fp.where('Runs', 'is_mpi','EQUALS', 0);
% fp.where('Runs', 'interference_path_length','EQUALS', 1000);
% fp.where('Runs', 'loop_id','GREATER_THAN', 11);
% fp.where('Runs', 'sir','EQUALS',18);
fp.where('Runs', 'wavelength','EQUALS', 1310);
% fp.where('Runs', 'db_mode','EQUALS', 0);
fp.where('Runs', 'rop_attenuation','EQUALS', 0);

fields =  db.getTableFieldNames('power_state_info');
fields = [fields; db.getTableFieldNames('dashboard_ungrouped_aug_nov_2025')];
[dataTable,~] = db.queryDB(fp, fields);

eqstructures = unique(dataTable.equalizer_structure);
% Create the figure
showFiltered = true;
showPrecoded = false;
show_bitrate = true;
figure(5);
hold on

for eqs = [equalizer_structure.vnle]
    
    % figure('Name',string([char(eqs),'']));
    % hold on

    for pre_emph = [0,1]

        dbmode_filtered = dataTable(dataTable.db_mode == ~pre_emph,:);

        eq_choice = equalizer_structure(eqs);

        if sum(eqstructures == eq_choice)~=1
            disp(eq_choice)
            continue
        end

        eq_filtered = dbmode_filtered(dbmode_filtered.equalizer_structure == eq_choice,:);

        % ===== NEW: compute averages + per-row keep masks (robust filtering) =====
        [Tav, keepMask, keepMaskP] = avgBerBySymbolrate(eq_filtered);   % <= NEW

        % x-values (bitrate) for raw points (same mapping as your lines)
        M = unique(eq_filtered.pam_level);   % (assumes single PAM per curve)
        if show_bitrate
            x_raw = eq_filtered.symbolrate.*1e-9 .* floor(log2(M)*10)/10;
        else
            x_raw = eq_filtered.symbolrate.*1e-9;
        end

        % ===== NEW: scatter kept raw BER points (hidden from legend) =====
        cols = cbrewer2('Paired',12);
        thisColor = cols((2*eqs)+1+pre_emph,:);
        scatter(x_raw(keepMask), ...            % kept points
            eq_filtered.BER(keepMask), ...
            14, thisColor, 'filled', ...
            'MarkerFaceAlpha', 0.35, ...
            'MarkerEdgeAlpha', 0.35, ...
            'HandleVisibility','off');

        if showPrecoded
        scatter(x_raw(keepMaskP), ...           % kept precoded points
            eq_filtered.BER_precoded(keepMaskP), ...
            14, thisColor, 'filled', ...
            'Marker', 'square', ...
            'MarkerFaceAlpha', 0.35, ...
            'MarkerEdgeAlpha', 0.35, ...
            'HandleVisibility','off');
        end

        % ===== NEW: optionally show filtered-out points in red =====
        if showFiltered
            bad = ~keepMask;
            if any(bad)
                scatter(x_raw(bad), eq_filtered.BER(bad), ...
                    18, 'r', 'x', 'LineWidth', 1.2, ...
                    'HandleVisibility','off');
            end
            badp = ~keepMaskP;
            if any(badp)
                scatter(x_raw(badp), eq_filtered.BER_precoded(badp), ...
                    18, 'r', '+', 'LineWidth', 1.2, ...
                    'HandleVisibility','off');
            end
        end

        % Keep your sorting and one-per-symbolrate behavior (using Tav)
        symbolrate_sorted = sortrows(Tav,{'symbolrate','avg_BER_calc'}, 'ascend');
        [~, ia] = unique(symbolrate_sorted.symbolrate, 'first');
        symbolrate_sorted = symbolrate_sorted(ia, :);

        if show_bitrate
            % Bitrate for the averaged curves (unchanged)
            xraw = symbolrate_sorted.symbolrate.*1e-9 .* floor(log2(M)*10)/10;
        else
            xraw = symbolrate_sorted.symbolrate.*1e-9;
        end
        % Use the MATLAB-averaged BERs
        ber          = symbolrate_sorted.avg_BER_calc;
        ber_precoded = symbolrate_sorted.avg_BER_precoded_calc;


        dname = strrep([char(eq_choice)],'_',' ');
        if pre_emph
            dname = [dname,' with pre-emph.'];
        else
            dname = [dname,'  w/o pre-emph.'];
        end

        plot(xraw, ber, ...
            'LineWidth', 1.5, 'MarkerSize', 5, ...
            'Marker','o','LineStyle','-', ...
            'Color',thisColor,'MarkerEdgeColor',thisColor,'MarkerFaceColor',[1,1,1], ...
            'DisplayName', dname);

        if showPrecoded
            plot(xraw, ber_precoded, ...
                'LineWidth', 1.5, 'MarkerSize', 5, ...
                'Marker','square','LineStyle',':', ...
                'Color',thisColor,'MarkerEdgeColor',thisColor,'MarkerFaceColor',[1,1,1], ...
                'DisplayName', [dname,'; pre-coded']);
        end

        grid on;

        if show_bitrate
            xlabel('Net bitrate [GBps]', 'FontSize', 12);
        else
            xlabel('Symbol rate [GBd]', 'FontSize', 12);
        end
        ylabel('BER',             'FontSize', 12);
        title('BER vs. Baud Rate','FontSize', 14, 'FontWeight', 'bold');

        set(gca, 'XScale', 'linear', ...
            'YScale', 'log', ...
            'TickLabelInterpreter', 'latex', ...
            'FontSize', 11);
        
        xticks(xraw);
        if show_bitrate
        %     xticks(200:25:500);
        %     xlim([350 500]);
            xlim([min(xraw), max(xraw)]);
        else
            xlim([min(xraw), max(xraw)]);
        end

        ylim([1e-4, 0.5]);

    end
    yline([2.2e-4, 4.85e-3, 2e-2],'LineWidth',1,'LineStyle','--','HandleVisibility','off');
end

function [Tav, keepAll, keepAllP] = avgBerBySymbolrate(T, ZT, MIN_G)
% Minimal robust averaging of BER per symbolrate (+ masks for kept points).
% Usage: [Tav, keepAll, keepAllP] = avgBerBySymbolrate(T, ZT, MIN_G)
% Defaults: ZT=3 (MAD z-thresh in log10), MIN_G=2 (min points to filter)

    if nargin < 2, ZT = 5; end
    if nargin < 5, MIN_G = 0; end

    hasP  = ismember('BER_precoded', T.Properties.VariableNames);
    hasNB = ismember('numBits',      T.Properties.VariableNames);

    [G,~,idx] = unique(T.symbolrate);
    nG = numel(G);

    avgBER  = nan(nG,1);
    avgBERp = nan(nG,1);
    keepAll  = false(height(T),1);
    keepAllP = false(height(T),1);

    for gi = 1:nG
        r = idx==gi;

        x  = T.BER(r);
        nb = hasNB * T.numBits(r) + ~hasNB; % if missing, nb==1 (scalar expansion ok)

        [avgBER(gi), keepAll(r)] = rmeanBer(x, nb, ZT, MIN_G);

        if hasP
            xp = T.BER_precoded(r);
            [avgBERp(gi), keepAllP(r)] = rmeanBer(xp, nb, ZT, MIN_G);
        end
    end

    Tav = table(G, avgBER, avgBERp, ...
        'VariableNames', {'symbolrate','avg_BER_calc','avg_BER_precoded_calc'});
end

function [mu, keep] = rmeanBer(x, nb, ZT, MIN_G, onlyHighOutliers, minKeepThreshold)
% Robust arithmetic mean of BER with log-domain MAD filtering (returns keep mask)
%
% Params:
%   x                : BER values
%   nb               : numBits (for floor)
%   ZT               : MAD z-threshold
%   MIN_G            : min group size before filtering
%   onlyHighOutliers : (bool) if true, only discard values above mean
%   minKeepThreshold : values below this BER are always kept
%
% Returns:
%   mu   : robust mean
%   keep : logical mask of kept samples

    if nargin < 5, onlyHighOutliers = false; end
    if nargin < 6, minKeepThreshold = 0; end

    x(~isfinite(x)) = NaN;

    if ~isscalar(nb), nb(~isfinite(nb)) = NaN; end
    if isscalar(nb) && ~isfinite(nb), nb = 1; end

    floorVal = realmin;
    if ~isscalar(nb) || (isscalar(nb) && isfinite(nb) && nb~=1)
        fv = 0.5 ./ max(nb, eps);         % rule-of-three style floor
        if isscalar(fv), floorVal = fv; else, floorVal = fv; end
    end

    xAdj = x;
    bad  = ~isfinite(xAdj) | xAdj <= 0;
    if isscalar(floorVal)
        xAdj(bad) = floorVal;
    else
        xAdj(bad) = floorVal(bad);
    end

    valid = isfinite(xAdj) & xAdj > 0;
    keep  = false(size(xAdj));

    if nnz(valid)==0
        mu = NaN; return
    end
    if nnz(valid) < MIN_G
        mu = mean(xAdj(valid),'omitnan'); keep(valid)=true; return
    end

    lx  = log10(xAdj(valid));
    med = median(lx,'omitnan');
    mad = median(abs(lx-med),'omitnan');

    if mad<=0 || ~isfinite(mad)
        keep(valid) = true;
        mu = mean(xAdj(valid),'omitnan');
        return
    end

    sigma = 1.4826*mad;
    ksel  = abs(lx-med) <= ZT*sigma;

    % convert to linear indices
    vIdx = find(valid);

    % === Extension A: only drop high outliers ===
    if onlyHighOutliers
        logMean = mean(lx,'omitnan');
        highIdx = lx > logMean;
        ksel = ksel | ~highIdx;  % always keep values below/equal to mean
    end

    % === Extension B: always keep values below minKeepThreshold ===
    belowThr = xAdj(valid) < minKeepThreshold;
    ksel = ksel | belowThr;

    keep(vIdx(ksel)) = true;

    if any(keep)
        mu = mean(xAdj(keep),'omitnan');
    else
        mu = mean(xAdj(valid),'omitnan');
        keep(valid) = true;
    end
end