Many changes in DBHandler

new general processing structure
just before splitting off the projects folder from this repo
This commit is contained in:
Silas Oettinghaus
2025-05-13 10:24:09 +02:00
parent 727c3d9364
commit 9ce23c4a10
38 changed files with 2440 additions and 1103 deletions

View File

@@ -16,7 +16,7 @@ filterParams.Configurations = struct( ...
'wavelength', 1310, ...
'precomp_amp', [], ...
'signal_attenuation', [], ...
'v_awg', [], ...
'v_awg', [], ...
'v_bias', [] ...
);
@@ -25,7 +25,7 @@ filterParams.Configurations = struct( ...
% % filterParams.EqualizerParameters.dc_buffer_len = 1;
% filterParams.EqualizerParameters.ffe_buffer_len = 1;
% filterParams.EqualizerParameters.smoothing_buffer_len = 4096;
% filterParams.EqualizerParameters.smoothing_buffer_update = 224;
% filterParams.EqualizerParameters.smoothing_buffer_update = 224;
% filterParams.EqualizerParameters.DCmu = 0;
% end
@@ -34,220 +34,291 @@ selectedFields = {'Configurations.run_id' 'Runs.loop_id' 'Runs.date_of_run' 'Run
'EqualizerParameters.equalizer_structure' 'EqualizerParameters.diff_precode' 'EqualizerParameters.eq_id' 'EqualizerParameters.dc_buffer_len' 'EqualizerParameters.ffe_buffer_len' 'EqualizerParameters.smoothing_buffer_len' 'EqualizerParameters.smoothing_buffer_update' 'EqualizerParameters.DCmu' 'Measurements.power_pd_in' ...
'Measurements.power_mpi_interference' 'Measurements.power_mpi_signal' 'Results.BER' 'Results.BER_precoded' 'Results.EVM' 'Results.SNR' 'Results.GMI' 'Results.Alpha' 'Results.date_of_processing'};
% [dataTable_raw,sql_query] = database.queryDB(filterParams, selectedFields);
[dataTable_raw,sql_query] = database.queryDB(filterParams, selectedFields);
%%
dataTable_clean = dataTable_raw;
dataTable_clean.SIR = -7 - round(dataTable_clean.power_mpi_interference);
dataTable_clean.NGMI = dataTable_clean.GMI ./ log2(dataTable_clean.pam_level);
dataTable_clean = cleanUpTable(dataTable_clean);
dataTable_clean(dataTable_clean.BER>0.2,:) = [];
%%
dataTable_clean = dataTable_raw;
dataTable_clean.SIR = round(-7.5 - dataTable_clean.power_mpi_interference);
dataTable_clean.NGMI = dataTable_clean.GMI ./ log2(dataTable_clean.pam_level);
dataTable_clean = cleanUpTable(dataTable_clean);
%%
dataTable = dataTable_clean;
figure(26);
plotBoundaries = 1;
plotRealizations = 0;
cols = linspecer(8); % Ensure color count matches
figure()
tiledlayout(1, 4, 'TileSpacing', 'compact', 'Padding', 'compact');
y_here = 0;
figcnt = 0;
for int_len = [0,50,300,1000]
figcnt = figcnt+1;
% figure(int_len+1);
nexttile;
hold on
mode = 4;
% ideal DC tracking
if 0
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
dataTable = dataTable(dataTable.DCmu == 0.005, :);
cols = cols(1:1+1,:);
method = 'ideal dc tracking';
% slow DC tracking
elseif 1
dataTable = dataTable(dataTable.dc_buffer_len == 224, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
dataTable = dataTable(dataTable.DCmu == 0.005, :);
cols = cols(2:2+1,:);
method = 'parallelized dc tracking';
% slow DC smoothing
elseif 0
for mode = [1,4]
hold on;
dataTable = dataTable_clean;
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 4096, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 224, :);
dataTable = dataTable(dataTable.DCmu == 0, :);
cols = cols(3:3+1,:);
method = 'dc smoothing';
elseif 0
% No compensation method
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
dataTable = dataTable(dataTable.DCmu == 0, :);
cols = cols(4:4+1,:);
method = 'ffe only';
end
dataTable(dataTable.eq_id==0,:) = [];
dataTable(dataTable.equalizer_structure~=1,:) = [];
plotBoundaries = 1;
plotRealizations = 1;
cols = linspecer(8); % Ensure color count matches
% dataTable.interference_path_length(dataTable.interference_path_length < 20, :) = 20;
if mode == 1
% No compensation method
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
dataTable = dataTable(dataTable.DCmu == 0, :);
cols = cols(1:1+1,:);
method = 'ffe only';
dataTable = dataTable(dataTable.interference_path_length == 1000, :);
% slow DC smoothing
elseif mode == 2
% dataTable(dataTable.interference_path_length ~= 50, :) = [];
% dataTable = dataTable(dataTable.interference_path_length < 51, :);
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 4096, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 224, :);
dataTable = dataTable(dataTable.DCmu == 0, :);
cols = cols(4:4+1,:);
method = 'dc smoothing';
% dataTable(dataTable.loop_id<200,:) = [];
% Filter by time
filter_by_time = 0;
if filter_by_time
startTime = datetime('2025-04-20 18:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
stopTime = datetime('2025-04-30 19:30:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
dataTable.date_of_run = datetime(dataTable.date_of_run, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
dataTable.date_of_processing = datetime(dataTable.date_of_processing, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
dataTable = dataTable(dataTable.date_of_processing > startTime, :);
dataTable = dataTable(dataTable.date_of_processing < stopTime, :);
end
% slow DC tracking
elseif mode == 3
% Group by smth
y_var = 'BER';
x_var = 'SIR';
dataTable = dataTable(dataTable.dc_buffer_len == 224, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
dataTable = dataTable(dataTable.DCmu == 0.005, :);
cols = cols(3:3+1,:);
method = 'parallelized dc tracking';
loop_var = 'interference_path_length';
fixedVars = {'equalizer_structure','interference_path_length',x_var};
elseif mode == 4
[dataTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var);
% ideal DC tracking
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
dataTable = dataTable(dataTable.DCmu == 0.005, :);
cols = cols(2:2+1,:);
method = 'ideal dc tracking';
end
dataTableGrpd_mean = groupIt(fixedVars, dataTable, @mean);
dataTableGrpd_min = groupIt(fixedVars, dataTable, @min);
dataTableGrpd_max = groupIt(fixedVars, dataTable, @max);
dataTable(dataTable.eq_id==0,:) = [];
dataTable(dataTable.equalizer_structure~=1,:) = [];
% Create a new figure
mkr = '.';
hold on
% Modify values in 'interference_path_length' where the condition is met
dataTable.interference_path_length(dataTable.interference_path_length < 101 & dataTable.interference_path_length > 1) = 50;
unique_loop_var = unique(dataTable.(loop_var));
dataTable.interference_path_length(dataTable.interference_path_length == 1) = 0;
dataTable = dataTable(dataTable.interference_path_length == int_len, :);
for i = 1:numel(unique_loop_var)
% Prepare filtered data for this loop variable
loopValue = unique_loop_var(i);
dataTable(dataTable.run_id == 3866, :) = [];
dataTable(dataTable.run_id == 3865, :) = [];
dataTable(dataTable.run_id == 3796, :) = [];
dataTable(dataTable.run_id == 3797, :) = [];
dataTable(dataTable.run_id == 3798, :) = [];
dataTable(dataTable.run_id == 4002, :) = [];
dataTable(dataTable.run_id == 4200, :) = [];
dataTable(dataTable.run_id == 4199, :) = [];
loopFiltGrpd = dataTableGrpd_mean.(loop_var) == loopValue;
if ~any(loopFiltGrpd)
continue; % Skip if no data for this loop var
end
% 0
% 1
% 10
% 15
% 20
% 50
% 100
% 300
% 1000
% Extract values
x_values = dataTableGrpd_mean.(x_var)(loopFiltGrpd, :);
y_mean = dataTableGrpd_mean.(y_var)(loopFiltGrpd, :);
y_min = dataTableGrpd_min.(y_var)(loopFiltGrpd, :);
y_max = dataTableGrpd_max.(y_var)(loopFiltGrpd, :);
% dataTable(dataTable.interference_path_length ~= 50, :) = [];
% dataTable = dataTable(dataTable.interference_path_length < 51, :);
% Compute bounds: distance from mean
y_lower = y_mean - y_min;
y_upper = y_max - y_mean;
y_bounds = [y_lower, y_upper];
% dataTable(dataTable.loop_id<200,:) = [];
% Display name (optional)
try
idx = find(dataTable.(loop_var) == loopValue, 1, 'first');
% dispname = char(equalizer_structure(dataTable.equalizer_structure(idx)));
dispname = [method];
dispname = [dispname, '/ ',num2str(unique_loop_var(i)) ,' m'];
% dispname = [dispname,'; ',num2str(unique(dataTable.interference_path_length)),' m'];
dispname = [dispname, '/ PAM ', num2str(filterParams.Configurations.pam_level)];
dispname = [dispname, '/ ', num2str(filterParams.Configurations.symbolrate.*1e-9),' GBd'];
end
% Filter by time
filter_by_time = 0;
if filter_by_time
startTime = datetime('2025-04-20 18:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
stopTime = datetime('2025-04-30 19:30:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
dataTable.date_of_run = datetime(dataTable.date_of_run, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
dataTable.date_of_processing = datetime(dataTable.date_of_processing, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
dataTable = dataTable(dataTable.date_of_processing > startTime, :);
dataTable = dataTable(dataTable.date_of_processing < stopTime, :);
end
if plotBoundaries
% Plot bounded line
[hl, hp] = boundedline(x_values, y_mean, y_bounds, ...
'alpha', 'transparency', 0.1, ...
'cmap', cols(i,:), ...
'nan', 'fill', ...
'orientation', 'vert');
% % Style the main line: thinnest, dotted, no marker
set(hl, 'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none', ...
'Color', cols(i,:), 'DisplayName', string(dispname));
% Group by smth
y_var = 'BER';
x_var = 'SIR';
plt = errorbar(x_values,y_mean,y_lower,y_upper,'LineWidth', 0.1, 'LineStyle', 'none', 'Marker', 'none','Color', cols(i,:), 'DisplayName', string(dispname),'HandleVisibility','off');
% Hide patch (shaded area) from legend
set(hp, 'HandleVisibility', 'off','LineStyle',':','LineWidth',0.5,'Marker','none');
% % Add invisible scatter for DataTips
% plt = scatter(x_values, y_mean, ...
% 'Marker', 'o', 'MarkerEdgeColor', 'none', 'MarkerFaceColor', 'none', ...
% 'HandleVisibility', 'off', 'PickableParts', 'all');
else
loop_var = 'interference_path_length';
fixedVars = {'equalizer_structure','interference_path_length',x_var};
plt= plot(x_values,y_mean,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none', ...
'Color', cols(i,:), 'DisplayName', string(dispname));
% plt= errorbar(x_values,y_mean,y_lower,y_upper,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none','Color', cols(i,:), 'DisplayName', string(dispname));
[dataTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var);
dataTableGrpd_mean = groupIt(fixedVars, dataTable, @mean);
dataTableGrpd_min = groupIt(fixedVars, dataTable, @min);
dataTableGrpd_max = groupIt(fixedVars, dataTable, @max);
% Create a new figure
hold on
unique_loop_var = unique(dataTable.(loop_var));
for i = 1:numel(unique_loop_var)
% Prepare filtered data for this loop variable
loopValue = unique_loop_var(i);
loopFiltGrpd = dataTableGrpd_mean.(loop_var) == loopValue;
if ~any(loopFiltGrpd)
continue; % Skip if no data for this loop var
end
% Extract values
x_values = dataTableGrpd_mean.(x_var)(loopFiltGrpd, :);
y_mean = dataTableGrpd_mean.(y_var)(loopFiltGrpd, :);
y_min = dataTableGrpd_min.(y_var)(loopFiltGrpd, :);
y_max = dataTableGrpd_max.(y_var)(loopFiltGrpd, :);
% Compute bounds: distance from mean
y_lower = y_mean - y_min;
y_upper = y_max - y_mean;
y_bounds = [y_lower, y_upper];
% Display name (optional)
try
idx = find(dataTable.(loop_var) == loopValue, 1, 'first');
% dispname = char(equalizer_structure(dataTable.equalizer_structure(idx)));
dispname = [method];
dispname = [dispname, '/ ',num2str(unique_loop_var(i)) ,' m'];
% dispname = [dispname,'; ',num2str(unique(dataTable.interference_path_length)),' m'];
% dispname = [dispname, '/ PAM ', num2str(filterParams.Configurations.pam_level)];
% dispname = [dispname, '/ ', num2str(filterParams.Configurations.symbolrate.*1e-9),' GBd'];
end
if plotBoundaries
% Plot bounded line
[hl, hp] = boundedline(x_values, y_mean, y_bounds, ...
'alpha', 'transparency', 0.1, ...
'cmap', cols(i,:), ...
'nan', 'fill', ...
'orientation', 'vert');
% % Style the main line: thinnest, dotted, no marker
set(hl, 'LineWidth', 0.5, 'LineStyle', ':', 'Marker', 'none', ...
'Color', cols(i,:), 'DisplayName', string(dispname));
plt = errorbar(x_values,y_mean,y_lower,y_upper,'LineWidth', 0.9, 'LineStyle', 'none', 'Marker', 'none','Color', cols(i,:), 'DisplayName', string(dispname),'HandleVisibility','off');
% Hide patch (shaded area) from legend
set(hp, 'HandleVisibility', 'off','LineStyle',':','LineWidth',0.5,'Marker','none');
% Fit a 4th-order polynomial to log10(BER)
p = polyfit(x_values, log10(y_mean), 3); % 4 is fitting order, adjust as needed
% Evaluate the fitted polynomial
x_fit = linspace(min(x_values), max(x_values), 300); % Fine points
y_fit_log = polyval(p, x_fit); % Still in log10 domain
y_fit = 10.^y_fit_log; % Back to BER domain
plot(x_fit,y_fit,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none', ...
'Color', cols(i,:), 'DisplayName', string(dispname),'HandleVisibility','off');
% % Add invisible scatter for DataTips
% plt = scatter(x_values, y_mean, ...
% 'Marker', 'o', 'MarkerEdgeColor', 'none', 'MarkerFaceColor', 'none', ...
% 'HandleVisibility', 'off', 'PickableParts', 'all');
else
plt= plot(x_values,y_mean,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none', ...
'Color', cols(i,:), 'DisplayName', string(dispname));
% plt= errorbar(x_values,y_mean,y_lower,y_upper,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none','Color', cols(i,:), 'DisplayName', string(dispname));
end
% Add data tips to the invisible scatter
pair_one = {'Run ID', dataTableGrpd_mean.run_id(loopFiltGrpd, :)};
pair_two = {'Rate', dataTableGrpd_mean.bitrate(loopFiltGrpd, :) * 1e-9};
pair_three = {'PD in', round(dataTableGrpd_mean.power_pd_in(loopFiltGrpd, :), 2)};
addDatatips(plt, pair_one, pair_two, pair_three);
% Optionally: outline bounds for better visibility (optional)
% hnew = outlinebounds(hl, hp);
% Tick marks (x-axis)
% Optional: scatter realizations
if plotRealizations
loopFiltSingle = dataTable.(loop_var) == loopValue;
x_single = double(dataTable.(x_var)(loopFiltSingle, :));
y_single = double(dataTable.(y_var)(loopFiltSingle, :));
mkr = '.';
sc = scatter(x_single+(mode*0.1)-0.2, y_single,15, 'Marker', mkr, 'MarkerEdgeColor', cols(i, :), ...
'LineWidth', 0.5, 'HandleVisibility', 'off', 'DisplayName', string(dispname));
pair_one = {'Run ID', dataTable.run_id(loopFiltSingle, :)};
pair_two = {'Rate', dataTable.bitrate(loopFiltSingle, :) * 1e-9};
pair_three = {'PD in', round(dataTable.power_pd_in(loopFiltSingle, :), 2)};
addDatatips(sc, pair_one, pair_two, pair_three);
end
end
% Label axes and title
legend('Interpreter', 'latex');
xlabel(x_var);
if ~y_here
ylabel(y_var);
yticklabels = [];
y_here = 1;
end
if int_len ~= 0
set(gca, 'YTick', []);
end
% title([x_var, ' vs. ', y_var]);
if string(y_var) == "BER"
yline(2.2e-4, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
yline(3.8e-3, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
yline(2e-2, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
ylim([1e-5, 0.1]);
end
xlim([15,35]);
ylim([9e-5 0.1 ]);
xticks([13:2:35]);
% Enable grid and beautify
grid on;
beautifyBERplot;
end
% Add data tips to the invisible scatter
pair_one = {'Run ID', dataTableGrpd_mean.run_id(loopFiltGrpd, :)};
pair_two = {'Rate', dataTableGrpd_mean.bitrate(loopFiltGrpd, :) * 1e-9};
pair_three = {'PD in', round(dataTableGrpd_mean.power_pd_in(loopFiltGrpd, :), 2)};
addDatatips(plt, pair_one, pair_two, pair_three);
% Optionally: outline bounds for better visibility (optional)
% hnew = outlinebounds(hl, hp);
% Tick marks (x-axis)
xticks(round(unique(x_values),2));
% Optional: scatter realizations
if plotRealizations
loopFiltSingle = dataTable.(loop_var) == loopValue;
x_single = double(dataTable.(x_var)(loopFiltSingle, :));
y_single = double(dataTable.(y_var)(loopFiltSingle, :));
sc = scatter(x_single, y_single, 'Marker', mkr, 'MarkerEdgeColor', cols(i, :), ...
'LineWidth', 0.5, 'HandleVisibility', 'off', 'DisplayName', string(dispname));
pair_one = {'Run ID', dataTable.run_id(loopFiltSingle, :)};
pair_two = {'Rate', dataTable.bitrate(loopFiltSingle, :) * 1e-9};
pair_three = {'PD in', round(dataTable.power_pd_in(loopFiltSingle, :), 2)};
addDatatips(sc, pair_one, pair_two, pair_three);
end
end
% Label axes and title
legend('Interpreter', 'latex');
xlabel(x_var);
ylabel(y_var);
title([x_var, ' vs. ', y_var]);
if string(y_var) == "BER"
yline(4e-4, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
yline(3.8e-3, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
yline(2e-2, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
ylim([1e-5, 0.1]);
end
xlim([13,35]);
% Enable grid and beautify
grid on;
beautifyBERplot;
function resultTable = groupIt(fixedVars, dataTable, aggregationFunction)
% groupIt Groups data in a table based on fixedVars and applies aggregationFunction to numeric data.
@@ -463,9 +534,9 @@ for groupIdx = 1:height(groupKeys)
continue;
end
% Detect outliers in log space
% Detect outliers in log space
y_log = log10(y_values);
outlierMask = isoutlier(y_log, 'quartiles'); % or 'median', 'grubbs', etc.
outlierMask = isoutlier(y_log, 'quartiles',1); % or 'median', 'grubbs', etc.
% If any outliers found, collect their data
if any(outlierMask)