silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

ecoc measurements

Browse Source
This commit is contained in:
Silas Labor Zizou
2025-04-13 16:29:12 +02:00
parent 00f1c557c0
commit 0236103b13
19 changed files with 709 additions and 359 deletions
Download Patch File Download Diff File Expand all files Collapse all files

325
projects/ECOC_2025/plots_from_database/plot_mpi_trial.m
View File

@@ -1,7 +1,7 @@
basePath = 'C:\Users\sioe\Documents\MATLAB\imdd_simulation\projects\ECOC_2025\';
database_name = 'ecoc2025.db';
database_name = 'ecoc2025_loops.db';
database = DBHandler("pathToDB", [basePath, database_name]);
filterParams = database.tables;
@@ -10,14 +10,14 @@ filterParams.Configurations = struct( ...
'fiber_length', 0, ...
'db_mode', '"no_db"', ...
'interference_attenuation', [], ...
'interference_path_length', 0.06, ...
'interference_path_length', 50, ...
'is_mpi', 1, ...
'pam_level', 4, ...
'wavelength', 1310, ...
'precomp_amp', [], ...
'signal_attenuation', [], ...
'v_awg', 0.9, ...
'v_bias', 2.8 ...
'v_awg', 0.95, ...
'v_bias', 2.5 ...
);
% filterParams.EqualizerParameters.diff_precode = int32(db_mode.db_encoded);
@@ -29,12 +29,12 @@ selectedFields = {'Configurations.run_id' 'Runs.date_of_run' 'Runs.rx_raw_path'
'Measurements.power_mpi_interference' 'Measurements.power_mpi_signal' 'Results.BER' 'Results.BER_precoded' 'Results.SNR' 'Results.GMI' 'Results.Alpha' 'Results.date_of_processing'};
[dataTable,sql_query] = database.queryDB(filterParams, selectedFields);
dataTable.SIR = dataTable.power_mpi_signal - dataTable.power_mpi_interference;
dataTable.SIR = round(-6 - dataTable.power_mpi_interference);
dataTable = cleanUpTable(dataTable);
% Filter by time
startTime = datetime('2025-04-11 14:40:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
stopTime = datetime('2025-04-11 15:30:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
startTime = datetime('2025-04-12 18:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
stopTime = datetime('2025-04-13 19:30:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
dataTable.date_of_run = datetime(dataTable.date_of_run, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
dataTable.date_of_processing = datetime(dataTable.date_of_processing, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
dataTable = dataTable(dataTable.date_of_processing > startTime, :);
@@ -46,21 +46,23 @@ x_var = 'SIR';
loop_var = 'eq_id';
fixedVars = {'eq_id',x_var};
% dataTableGrpd_mean = groupIt(fixedVars,dataTable);
[dataTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var);
dataTableGrpd_mean = groupIt(fixedVars, dataTable, @mean);
dataTableGrpd_min = groupIt(fixedVars, dataTable, @min);
dataTableGrpd_max = groupIt(fixedVars, dataTable, @max);
plotRealizations = 1;
% Create a new figure
mkr = '*';
mkr = '.';
figure();
hold on
unique_loop_var = unique(dataTable.(loop_var));
cols = linspecer(numel(unique_loop_var)); % Ensure color count matches
for i = 1:numel(unique_loop_var)
for i = 1:2%1:numel(unique_loop_var)
% Prepare filtered data for this loop variable
loopValue = unique_loop_var(i);
@@ -139,7 +141,7 @@ ylabel(y_var);
title([x_var, ' vs. ', y_var]);
if y_var == 'BER'
yline(3.8e-3, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
yline(4e-4, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
ylim([1e-5, 0.1]);
end
@@ -163,57 +165,58 @@ function resultTable = groupIt(fixedVars, dataTable, aggregationFunction)
% Output:
% resultTable - Grouped and aggregated table
% Group data
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
% Prepare aggregation
varNames = dataTable.Properties.VariableNames;
nVars = numel(varNames);
aggData = cell(height(groupKeys), nVars);
groupCount = zeros(height(groupKeys), 1); % Store number of rows in each group
% Group data
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
% Loop over groups
for i = 1:height(groupKeys)
idx = (G == i); % Logical index for group i
groupCount(i) = sum(idx); % Count rows in group
% Prepare aggregation
varNames = dataTable.Properties.VariableNames;
nVars = numel(varNames);
aggData = cell(height(groupKeys), nVars);
groupCount = zeros(height(groupKeys), 1); % Store number of rows in each group
% Loop over each variable
for j = 1:nVars
colData = dataTable.(varNames{j});
% Loop over groups
for i = 1:height(groupKeys)
idx = (G == i); % Logical index for group i
groupCount(i) = sum(idx); % Count rows in group
if isnumeric(colData)
% Numeric: apply aggregation function (skip empty groups safely)
if any(idx)
aggData{i, j} = aggregationFunction(colData(idx));
% Loop over each variable
for j = 1:nVars
colData = dataTable.(varNames{j});
if isnumeric(colData)
% Numeric: apply aggregation function (skip empty groups safely)
if any(idx)
% aggData{i, j} = rmoutliers(double(colData(idx)));
aggData{i, j} = aggregationFunction(colData(idx));
else
aggData{i, j} = NaN;
end
else
% Non-numeric: take first non-empty value
if iscell(colData)
nonEmptyIdx = find(idx & ~cellfun(@isempty, colData), 1);
if ~isempty(nonEmptyIdx)
aggData{i, j} = colData{nonEmptyIdx};
else
aggData{i, j} = NaN;
aggData{i, j} = [];
end
else
% Non-numeric: take first non-empty value
if iscell(colData)
nonEmptyIdx = find(idx & ~cellfun(@isempty, colData), 1);
if ~isempty(nonEmptyIdx)
aggData{i, j} = colData{nonEmptyIdx};
else
aggData{i, j} = [];
end
nonEmptyIdx = find(idx, 1);
if ~isempty(nonEmptyIdx)
aggData{i, j} = colData(nonEmptyIdx);
else
nonEmptyIdx = find(idx, 1);
if ~isempty(nonEmptyIdx)
aggData{i, j} = colData(nonEmptyIdx);
else
aggData{i, j} = [];
end
aggData{i, j} = [];
end
end
end
end
end
% Convert aggregated data to table
resultTable = cell2table(aggData, 'VariableNames', varNames);
% Convert aggregated data to table
resultTable = cell2table(aggData, 'VariableNames', varNames);
% Add group size as new column
resultTable.nRows = groupCount;
% Add group size as new column
resultTable.nRows = groupCount;
end
@@ -221,7 +224,7 @@ end
function addDatatips(sc, varargin)
% addDatatips Adds custom data tip rows to a scatter plot.
%
% addDatatips(sc, pair1, pair2, ...) adds one or more custom rows to the
% addDatatips(sc, pair1, pair2, ...) adds one or more custom rows to the
% data tip display of the scatter plot identified by sc.
%
% Each pair should be provided as a 1x2 cell array: {label, value}.
@@ -233,27 +236,27 @@ function addDatatips(sc, varargin)
% pair_one = {'Attenuation', attenuationVector};
% addDatatips(sc, pair_one);
numPoints = numel(sc.XData);
for k = 1:length(varargin)
pair = varargin{k};
if ~iscell(pair) || numel(pair) ~= 2
error('Each pair must be a 1x2 cell array: {label, value}.');
end
label = pair{1};
value = pair{2};
% If value is a vector, ensure its length is either 1 or equal to the number of scatter points.
if isvector(value) && numel(value) ~= 1 && numel(value) ~= numPoints
error('The vector for "%s" must be a scalar or have %d elements matching the scatter data points.', label, numPoints);
end
% Create a new data tip row using the provided label and vector.
newRow = dataTipTextRow(label, value);
sc.DataTipTemplate.DataTipRows(end+1) = newRow;
numPoints = numel(sc.XData);
for k = 1:length(varargin)
pair = varargin{k};
if ~iscell(pair) || numel(pair) ~= 2
error('Each pair must be a 1x2 cell array: {label, value}.');
end
label = pair{1};
value = pair{2};
% If value is a vector, ensure its length is either 1 or equal to the number of scatter points.
if isvector(value) && numel(value) ~= 1 && numel(value) ~= numPoints
error('The vector for "%s" must be a scalar or have %d elements matching the scatter data points.', label, numPoints);
end
% Create a new data tip row using the provided label and vector.
newRow = dataTipTextRow(label, value);
sc.DataTipTemplate.DataTipRows(end+1) = newRow;
end
end
@@ -274,69 +277,135 @@ function cleanedTable = cleanUpTable(inputTable)
% Output:
% cleanedTable - Cleaned MATLAB table with proper numeric types
cleanedTable = inputTable;
varNames = cleanedTable.Properties.VariableNames;
for i = 1:numel(varNames)
col = cleanedTable.(varNames{i});
% Case 1: If it's a cell array (likely mixed strings/struct)
if iscell(col)
% Convert struct 'NaN' entries to string 'NaN'
col = cellfun(@(x) convertStructToString(x), col, 'UniformOutput', false);
% Try to convert string numbers to actual numbers
numericCol = str2double(col);
if all(isnan(numericCol) == strcmpi(col, 'NaN') | cellfun(@isempty, col))
% If conversion is successful (NaNs correspond to 'NaN' strings), use it
cleanedTable.(varNames{i}) = numericCol;
else
% Else, try to convert to datetime
try
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
catch
% If it fails, leave as cell array of strings
cleanedTable.(varNames{i}) = string(col);
end
end
% Case 2: If it's already a string array
elseif isstring(col)
numericCol = str2double(col);
if all(isnan(numericCol) == strcmpi(col, "NaN"))
cleanedTable.(varNames{i}) = numericCol;
else
% Try convert to datetime
try
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
catch
% Leave as string
end
end
% Case 3: If it's already numeric, keep as is
elseif isnumeric(col)
continue;
% Case 4: If it's datetime, keep as is
elseif isdatetime(col)
continue;
cleanedTable = inputTable;
varNames = cleanedTable.Properties.VariableNames;
for i = 1:numel(varNames)
col = cleanedTable.(varNames{i});
% Case 1: If it's a cell array (likely mixed strings/struct)
if iscell(col)
% Convert struct 'NaN' entries to string 'NaN'
col = cellfun(@(x) convertStructToString(x), col, 'UniformOutput', false);
% Try to convert string numbers to actual numbers
numericCol = str2double(col);
if all(isnan(numericCol) == strcmpi(col, 'NaN') | cellfun(@isempty, col))
% If conversion is successful (NaNs correspond to 'NaN' strings), use it
cleanedTable.(varNames{i}) = numericCol;
else
% Catch-all for unexpected types, convert to string
cleanedTable.(varNames{i}) = string(col);
% Else, try to convert to datetime
try
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
catch
% If it fails, leave as cell array of strings
cleanedTable.(varNames{i}) = string(col);
end
end
% Case 2: If it's already a string array
elseif isstring(col)
numericCol = str2double(col);
if all(isnan(numericCol) == strcmpi(col, "NaN"))
cleanedTable.(varNames{i}) = numericCol;
else
% Try convert to datetime
try
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
catch
% Leave as string
end
end
% Case 3: If it's already numeric, keep as is
elseif isnumeric(col)
continue;
% Case 4: If it's datetime, keep as is
elseif isdatetime(col)
continue;
else
% Catch-all for unexpected types, convert to string
cleanedTable.(varNames{i}) = string(col);
end
end
end
function out = convertStructToString(x)
% Helper function to convert struct NaN to string 'NaN'
if isstruct(x)
out = "NaN";
elseif isstring(x) || ischar(x)
out = string(x);
else
out = x;
end
if isstruct(x)
out = "NaN";
elseif isstring(x) || ischar(x)
out = string(x);
else
out = x;
end
end
function [cleanedTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var)
% Group the data
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
% Initialize logical index to keep rows
keepIdx = true(height(dataTable), 1);
% Prepare storage for outliers
outlierRecords = [];
% Loop over each group
for groupIdx = 1:height(groupKeys)
% Find indices of current group
groupRows = (G == groupIdx);
% Extract y-values of this group
y_values = dataTable.(y_var)(groupRows);
% Skip groups with fewer than 3 points (optional)
if sum(groupRows) < 3
continue;
end
% Detect outliers
outlierMask = isoutlier(y_values, 'quartiles');
% If any outliers found, collect their data
if any(outlierMask)
groupData = dataTable(groupRows, :);
% Prepare table for current group outliers
outlierGroupTable = groupData(outlierMask, :);
% Add group key values for traceability
for k = 1:numel(fixedVars)
outlierGroupTable.(['Group_', fixedVars{k}]) = repmat(groupKeys{groupIdx, k}, height(outlierGroupTable), 1);
end
% Append to collection
outlierRecords = [outlierRecords; outlierGroupTable]; %#ok<AGROW>
end
% Mark outliers for removal
groupRowIdx = find(groupRows);
keepIdx(groupRowIdx(outlierMask)) = false;
end
% Apply mask to dataTable
cleanedTable = dataTable(keepIdx, :);
% Prepare output: if no outliers, return empty table
if isempty(outlierRecords)
outliersTable = table();
else
outliersTable = outlierRecords;
end
nRemoved = sum(~keepIdx);
nTotalOriginal = height(dataTable) + nRemoved;
percentageRemoved = (nRemoved / nTotalOriginal) * 100;
fprintf('Removed %d outliers from the data table (%.2f%% of total %d entries).\n', ...
nRemoved, percentageRemoved, nTotalOriginal);
end