CLEANUP - changes to folder structure
This commit is contained in:
Silas Oettinghaus
269
projects/ECOC_2025_MPI/plots_from_database/bias_vs_ber.m
Normal file
269
projects/ECOC_2025_MPI/plots_from_database/bias_vs_ber.m
Normal file
@@ -0,0 +1,269 @@
|
||||
|
||||
|
||||
basePath = 'C:\Users\sioe\Documents\MATLAB\imdd_simulation\projects\ECOC_2025\';
|
||||
database_name = 'ecoc2025.db';
|
||||
database = DBHandler("pathToDB", [basePath, database_name],"type",'mysql');
|
||||
|
||||
filterParams = database.tables;
|
||||
filterParams.Configurations = struct( ...
|
||||
'symbolrate', 112e9, ... %[224,336,360,390,420,448]
|
||||
'fiber_length', 0, ...
|
||||
'db_mode', '"no_db"', ...
|
||||
'interference_attenuation', [], ...
|
||||
'interference_path_length', [], ...
|
||||
'is_mpi', 1, ...
|
||||
'pam_level', 4, ...
|
||||
'wavelength', 1310, ...
|
||||
'precomp_amp', -64, ...
|
||||
'signal_attenuation', '0', ...
|
||||
'v_awg', [], ...
|
||||
'v_bias', [] ...
|
||||
);
|
||||
|
||||
% filterParams.EqualizerParameters.diff_precode = int32(db_mode.db_encoded);
|
||||
% filterParams.EqualizerParameters.equalizer_structure = int32(equalizer_structure.vnle);
|
||||
|
||||
selectedFields = {'Configurations.run_id' 'Runs.date_of_run' 'Runs.rx_raw_path' 'Configurations.bitrate' 'Configurations.v_bias' 'Configurations.v_awg' 'Configurations.precomp_amp' 'Configurations.symbolrate' 'Configurations.pam_level'...
|
||||
'Configurations.db_mode' 'Configurations.rop_attenuation' 'Configurations.is_mpi' 'Configurations.interference_attenuation' 'Configurations.signal_attenuation' ...
|
||||
'EqualizerParameters.equalizer_structure' 'EqualizerParameters.diff_precode' 'EqualizerParameters.eq_id' 'Measurements.power_pd_in' ...
|
||||
'Measurements.power_mpi_interference' 'Measurements.power_mpi_signal' 'Results.BER' 'Results.BER_precoded' 'Results.SNR' 'Results.GMI' 'Results.Alpha' 'Results.date_of_processing'};
|
||||
|
||||
[dataTable,sql_query] = database.queryDB(filterParams, selectedFields);
|
||||
|
||||
dataTable = cleanUpTable(dataTable);
|
||||
|
||||
% Filter by time
|
||||
startTime = datetime('2025-04-11 13:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
stopTime = datetime('2025-04-11 14:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
dataTable.date_of_run = datetime(dataTable.date_of_run, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
dataTable.date_of_processing = datetime(dataTable.date_of_processing, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
dataTable = dataTable(dataTable.date_of_processing > startTime, :);
|
||||
dataTable = dataTable(dataTable.date_of_processing < stopTime, :);
|
||||
|
||||
% Group by smth
|
||||
y_var = 'SNR';
|
||||
x_var = 'v_bias';
|
||||
|
||||
loop_var = 'v_awg';
|
||||
fixedVars = {'eq_id',loop_var};
|
||||
dataTableGrpd = groupIt(fixedVars,dataTable);
|
||||
|
||||
plotRealizations = 1;
|
||||
|
||||
% Create a new figure
|
||||
mkr = 'x';
|
||||
|
||||
figure(10);
|
||||
hold on
|
||||
|
||||
unique_loop_var = unique(dataTable.(loop_var));
|
||||
cols = linspecer(8);
|
||||
|
||||
for i = 1:numel(unique_loop_var)
|
||||
|
||||
idx = find(dataTable.(loop_var)== unique_loop_var(i), 1, 'first');
|
||||
% dispname = equalizer_structure(dataTable.equalizer_structure(idx));
|
||||
dispname = num2str(unique_loop_var(i));
|
||||
|
||||
% Plot SCATTERS: timestamp vs. interference_attenuation
|
||||
loop_filt_2 = dataTable.(loop_var)==unique_loop_var(i);
|
||||
if plotRealizations
|
||||
|
||||
y_values = dataTable.(y_var)(loop_filt_2,:);
|
||||
x_values = dataTable.(x_var)(loop_filt_2,:);
|
||||
y_values = double(y_values);
|
||||
x_values = double(x_values);
|
||||
|
||||
sc = scatter(x_values, y_values, 'LineWidth', 0.5,'Marker',mkr,'MarkerEdgeColor',cols(i,:),'HandleVisibility','on','DisplayName',string(dispname));
|
||||
|
||||
pair_one = {'Run ID', dataTable.run_id(loop_filt_2,:)};
|
||||
pair_two = {'Rate', dataTable.bitrate(loop_filt_2,:).*1e-9};
|
||||
pair_three = {'PD in', round(dataTable.power_pd_in(loop_filt_2,:),2)};
|
||||
addDatatips(sc, pair_one, pair_two,pair_three);
|
||||
xticks(unique(x_values));
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
% Label the axes and add a title
|
||||
legend('Interpreter','latex');
|
||||
xlabel(x_var);
|
||||
ylabel(y_var);
|
||||
title([x_var,' vs. ',y_var]);
|
||||
if y_var == 'BER'
|
||||
yline(3.8e-3,'LineWidth',1,'LineStyle','--','HandleVisibility','off');
|
||||
ylim([1e-4 0.5]);
|
||||
end
|
||||
% Enable grid for better readability
|
||||
grid on;
|
||||
|
||||
beautifyBERplot;
|
||||
|
||||
|
||||
|
||||
|
||||
function resultTable = groupIt(fixedVars,dataTable)
|
||||
|
||||
% Group by run_id and eq_id (adjust grouping keys as needed)
|
||||
|
||||
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
|
||||
|
||||
% Preallocate a cell array for aggregated data.
|
||||
varNames = dataTable.Properties.VariableNames;
|
||||
nVars = numel(varNames);
|
||||
aggData = cell(height(groupKeys), nVars);
|
||||
groupCount = zeros(height(groupKeys), 1); % To store the size of each group
|
||||
|
||||
% Loop over each group.
|
||||
for i = 1:height(groupKeys)
|
||||
idx = (G == i); % Logical index for group i
|
||||
groupCount(i) = sum(idx); % Count number of rows in this group
|
||||
% For each variable in the table:
|
||||
for j = 1:nVars
|
||||
colData = dataTable.(varNames{j});
|
||||
if isnumeric(colData)
|
||||
% For numeric data, compute the mean.
|
||||
aggData{i, j} = min(colData(idx));
|
||||
else
|
||||
% For non-numeric data, take the first entry.
|
||||
if iscell(colData)
|
||||
aggData{i, j} = colData{find(idx, 1)};
|
||||
else
|
||||
aggData{i, j} = colData(find(idx, 1));
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
% Convert the aggregated cell array into a table.
|
||||
resultTable = cell2table(aggData, 'VariableNames', varNames);
|
||||
|
||||
% Append the group count as a new column.
|
||||
resultTable.nRows = groupCount;
|
||||
|
||||
end
|
||||
|
||||
|
||||
function addDatatips(sc, varargin)
|
||||
% addDatatips Adds custom data tip rows to a scatter plot.
|
||||
%
|
||||
% addDatatips(sc, pair1, pair2, ...) adds one or more custom rows to the
|
||||
% data tip display of the scatter plot identified by sc.
|
||||
%
|
||||
% Each pair should be provided as a 1x2 cell array: {label, value}.
|
||||
% The value can be a scalar or a vector. If a vector is provided, its length
|
||||
% must match the number of scatter plot points.
|
||||
%
|
||||
% Example:
|
||||
% sc = scatter(x, y, 'LineWidth', 1.5, 'Marker', 'o');
|
||||
% pair_one = {'Attenuation', attenuationVector};
|
||||
% addDatatips(sc, pair_one);
|
||||
|
||||
numPoints = numel(sc.XData);
|
||||
|
||||
for k = 1:length(varargin)
|
||||
pair = varargin{k};
|
||||
|
||||
if ~iscell(pair) || numel(pair) ~= 2
|
||||
error('Each pair must be a 1x2 cell array: {label, value}.');
|
||||
end
|
||||
|
||||
label = pair{1};
|
||||
value = pair{2};
|
||||
|
||||
% If value is a vector, ensure its length is either 1 or equal to the number of scatter points.
|
||||
if isvector(value) && numel(value) ~= 1 && numel(value) ~= numPoints
|
||||
error('The vector for "%s" must be a scalar or have %d elements matching the scatter data points.', label, numPoints);
|
||||
end
|
||||
|
||||
% Create a new data tip row using the provided label and vector.
|
||||
newRow = dataTipTextRow(label, value);
|
||||
sc.DataTipTemplate.DataTipRows(end+1) = newRow;
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
function cleanedTable = cleanUpTable(inputTable)
|
||||
% cleanUpTable Cleans a MATLAB table where numbers and NaNs are stored as strings or structs.
|
||||
%
|
||||
% cleanedTable = cleanUpTable(inputTable)
|
||||
%
|
||||
% This function goes through all columns of the input table:
|
||||
% - Converts strings of numbers to numeric values
|
||||
% - Converts string 'NaN' and struct NaNs to real NaN
|
||||
% - Converts date strings to datetime (if possible)
|
||||
%
|
||||
% Input:
|
||||
% inputTable - MATLAB table with mixed types
|
||||
%
|
||||
% Output:
|
||||
% cleanedTable - Cleaned MATLAB table with proper numeric types
|
||||
|
||||
cleanedTable = inputTable;
|
||||
varNames = cleanedTable.Properties.VariableNames;
|
||||
|
||||
for i = 1:numel(varNames)
|
||||
col = cleanedTable.(varNames{i});
|
||||
|
||||
% Case 1: If it's a cell array (likely mixed strings/struct)
|
||||
if iscell(col)
|
||||
% Convert struct 'NaN' entries to string 'NaN'
|
||||
col = cellfun(@(x) convertStructToString(x), col, 'UniformOutput', false);
|
||||
|
||||
% Try to convert string numbers to actual numbers
|
||||
numericCol = str2double(col);
|
||||
|
||||
if all(isnan(numericCol) == strcmpi(col, 'NaN') | cellfun(@isempty, col))
|
||||
% If conversion is successful (NaNs correspond to 'NaN' strings), use it
|
||||
cleanedTable.(varNames{i}) = numericCol;
|
||||
else
|
||||
% Else, try to convert to datetime
|
||||
try
|
||||
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
catch
|
||||
% If it fails, leave as cell array of strings
|
||||
cleanedTable.(varNames{i}) = string(col);
|
||||
end
|
||||
end
|
||||
|
||||
% Case 2: If it's already a string array
|
||||
elseif isstring(col)
|
||||
numericCol = str2double(col);
|
||||
if all(isnan(numericCol) == strcmpi(col, "NaN"))
|
||||
cleanedTable.(varNames{i}) = numericCol;
|
||||
else
|
||||
% Try convert to datetime
|
||||
try
|
||||
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
catch
|
||||
% Leave as string
|
||||
end
|
||||
end
|
||||
|
||||
% Case 3: If it's already numeric, keep as is
|
||||
elseif isnumeric(col)
|
||||
continue;
|
||||
|
||||
% Case 4: If it's datetime, keep as is
|
||||
elseif isdatetime(col)
|
||||
continue;
|
||||
|
||||
else
|
||||
% Catch-all for unexpected types, convert to string
|
||||
cleanedTable.(varNames{i}) = string(col);
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function out = convertStructToString(x)
|
||||
% Helper function to convert struct NaN to string 'NaN'
|
||||
if isstruct(x)
|
||||
out = "NaN";
|
||||
elseif isstring(x) || ischar(x)
|
||||
out = string(x);
|
||||
else
|
||||
out = x;
|
||||
end
|
||||
end
|
||||
@@ -0,0 +1,271 @@
|
||||
|
||||
|
||||
basePath = 'C:\Users\sioe\Documents\MATLAB\imdd_simulation\projects\ECOC_2025\';
|
||||
database_name = 'ecoc2025.db';
|
||||
database = DBHandler("pathToDB", [basePath, database_name]);
|
||||
|
||||
filterParams = database.tables;
|
||||
filterParams.Configurations = struct( ...
|
||||
'symbolrate', 112e9, ... %[224,336,360,390,420,448]
|
||||
'fiber_length', 0, ...
|
||||
'db_mode', '"no_db"', ...
|
||||
'interference_attenuation', [], ...
|
||||
'interference_path_length', [], ...
|
||||
'is_mpi', 0, ...
|
||||
'pam_level', 4, ...
|
||||
'wavelength', 1310, ...
|
||||
'precomp_amp', [], ...
|
||||
'signal_attenuation', [], ...
|
||||
'v_awg', 0.8, ...
|
||||
'v_bias', 2.8 ...
|
||||
);
|
||||
|
||||
% filterParams.EqualizerParameters.diff_precode = int32(db_mode.db_encoded);
|
||||
% filterParams.EqualizerParameters.equalizer_structure = int32(equalizer_structure.vnle);
|
||||
|
||||
selectedFields = {'Configurations.run_id' 'Runs.date_of_run' 'Runs.rx_raw_path' 'Configurations.bitrate' 'Configurations.v_bias' 'Configurations.v_awg' 'Configurations.precomp_amp' 'Configurations.symbolrate' 'Configurations.pam_level'...
|
||||
'Configurations.db_mode' 'Configurations.rop_attenuation' 'Configurations.is_mpi' 'Configurations.interference_attenuation' 'Configurations.signal_attenuation' ...
|
||||
'EqualizerParameters.equalizer_structure' 'EqualizerParameters.diff_precode' 'EqualizerParameters.eq_id' 'Measurements.power_pd_in' ...
|
||||
'Measurements.power_mpi_interference' 'Measurements.power_mpi_signal' 'Results.BER' 'Results.BER_precoded' 'Results.SNR' 'Results.GMI' 'Results.Alpha' 'Results.date_of_processing'};
|
||||
|
||||
[dataTable,sql_query] = database.queryDB(filterParams, selectedFields);
|
||||
|
||||
dataTable = cleanUpTable(dataTable);
|
||||
|
||||
% Filter by time
|
||||
startTime = datetime('2025-04-11 13:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
stopTime = datetime('2025-04-11 14:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
dataTable.date_of_run = datetime(dataTable.date_of_run, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
dataTable.date_of_processing = datetime(dataTable.date_of_processing, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
dataTable = dataTable(dataTable.date_of_processing > startTime, :);
|
||||
dataTable = dataTable(dataTable.date_of_processing < stopTime, :);
|
||||
|
||||
% Group by smth
|
||||
y_var = 'BER';
|
||||
x_var = 'interference_attenuation';
|
||||
|
||||
loop_var = 'eq_id';
|
||||
fixedVars = {'eq_id',loop_var};
|
||||
dataTableGrpd = groupIt(fixedVars,dataTable);
|
||||
|
||||
plotRealizations = 1;
|
||||
|
||||
% Create a new figure
|
||||
mkr = '*';
|
||||
|
||||
figure(10);
|
||||
hold on
|
||||
|
||||
unique_loop_var = unique(dataTable.(loop_var));
|
||||
cols = linspecer(8);
|
||||
|
||||
for i = 1:numel(unique_loop_var)
|
||||
|
||||
idx = find(dataTable.(loop_var)== unique_loop_var(i), 1, 'first');
|
||||
|
||||
dispname = equalizer_structure(dataTable.equalizer_structure(idx));
|
||||
|
||||
% dispname = num2str(unique_loop_var(i));
|
||||
|
||||
% Plot SCATTERS: timestamp vs. interference_attenuation
|
||||
loop_filt_2 = dataTable.(loop_var)==unique_loop_var(i);
|
||||
if plotRealizations
|
||||
|
||||
y_values = dataTable.(y_var)(loop_filt_2,:);
|
||||
x_values = dataTable.(x_var)(loop_filt_2,:);
|
||||
y_values = double(y_values);
|
||||
x_values = double(x_values);
|
||||
|
||||
sc = scatter(x_values, y_values, 'LineWidth', 0.5,'Marker',mkr,'MarkerEdgeColor',cols(i,:),'HandleVisibility','on','DisplayName',string(dispname));
|
||||
|
||||
pair_one = {'Run ID', dataTable.run_id(loop_filt_2,:)};
|
||||
pair_two = {'Rate', dataTable.bitrate(loop_filt_2,:).*1e-9};
|
||||
pair_three = {'PD in', round(dataTable.power_pd_in(loop_filt_2,:),2)};
|
||||
addDatatips(sc, pair_one, pair_two,pair_three);
|
||||
xticks(unique(x_values));
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
% Label the axes and add a title
|
||||
legend('Interpreter','latex');
|
||||
xlabel(x_var);
|
||||
ylabel(y_var);
|
||||
title([x_var,' vs. ',y_var]);
|
||||
if y_var == 'BER'
|
||||
yline(3.8e-3,'LineWidth',1,'LineStyle','--','HandleVisibility','off');
|
||||
ylim([1e-5 0.1]);
|
||||
end
|
||||
% Enable grid for better readability
|
||||
grid on;
|
||||
|
||||
beautifyBERplot;
|
||||
|
||||
|
||||
|
||||
|
||||
function resultTable = groupIt(fixedVars,dataTable)
|
||||
|
||||
% Group by run_id and eq_id (adjust grouping keys as needed)
|
||||
|
||||
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
|
||||
|
||||
% Preallocate a cell array for aggregated data.
|
||||
varNames = dataTable.Properties.VariableNames;
|
||||
nVars = numel(varNames);
|
||||
aggData = cell(height(groupKeys), nVars);
|
||||
groupCount = zeros(height(groupKeys), 1); % To store the size of each group
|
||||
|
||||
% Loop over each group.
|
||||
for i = 1:height(groupKeys)
|
||||
idx = (G == i); % Logical index for group i
|
||||
groupCount(i) = sum(idx); % Count number of rows in this group
|
||||
% For each variable in the table:
|
||||
for j = 1:nVars
|
||||
colData = dataTable.(varNames{j});
|
||||
if isnumeric(colData)
|
||||
% For numeric data, compute the mean.
|
||||
aggData{i, j} = min(colData(idx));
|
||||
else
|
||||
% For non-numeric data, take the first entry.
|
||||
if iscell(colData)
|
||||
aggData{i, j} = colData{find(idx, 1)};
|
||||
else
|
||||
aggData{i, j} = colData(find(idx, 1));
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
% Convert the aggregated cell array into a table.
|
||||
resultTable = cell2table(aggData, 'VariableNames', varNames);
|
||||
|
||||
% Append the group count as a new column.
|
||||
resultTable.nRows = groupCount;
|
||||
|
||||
end
|
||||
|
||||
|
||||
function addDatatips(sc, varargin)
|
||||
% addDatatips Adds custom data tip rows to a scatter plot.
|
||||
%
|
||||
% addDatatips(sc, pair1, pair2, ...) adds one or more custom rows to the
|
||||
% data tip display of the scatter plot identified by sc.
|
||||
%
|
||||
% Each pair should be provided as a 1x2 cell array: {label, value}.
|
||||
% The value can be a scalar or a vector. If a vector is provided, its length
|
||||
% must match the number of scatter plot points.
|
||||
%
|
||||
% Example:
|
||||
% sc = scatter(x, y, 'LineWidth', 1.5, 'Marker', 'o');
|
||||
% pair_one = {'Attenuation', attenuationVector};
|
||||
% addDatatips(sc, pair_one);
|
||||
|
||||
numPoints = numel(sc.XData);
|
||||
|
||||
for k = 1:length(varargin)
|
||||
pair = varargin{k};
|
||||
|
||||
if ~iscell(pair) || numel(pair) ~= 2
|
||||
error('Each pair must be a 1x2 cell array: {label, value}.');
|
||||
end
|
||||
|
||||
label = pair{1};
|
||||
value = pair{2};
|
||||
|
||||
% If value is a vector, ensure its length is either 1 or equal to the number of scatter points.
|
||||
if isvector(value) && numel(value) ~= 1 && numel(value) ~= numPoints
|
||||
error('The vector for "%s" must be a scalar or have %d elements matching the scatter data points.', label, numPoints);
|
||||
end
|
||||
|
||||
% Create a new data tip row using the provided label and vector.
|
||||
newRow = dataTipTextRow(label, value);
|
||||
sc.DataTipTemplate.DataTipRows(end+1) = newRow;
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
|
||||
function cleanedTable = cleanUpTable(inputTable)
|
||||
% cleanUpTable Cleans a MATLAB table where numbers and NaNs are stored as strings or structs.
|
||||
%
|
||||
% cleanedTable = cleanUpTable(inputTable)
|
||||
%
|
||||
% This function goes through all columns of the input table:
|
||||
% - Converts strings of numbers to numeric values
|
||||
% - Converts string 'NaN' and struct NaNs to real NaN
|
||||
% - Converts date strings to datetime (if possible)
|
||||
%
|
||||
% Input:
|
||||
% inputTable - MATLAB table with mixed types
|
||||
%
|
||||
% Output:
|
||||
% cleanedTable - Cleaned MATLAB table with proper numeric types
|
||||
|
||||
cleanedTable = inputTable;
|
||||
varNames = cleanedTable.Properties.VariableNames;
|
||||
|
||||
for i = 1:numel(varNames)
|
||||
col = cleanedTable.(varNames{i});
|
||||
|
||||
% Case 1: If it's a cell array (likely mixed strings/struct)
|
||||
if iscell(col)
|
||||
% Convert struct 'NaN' entries to string 'NaN'
|
||||
col = cellfun(@(x) convertStructToString(x), col, 'UniformOutput', false);
|
||||
|
||||
% Try to convert string numbers to actual numbers
|
||||
numericCol = str2double(col);
|
||||
|
||||
if all(isnan(numericCol) == strcmpi(col, 'NaN') | cellfun(@isempty, col))
|
||||
% If conversion is successful (NaNs correspond to 'NaN' strings), use it
|
||||
cleanedTable.(varNames{i}) = numericCol;
|
||||
else
|
||||
% Else, try to convert to datetime
|
||||
try
|
||||
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
catch
|
||||
% If it fails, leave as cell array of strings
|
||||
cleanedTable.(varNames{i}) = string(col);
|
||||
end
|
||||
end
|
||||
|
||||
% Case 2: If it's already a string array
|
||||
elseif isstring(col)
|
||||
numericCol = str2double(col);
|
||||
if all(isnan(numericCol) == strcmpi(col, "NaN"))
|
||||
cleanedTable.(varNames{i}) = numericCol;
|
||||
else
|
||||
% Try convert to datetime
|
||||
try
|
||||
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
catch
|
||||
% Leave as string
|
||||
end
|
||||
end
|
||||
|
||||
% Case 3: If it's already numeric, keep as is
|
||||
elseif isnumeric(col)
|
||||
continue;
|
||||
|
||||
% Case 4: If it's datetime, keep as is
|
||||
elseif isdatetime(col)
|
||||
continue;
|
||||
|
||||
else
|
||||
% Catch-all for unexpected types, convert to string
|
||||
cleanedTable.(varNames{i}) = string(col);
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function out = convertStructToString(x)
|
||||
% Helper function to convert struct NaN to string 'NaN'
|
||||
if isstruct(x)
|
||||
out = "NaN";
|
||||
elseif isstring(x) || ischar(x)
|
||||
out = string(x);
|
||||
else
|
||||
out = x;
|
||||
end
|
||||
end
|
||||
599
projects/ECOC_2025_MPI/plots_from_database/plot_mpi_trial.m
Normal file
599
projects/ECOC_2025_MPI/plots_from_database/plot_mpi_trial.m
Normal file
@@ -0,0 +1,599 @@
|
||||
|
||||
|
||||
% dsp_options.database_type = 'mysql';
|
||||
% dsp_options.dataBase = 'labor';
|
||||
% dsp_options.storage_path = 'Z:\2025\ECOC Silas\ecoc_2025\';
|
||||
% database = DBHandler("dataBase", [dsp_options.dataBase], "type", dsp_options.database_type);
|
||||
% filterParams = database.tables;
|
||||
% filterParams.Runs.loop_id = 209;
|
||||
% % filterParams.Configurations = struct( ...
|
||||
% % 'symbolrate', 112e9, ... %[224,336,360,390,420,448]
|
||||
% % 'fiber_length', 0, ...
|
||||
% % 'db_mode', '"no_db"', ...
|
||||
% % 'interference_attenuation', [], ...
|
||||
% % 'interference_path_length', 1000, ...
|
||||
% % 'is_mpi', 1, ...
|
||||
% % 'pam_level', 4, ...
|
||||
% % 'wavelength', 1310, ...
|
||||
% % 'precomp_amp', [], ...
|
||||
% % 'signal_attenuation', [], ...
|
||||
% % 'v_awg', [], ...
|
||||
% % 'v_bias', [] ...
|
||||
% % );
|
||||
%
|
||||
% % if 1
|
||||
% % % filterParams.EqualizerParameters.dc_buffer_len = 1;
|
||||
% % filterParams.EqualizerParameters.ffe_buffer_len = 1;
|
||||
% % filterParams.EqualizerParameters.smoothing_buffer_len = 4096;
|
||||
% % filterParams.EqualizerParameters.smoothing_buffer_update = 224;
|
||||
% % filterParams.EqualizerParameters.DCmu = 0;
|
||||
% % end
|
||||
% a = database.getTableFieldNames('Runs');
|
||||
% b = database.getTableFieldNames('Results');
|
||||
% c = database.getTableFieldNames('EqualizerParameters');
|
||||
% d = [a;b;c];
|
||||
%
|
||||
% [dataTable,~] = database.queryDB(filterParams, d);
|
||||
%
|
||||
% selectedFields = {'Configurations.run_id' 'Runs.loop_id' 'Runs.date_of_run' 'Runs.rx_raw_path' 'Runs.bitrate' 'Runs.v_bias' 'Runs.v_awg' 'Runs.precomp_amp' 'Runs.symbolrate' 'Runs.pam_level'...
|
||||
% 'Runs.db_mode' 'Runs.rop_attenuation' 'Runs.is_mpi' 'Runs.interference_attenuation' 'Runs.interference_path_length' 'Runs.signal_attenuation' ...
|
||||
% 'EqualizerParameters.equalizer_structure' 'EqualizerParameters.diff_precode' 'EqualizerParameters.eq_id' 'EqualizerParameters.dc_buffer_len' 'EqualizerParameters.ffe_buffer_len' 'EqualizerParameters.smoothing_buffer_len' 'EqualizerParameters.smoothing_buffer_update' 'EqualizerParameters.DCmu' 'Measurements.power_pd_in' ...
|
||||
% 'Measurements.power_mpi_interference' 'Measurements.power_mpi_signal' 'Results.BER' 'Results.BER_precoded' 'Results.EVM' 'Results.SNR' 'Results.GMI' 'Results.Alpha' 'Results.date_of_processing'};
|
||||
|
||||
db = DBHandler("type","mysql","dataBase",'labor');
|
||||
|
||||
fp = QueryFilter();
|
||||
% fp.where('mpi_superview', 'loop_id','EQUALS', 209);
|
||||
fp.where('mpi_superview', 'symbolrate','EQUALS', 112e9);
|
||||
fp.where('mpi_superview', 'pam_level','EQUALS', 4);
|
||||
fn = [db.getTableFieldNames('mpi_superview')];
|
||||
[dataTable,sql_query] = db.queryDB(fp,fn);
|
||||
|
||||
|
||||
%%
|
||||
|
||||
dataTable_clean = dataTable;
|
||||
dataTable_clean.SIR = -7 - round(dataTable_clean.power_mpi_interference);
|
||||
dataTable_clean.NGMI = dataTable_clean.GMI ./ log2(dataTable_clean.pam_level);
|
||||
dataTable_clean = cleanUpTable(dataTable_clean);
|
||||
|
||||
dataTable_clean(dataTable_clean.BER>0.2,:) = [];
|
||||
|
||||
%%
|
||||
|
||||
|
||||
cols = linspecer(8); % Ensure color count matches
|
||||
figure()
|
||||
tiledlayout(1, 4, 'TileSpacing', 'compact', 'Padding', 'compact');
|
||||
y_here = 0;
|
||||
figcnt = 0;
|
||||
for int_len = [0,50,300,1000]
|
||||
figcnt = figcnt+1;
|
||||
% figure(int_len+1);
|
||||
nexttile;
|
||||
hold on
|
||||
mode = 4;
|
||||
|
||||
|
||||
for mode = [1,2]
|
||||
|
||||
hold on;
|
||||
dataTable = dataTable_clean;
|
||||
|
||||
|
||||
plotBoundaries = 1;
|
||||
plotRealizations = 1;
|
||||
cols = linspecer(8); % Ensure color count matches
|
||||
|
||||
if mode == 1
|
||||
% No compensation method
|
||||
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
|
||||
dataTable = dataTable(dataTable.DCmu == 0, :);
|
||||
cols = cols(1:1+1,:);
|
||||
method = 'ffe only';
|
||||
|
||||
% slow DC smoothing
|
||||
elseif mode == 2
|
||||
|
||||
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_len == 4096, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_update == 224, :);
|
||||
dataTable = dataTable(dataTable.DCmu == 0, :);
|
||||
|
||||
cols = cols(4:4+1,:);
|
||||
method = 'dc smoothing';
|
||||
|
||||
|
||||
% slow DC tracking
|
||||
elseif mode == 3
|
||||
|
||||
dataTable = dataTable(dataTable.dc_buffer_len == 224, :);
|
||||
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
|
||||
dataTable = dataTable(dataTable.DCmu == 0.005, :);
|
||||
|
||||
cols = cols(3:3+1,:);
|
||||
method = 'parallelized dc tracking';
|
||||
|
||||
elseif mode == 4
|
||||
|
||||
% ideal DC tracking
|
||||
dataTable = dataTable(dataTable.dc_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.ffe_buffer_len == 1, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_len == 0, :);
|
||||
dataTable = dataTable(dataTable.smoothing_buffer_update == 0, :);
|
||||
dataTable = dataTable(dataTable.DCmu == 0.005, :);
|
||||
|
||||
cols = cols(2:2+1,:);
|
||||
method = 'ideal dc tracking';
|
||||
end
|
||||
|
||||
% dataTable(dataTable.eq_id==0,:) = [];
|
||||
dataTable(dataTable.equalizer_structure~=1,:) = [];
|
||||
|
||||
% Modify values in 'interference_path_length' where the condition is met
|
||||
dataTable.interference_path_length(dataTable.interference_path_length < 101 & dataTable.interference_path_length > 1) = 50;
|
||||
|
||||
dataTable.interference_path_length(dataTable.interference_path_length == 1) = 0;
|
||||
dataTable = dataTable(dataTable.interference_path_length == int_len, :);
|
||||
|
||||
|
||||
dataTable(dataTable.run_id == 3866, :) = [];
|
||||
dataTable(dataTable.run_id == 3865, :) = [];
|
||||
dataTable(dataTable.run_id == 3796, :) = [];
|
||||
dataTable(dataTable.run_id == 3797, :) = [];
|
||||
dataTable(dataTable.run_id == 3798, :) = [];
|
||||
dataTable(dataTable.run_id == 4002, :) = [];
|
||||
dataTable(dataTable.run_id == 4200, :) = [];
|
||||
dataTable(dataTable.run_id == 4199, :) = [];
|
||||
|
||||
% 0
|
||||
% 1
|
||||
% 10
|
||||
% 15
|
||||
% 20
|
||||
% 50
|
||||
% 100
|
||||
% 300
|
||||
% 1000
|
||||
|
||||
% dataTable(dataTable.interference_path_length ~= 50, :) = [];
|
||||
% dataTable = dataTable(dataTable.interference_path_length < 51, :);
|
||||
|
||||
% dataTable(dataTable.loop_id<200,:) = [];
|
||||
|
||||
% Filter by time
|
||||
filter_by_time = 0;
|
||||
if filter_by_time
|
||||
startTime = datetime('2025-04-20 18:00:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
stopTime = datetime('2025-04-30 19:30:00', 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
dataTable.date_of_run = datetime(dataTable.date_of_run, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
dataTable.date_of_processing = datetime(dataTable.date_of_processing, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
dataTable = dataTable(dataTable.date_of_processing > startTime, :);
|
||||
dataTable = dataTable(dataTable.date_of_processing < stopTime, :);
|
||||
end
|
||||
|
||||
% Group by smth
|
||||
y_var = 'BER';
|
||||
x_var = 'SIR';
|
||||
|
||||
loop_var = 'interference_path_length';
|
||||
fixedVars = {'equalizer_structure','interference_path_length',x_var};
|
||||
|
||||
[dataTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var);
|
||||
|
||||
dataTableGrpd_mean = groupIt(fixedVars, dataTable, @mean);
|
||||
dataTableGrpd_min = groupIt(fixedVars, dataTable, @min);
|
||||
dataTableGrpd_max = groupIt(fixedVars, dataTable, @max);
|
||||
|
||||
% dataTableGrpd_mean(dataTableGrpd_mean.nRows<50,:) = [];
|
||||
% dataTableGrpd_min(dataTableGrpd_min.nRows<50,:) = [];
|
||||
% dataTableGrpd_max(dataTableGrpd_max.nRows<50,:) = [];
|
||||
|
||||
% Create a new figure
|
||||
|
||||
hold on
|
||||
|
||||
unique_loop_var = unique(dataTable.(loop_var));
|
||||
|
||||
for i = 1:numel(unique_loop_var)
|
||||
|
||||
% Prepare filtered data for this loop variable
|
||||
loopValue = unique_loop_var(i);
|
||||
|
||||
loopFiltGrpd = dataTableGrpd_mean.(loop_var) == loopValue;
|
||||
if ~any(loopFiltGrpd)
|
||||
continue; % Skip if no data for this loop var
|
||||
end
|
||||
|
||||
% Extract values
|
||||
x_values = dataTableGrpd_mean.(x_var)(loopFiltGrpd, :);
|
||||
y_mean = dataTableGrpd_mean.(y_var)(loopFiltGrpd, :);
|
||||
y_min = dataTableGrpd_min.(y_var)(loopFiltGrpd, :);
|
||||
y_max = dataTableGrpd_max.(y_var)(loopFiltGrpd, :);
|
||||
|
||||
% Compute bounds: distance from mean
|
||||
y_lower = y_mean - y_min;
|
||||
y_upper = y_max - y_mean;
|
||||
y_bounds = [y_lower, y_upper];
|
||||
|
||||
% Display name (optional)
|
||||
try
|
||||
idx = find(dataTable.(loop_var) == loopValue, 1, 'first');
|
||||
% dispname = char(equalizer_structure(dataTable.equalizer_structure(idx)));
|
||||
dispname = [method];
|
||||
dispname = [dispname, '/ ',num2str(unique_loop_var(i)) ,' m'];
|
||||
% dispname = [dispname,'; ',num2str(unique(dataTable.interference_path_length)),' m'];
|
||||
% dispname = [dispname, '/ PAM ', num2str(filterParams.Configurations.pam_level)];
|
||||
% dispname = [dispname, '/ ', num2str(filterParams.Configurations.symbolrate.*1e-9),' GBd'];
|
||||
end
|
||||
|
||||
if plotBoundaries
|
||||
% Plot bounded line
|
||||
[hl, hp] = boundedline(x_values, y_mean, y_bounds, ...
|
||||
'alpha', 'transparency', 0.1, ...
|
||||
'cmap', cols(i,:), ...
|
||||
'nan', 'fill', ...
|
||||
'orientation', 'vert');
|
||||
|
||||
% % Style the main line: thinnest, dotted, no marker
|
||||
set(hl, 'LineWidth', 0.5, 'LineStyle', ':', 'Marker', 'none', ...
|
||||
'Color', cols(i,:), 'DisplayName', string(dispname));
|
||||
|
||||
plt = errorbar(x_values,y_mean,y_lower,y_upper,'LineWidth', 0.9, 'LineStyle', 'none', 'Marker', 'none','Color', cols(i,:), 'DisplayName', string(dispname),'HandleVisibility','off');
|
||||
|
||||
% Hide patch (shaded area) from legend
|
||||
set(hp, 'HandleVisibility', 'off','LineStyle',':','LineWidth',0.5,'Marker','none');
|
||||
|
||||
|
||||
|
||||
% Fit a 4th-order polynomial to log10(BER)
|
||||
p = polyfit(x_values, log10(y_mean), 3); % 4 is fitting order, adjust as needed
|
||||
|
||||
% Evaluate the fitted polynomial
|
||||
x_fit = linspace(min(x_values), max(x_values), 300); % Fine points
|
||||
y_fit_log = polyval(p, x_fit); % Still in log10 domain
|
||||
y_fit = 10.^y_fit_log; % Back to BER domain
|
||||
|
||||
|
||||
|
||||
plot(x_fit,y_fit,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none', ...
|
||||
'Color', cols(i,:), 'DisplayName', string(dispname),'HandleVisibility','off');
|
||||
|
||||
% % Add invisible scatter for DataTips
|
||||
% plt = scatter(x_values, y_mean, ...
|
||||
% 'Marker', 'o', 'MarkerEdgeColor', 'none', 'MarkerFaceColor', 'none', ...
|
||||
% 'HandleVisibility', 'off', 'PickableParts', 'all');
|
||||
else
|
||||
|
||||
plt= plot(x_values,y_mean,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none', ...
|
||||
'Color', cols(i,:), 'DisplayName', string(dispname));
|
||||
% plt= errorbar(x_values,y_mean,y_lower,y_upper,'LineWidth', 1, 'LineStyle', '-', 'Marker', 'none','Color', cols(i,:), 'DisplayName', string(dispname));
|
||||
|
||||
end
|
||||
% Add data tips to the invisible scatter
|
||||
pair_one = {'Run ID', dataTableGrpd_mean.run_id(loopFiltGrpd, :)};
|
||||
pair_two = {'Rate', dataTableGrpd_mean.bitrate(loopFiltGrpd, :) * 1e-9};
|
||||
pair_three = {'PD in', round(dataTableGrpd_mean.power_pd_in(loopFiltGrpd, :), 2)};
|
||||
addDatatips(plt, pair_one, pair_two, pair_three);
|
||||
|
||||
|
||||
|
||||
% Optionally: outline bounds for better visibility (optional)
|
||||
% hnew = outlinebounds(hl, hp);
|
||||
|
||||
% Tick marks (x-axis)
|
||||
|
||||
|
||||
% Optional: scatter realizations
|
||||
if plotRealizations
|
||||
loopFiltSingle = dataTable.(loop_var) == loopValue;
|
||||
x_single = double(dataTable.(x_var)(loopFiltSingle, :));
|
||||
y_single = double(dataTable.(y_var)(loopFiltSingle, :));
|
||||
|
||||
mkr = '.';
|
||||
sc = scatter(x_single+(mode*0.1)-0.2, y_single,15, 'Marker', mkr, 'MarkerEdgeColor', cols(i, :), ...
|
||||
'LineWidth', 0.5, 'HandleVisibility', 'off', 'DisplayName', string(dispname));
|
||||
|
||||
pair_one = {'Run ID', dataTable.run_id(loopFiltSingle, :)};
|
||||
pair_two = {'Baud', dataTable.symbolrate(loopFiltSingle, :) * 1e-9};
|
||||
pair_three = {'PD in', round(dataTable.power_pd_in(loopFiltSingle, :), 2)};
|
||||
pair_four = {'#bits', round(dataTable.numBits(loopFiltSingle, :), 2)};
|
||||
addDatatips(sc, pair_one, pair_two, pair_three,pair_four);
|
||||
end
|
||||
end
|
||||
|
||||
% Label axes and title
|
||||
legend('Interpreter', 'latex');
|
||||
xlabel(x_var);
|
||||
if ~y_here
|
||||
ylabel(y_var);
|
||||
yticklabels = [];
|
||||
|
||||
y_here = 1;
|
||||
end
|
||||
if int_len ~= 0
|
||||
set(gca, 'YTick', []);
|
||||
end
|
||||
% title([x_var, ' vs. ', y_var]);
|
||||
|
||||
if string(y_var) == "BER"
|
||||
yline(2.2e-4, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
|
||||
yline(3.8e-3, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
|
||||
yline(2e-2, 'LineWidth', 1, 'LineStyle', '--', 'HandleVisibility', 'off');
|
||||
ylim([1e-5, 0.1]);
|
||||
end
|
||||
|
||||
xlim([15,35]);
|
||||
ylim([9e-5 0.1 ]);
|
||||
xticks([13:2:35]);
|
||||
|
||||
% Enable grid and beautify
|
||||
grid on;
|
||||
beautifyBERplot;
|
||||
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
function resultTable = groupIt(fixedVars, dataTable, aggregationFunction)
|
||||
% groupIt Groups data in a table based on fixedVars and applies aggregationFunction to numeric data.
|
||||
%
|
||||
% resultTable = groupIt(fixedVars, dataTable, aggregationFunction)
|
||||
%
|
||||
% Inputs:
|
||||
% fixedVars - Cell array of variable names to group by
|
||||
% dataTable - Input MATLAB table
|
||||
% aggregationFunction - Function handle (e.g., @mean, @min, @max)
|
||||
%
|
||||
% Output:
|
||||
% resultTable - Grouped and aggregated table
|
||||
|
||||
% Group data
|
||||
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
|
||||
|
||||
% Prepare aggregation
|
||||
varNames = dataTable.Properties.VariableNames;
|
||||
nVars = numel(varNames);
|
||||
aggData = cell(height(groupKeys), nVars);
|
||||
groupCount = zeros(height(groupKeys), 1); % Store number of rows in each group
|
||||
|
||||
% Loop over groups
|
||||
for i = 1:height(groupKeys)
|
||||
idx = (G == i); % Logical index for group i
|
||||
groupCount(i) = sum(idx); % Count rows in group
|
||||
|
||||
% Loop over each variable
|
||||
for j = 1:nVars
|
||||
colData = dataTable.(varNames{j});
|
||||
|
||||
if isnumeric(colData)
|
||||
% Numeric: apply aggregation function (skip empty groups safely)
|
||||
if any(idx)
|
||||
% aggData{i, j} = rmoutliers(double(colData(idx)));
|
||||
aggData{i, j} = aggregationFunction(colData(idx));
|
||||
else
|
||||
aggData{i, j} = NaN;
|
||||
end
|
||||
else
|
||||
% Non-numeric: take first non-empty value
|
||||
if iscell(colData)
|
||||
nonEmptyIdx = find(idx & ~cellfun(@isempty, colData), 1);
|
||||
if ~isempty(nonEmptyIdx)
|
||||
aggData{i, j} = colData{nonEmptyIdx};
|
||||
else
|
||||
aggData{i, j} = [];
|
||||
end
|
||||
else
|
||||
nonEmptyIdx = find(idx, 1);
|
||||
if ~isempty(nonEmptyIdx)
|
||||
aggData{i, j} = colData(nonEmptyIdx);
|
||||
else
|
||||
aggData{i, j} = [];
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
% Convert aggregated data to table
|
||||
resultTable = cell2table(aggData, 'VariableNames', varNames);
|
||||
|
||||
% Add group size as new column
|
||||
resultTable.nRows = groupCount;
|
||||
end
|
||||
|
||||
|
||||
function addDatatips(sc, varargin)
|
||||
% addDatatips Adds custom data tip rows to a scatter plot.
|
||||
%
|
||||
% addDatatips(sc, pair1, pair2, ...) adds one or more custom rows to the
|
||||
% data tip display of the scatter plot identified by sc.
|
||||
%
|
||||
% Each pair should be provided as a 1x2 cell array: {label, value}.
|
||||
% The value can be a scalar or a vector. If a vector is provided, its length
|
||||
% must match the number of scatter plot points.
|
||||
%
|
||||
% Example:
|
||||
% sc = scatter(x, y, 'LineWidth', 1.5, 'Marker', 'o');
|
||||
% pair_one = {'Attenuation', attenuationVector};
|
||||
% addDatatips(sc, pair_one);
|
||||
|
||||
numPoints = numel(sc.XData);
|
||||
|
||||
for k = 1:length(varargin)
|
||||
pair = varargin{k};
|
||||
|
||||
if ~iscell(pair) || numel(pair) ~= 2
|
||||
error('Each pair must be a 1x2 cell array: {label, value}.');
|
||||
end
|
||||
|
||||
label = pair{1};
|
||||
value = pair{2};
|
||||
|
||||
% If value is a vector, ensure its length is either 1 or equal to the number of scatter points.
|
||||
if isvector(value) && numel(value) ~= 1 && numel(value) ~= numPoints
|
||||
error('The vector for "%s" must be a scalar or have %d elements matching the scatter data points.', label, numPoints);
|
||||
end
|
||||
|
||||
% Create a new data tip row using the provided label and vector.
|
||||
newRow = dataTipTextRow(label, value);
|
||||
sc.DataTipTemplate.DataTipRows(end+1) = newRow;
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
function cleanedTable = cleanUpTable(inputTable)
|
||||
% cleanUpTable Cleans a MATLAB table where numbers and NaNs are stored as strings or structs.
|
||||
%
|
||||
% cleanedTable = cleanUpTable(inputTable)
|
||||
%
|
||||
% This function goes through all columns of the input table:
|
||||
% - Converts strings of numbers to numeric values
|
||||
% - Converts string 'NaN' and struct NaNs to real NaN
|
||||
% - Converts date strings to datetime (if possible)
|
||||
%
|
||||
% Input:
|
||||
% inputTable - MATLAB table with mixed types
|
||||
%
|
||||
% Output:
|
||||
% cleanedTable - Cleaned MATLAB table with proper numeric types
|
||||
|
||||
cleanedTable = inputTable;
|
||||
varNames = cleanedTable.Properties.VariableNames;
|
||||
|
||||
for i = 1:numel(varNames)
|
||||
col = cleanedTable.(varNames{i});
|
||||
|
||||
% Case 1: If it's a cell array (likely mixed strings/struct)
|
||||
if iscell(col)
|
||||
% Convert struct 'NaN' entries to string 'NaN'
|
||||
col = cellfun(@(x) convertStructToString(x), col, 'UniformOutput', false);
|
||||
|
||||
% Try to convert string numbers to actual numbers
|
||||
numericCol = str2double(col);
|
||||
|
||||
if all(isnan(numericCol) == strcmpi(col, 'NaN') | cellfun(@isempty, col))
|
||||
% If conversion is successful (NaNs correspond to 'NaN' strings), use it
|
||||
cleanedTable.(varNames{i}) = numericCol;
|
||||
else
|
||||
% Else, try to convert to datetime
|
||||
try
|
||||
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss.SSSSSS');
|
||||
catch
|
||||
% If it fails, leave as cell array of strings
|
||||
cleanedTable.(varNames{i}) = string(col);
|
||||
end
|
||||
end
|
||||
|
||||
% Case 2: If it's already a string array
|
||||
elseif isstring(col)
|
||||
numericCol = str2double(col);
|
||||
if all(isnan(numericCol) == strcmpi(col, "NaN"))
|
||||
cleanedTable.(varNames{i}) = numericCol;
|
||||
else
|
||||
% Try convert to datetime
|
||||
try
|
||||
cleanedTable.(varNames{i}) = datetime(col, 'InputFormat', 'yyyy-MM-dd HH:mm:ss');
|
||||
catch
|
||||
% Leave as string
|
||||
end
|
||||
end
|
||||
|
||||
% Case 3: If it's already numeric, keep as is
|
||||
elseif isnumeric(col)
|
||||
continue;
|
||||
|
||||
% Case 4: If it's datetime, keep as is
|
||||
elseif isdatetime(col)
|
||||
continue;
|
||||
|
||||
else
|
||||
% Catch-all for unexpected types, convert to string
|
||||
cleanedTable.(varNames{i}) = string(col);
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function out = convertStructToString(x)
|
||||
% Helper function to convert struct NaN to string 'NaN'
|
||||
if isstruct(x)
|
||||
out = "NaN";
|
||||
elseif isstring(x) || ischar(x)
|
||||
out = string(x);
|
||||
else
|
||||
out = x;
|
||||
end
|
||||
end
|
||||
|
||||
function [cleanedTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var)
|
||||
|
||||
% Group the data
|
||||
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
|
||||
|
||||
% Initialize logical index to keep rows
|
||||
keepIdx = true(height(dataTable), 1);
|
||||
|
||||
% Prepare storage for outliers
|
||||
outlierRecords = [];
|
||||
|
||||
% Loop over each group
|
||||
for groupIdx = 1:height(groupKeys)
|
||||
% Find indices of current group
|
||||
groupRows = (G == groupIdx);
|
||||
|
||||
% Extract y-values of this group
|
||||
y_values = dataTable.(y_var)(groupRows);
|
||||
|
||||
% Skip groups with fewer than 3 points (optional)
|
||||
if sum(groupRows) < 3
|
||||
continue;
|
||||
end
|
||||
|
||||
% Detect outliers in log space
|
||||
y_log = log10(y_values);
|
||||
outlierMask = isoutlier(y_log, 'quartiles',1); % or 'median', 'grubbs', etc.
|
||||
|
||||
% If any outliers found, collect their data
|
||||
if any(outlierMask)
|
||||
groupData = dataTable(groupRows, :);
|
||||
|
||||
% Prepare table for current group outliers
|
||||
outlierGroupTable = groupData(outlierMask, :);
|
||||
|
||||
% Add group key values for traceability
|
||||
for k = 1:numel(fixedVars)
|
||||
outlierGroupTable.(['Group_', fixedVars{k}]) = repmat(groupKeys{groupIdx, k}, height(outlierGroupTable), 1);
|
||||
end
|
||||
|
||||
% Append to collection
|
||||
outlierRecords = [outlierRecords; outlierGroupTable]; %#ok<AGROW>
|
||||
end
|
||||
|
||||
% Mark outliers for removal
|
||||
groupRowIdx = find(groupRows);
|
||||
keepIdx(groupRowIdx(outlierMask)) = false;
|
||||
end
|
||||
|
||||
% Apply mask to dataTable
|
||||
cleanedTable = dataTable(keepIdx, :);
|
||||
|
||||
% Prepare output: if no outliers, return empty table
|
||||
if isempty(outlierRecords)
|
||||
outliersTable = table();
|
||||
else
|
||||
outliersTable = outlierRecords;
|
||||
end
|
||||
|
||||
nRemoved = sum(~keepIdx);
|
||||
nTotalOriginal = height(dataTable) + nRemoved;
|
||||
percentageRemoved = (nRemoved / nTotalOriginal) * 100;
|
||||
|
||||
fprintf('Removed %d outliers from the data table (%.2f%% of total %d entries).\n', ...
|
||||
nRemoved, percentageRemoved, nTotalOriginal);
|
||||
end
|
||||
Reference in New Issue
Block a user