9ce23c4a10
new general processing structure just before splitting off the projects folder from this repo
61 lines
1.9 KiB
Matlab
61 lines
1.9 KiB
Matlab
function [cleanedTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var)
|
|
% removeGroupOutliers removes outliers in y_var within each group defined by fixedVars.
|
|
%
|
|
% [cleanedTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var)
|
|
%
|
|
% Inputs:
|
|
% dataTable - Input MATLAB table
|
|
% fixedVars - Cell array of variable names to group by
|
|
% y_var - Name of the variable to check for outliers (string or char)
|
|
%
|
|
% Outputs:
|
|
% cleanedTable - Table with outliers removed
|
|
% outliersTable - Table of removed outlier rows
|
|
|
|
[G, groupKeys] = findgroups(dataTable(:, fixedVars));
|
|
keepIdx = true(height(dataTable), 1);
|
|
outlierRecords = [];
|
|
|
|
for groupIdx = 1:height(groupKeys)
|
|
groupRows = (G == groupIdx);
|
|
y_values = dataTable.(y_var)(groupRows);
|
|
|
|
% Skip groups with fewer than 3 points
|
|
if sum(groupRows) < 3
|
|
continue;
|
|
end
|
|
|
|
% Detect outliers in log10 space (robust for BER, etc.)
|
|
y_log = log10(y_values);
|
|
outlierMask = isoutlier(y_log, 'quartiles', 1);
|
|
|
|
if any(outlierMask)
|
|
groupData = dataTable(groupRows, :);
|
|
outlierGroupTable = groupData(outlierMask, :);
|
|
|
|
% Optionally, add group key values for traceability
|
|
for k = 1:numel(fixedVars)
|
|
outlierGroupTable.(['Group_', fixedVars{k}]) = repmat(groupKeys{groupIdx, k}, height(outlierGroupTable), 1);
|
|
end
|
|
|
|
outlierRecords = [outlierRecords; outlierGroupTable]; %#ok<AGROW>
|
|
end
|
|
|
|
% Mark outliers for removal
|
|
groupRowIdx = find(groupRows);
|
|
keepIdx(groupRowIdx(outlierMask)) = false;
|
|
end
|
|
|
|
cleanedTable = dataTable(keepIdx, :);
|
|
|
|
if isempty(outlierRecords)
|
|
outliersTable = table();
|
|
else
|
|
outliersTable = outlierRecords;
|
|
end
|
|
|
|
nRemoved = sum(~keepIdx);
|
|
nTotalOriginal = height(dataTable);
|
|
fprintf('Removed %d outliers from the data table (%.2f%% of total %d entries).\n', ...
|
|
nRemoved, 100*nRemoved/nTotalOriginal, nTotalOriginal);
|
|
end |