Many changes in DBHandler

new general processing structure
just before splitting off the projects folder from this repo

Classes/DataBaseHandler/related functions/removeGroupOutliers.m

@@ -0,0 +1,61 @@
+function [cleanedTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var)
+% removeGroupOutliers removes outliers in y_var within each group defined by fixedVars.
+%
+%   [cleanedTable, outliersTable] = removeGroupOutliers(dataTable, fixedVars, y_var)
+%
+%   Inputs:
+%       dataTable  - Input MATLAB table
+%       fixedVars  - Cell array of variable names to group by
+%       y_var      - Name of the variable to check for outliers (string or char)
+%
+%   Outputs:
+%       cleanedTable  - Table with outliers removed
+%       outliersTable - Table of removed outlier rows
+
+[G, groupKeys] = findgroups(dataTable(:, fixedVars));
+keepIdx = true(height(dataTable), 1);
+outlierRecords = [];
+
+for groupIdx = 1:height(groupKeys)
+    groupRows = (G == groupIdx);
+    y_values = dataTable.(y_var)(groupRows);
+
+    % Skip groups with fewer than 3 points
+    if sum(groupRows) < 3
+        continue;
+    end
+
+    % Detect outliers in log10 space (robust for BER, etc.)
+    y_log = log10(y_values);
+    outlierMask = isoutlier(y_log, 'quartiles', 1);
+
+    if any(outlierMask)
+        groupData = dataTable(groupRows, :);
+        outlierGroupTable = groupData(outlierMask, :);
+
+        % Optionally, add group key values for traceability
+        for k = 1:numel(fixedVars)
+            outlierGroupTable.(['Group_', fixedVars{k}]) = repmat(groupKeys{groupIdx, k}, height(outlierGroupTable), 1);
+        end
+
+        outlierRecords = [outlierRecords; outlierGroupTable]; %#ok<AGROW>
+    end
+
+    % Mark outliers for removal
+    groupRowIdx = find(groupRows);
+    keepIdx(groupRowIdx(outlierMask)) = false;
+end
+
+cleanedTable = dataTable(keepIdx, :);
+
+if isempty(outlierRecords)
+    outliersTable = table();
+else
+    outliersTable = outlierRecords;
+end
+
+nRemoved = sum(~keepIdx);
+nTotalOriginal = height(dataTable);
+fprintf('Removed %d outliers from the data table (%.2f%% of total %d entries).\n', ...
+    nRemoved, 100*nRemoved/nTotalOriginal, nTotalOriginal);
+end