Files
imdd_silas/Classes/DataBaseHandler/DBHandler.m
sioe e47a4dbbbe Many changes for 400G DSP
Minimal Example
...
2024-12-17 16:17:58 +01:00

660 lines
28 KiB
Matlab

classdef DBHandler < handle
% DBHANDLER Class to handle database queries
% This class provides methods to interact with an SQLite database, including
% inserting data, retrieving table names, and appending new rows.
properties
conn % Database connection object
pathToDB % Path to the SQLite database
tableNames % Cell array containing names of all tables in the database
tables = struct(); % Structure containing MATLAB tables for each database table
distinctValues
end
methods
function obj = DBHandler(options)
% DBHANDLER Constructor for the DBHandler class
% Initializes the database connection and retrieves table and field names.
%
% Usage:
% obj = DBHandler('pathToDB', 'path/to/database.db');
arguments
options.pathToDB = ""; % Default value for pathToDB if not provided
end
% Assign values to class properties based on input arguments
fn = fieldnames(options);
for n = 1:numel(fn)
try
obj.(fn{n}) = options.(fn{n});
end
end
% Establish a connection to the SQLite database
try
obj.conn = sqlite(obj.pathToDB);
catch e
error('Failed to connect to the database: %s', e.message);
end
if obj.dbIsHealthy
obj.refresh();
else
error('DB seems to be corrupt')
end
end
function obj = refresh(obj)
% Get table names and the first rows of each table to understand the structure
obj.getTableNames();
obj.getTables();
obj.getDistinctValues();
end
function obj = getTableNames(obj)
% Get all table names from the database
try
result = fetch(obj.conn, 'SELECT name FROM sqlite_master WHERE type="table"');
obj.tableNames = result.name;
catch e
error('Failed to retrieve table names: %s', e.message);
end
end
function obj = getTables(obj)
% Get a preview (first row) of each table to understand its structure
for i = 1:numel(obj.tableNames)
try
tableName = obj.tableNames{i};
results = fetch(obj.conn, sprintf('SELECT * FROM %s WHERE 1 = 2', tableName));
% Matlab cant handle if there is a NULL in a returned
% datarow... therefore do not return a row using the
% above condition which is never true
% results = sqlread(obj.conn, tableName, MaxRows=1);
for l = 1:numel(results.Properties.VariableNames)
varName = results.Properties.VariableNames{l};
obj.tables.(tableName).(varName) = []; % Store the preview as a reference
end
catch e
warning('Failed to read the table %s: %s', tableName, e.message);
end
end
end
function obj = getDistinctValues(obj)
% getDistinctValues Retrieves distinct values for each relevant field in all tables
% excluding fields ending with "_id". Stores distinct values in the 'distinctValues'
% property.
% Initialize a structure to store distinct values for each table
obj.distinctValues = struct();
% Iterate over each table in obj.tables
tableNames_ = fieldnames(obj.tables);
for i = 1:numel(tableNames_)
tableName = tableNames_{i};
% Initialize a sub-struct to store distinct values for each field in the table
obj.distinctValues.(tableName) = struct();
% Get all fields of the current table
fieldNames = fieldnames(obj.tables.(tableName));
% Iterate over each field
for j = 1:numel(fieldNames)
fieldName = fieldNames{j};
% % Skip fields ending with '_id' as they don't contain useful distinct values
% if endsWith(fieldName, '_id')
% continue;
% end
% Construct SQL to get distinct values for the current field
query = sprintf('SELECT DISTINCT %s FROM %s', fieldName, tableName);
% Execute query and fetch distinct values
try
result = fetch(obj.conn, query);
% Store the distinct values in the structure
if ~isempty(result)
distinctValues = table2array(result);
else
distinctValues = [];
end
obj.distinctValues.(tableName).(fieldName) = distinctValues;
catch e
% warning('Failed to retrieve distinct values for %s.%s: %s', tableName, fieldName, e.message);
obj.distinctValues.(tableName).(fieldName) = [];
end
end
end
end
function healthyDB = dbIsHealthy(obj)
healthyDB = false;
num_runs = obj.fetch('SELECT COUNT(*) AS total_runs FROM Runs');
num_configs = obj.fetch('SELECT COUNT(*) AS total_configurations FROM Configurations');
num_meas = obj.fetch('SELECT COUNT(*) AS total_measurements FROM Measurements');
assert((num_runs{1,1}==num_configs{1,1})&&(num_configs{1,1}==num_meas{1,1}),'Different num of entries per table');
%Check for any duplicate paths
duplictae_raw = obj.fetch("SELECT COALESCE(Runs.rx_raw_path,'NaN') AS rx_raw_path, COUNT(*) AS occurrences FROM Runs GROUP BY rx_raw_path HAVING COUNT(*) > 1");
duplictae_sync = obj.fetch("SELECT COALESCE(Runs.rx_sync_path,'NaN') AS rx_sync_path, COUNT(*) AS occurrences FROM Runs GROUP BY rx_sync_path HAVING COUNT(*) > 1");
if size(duplictae_raw,2) > 1
for i = 1:size(duplictae_raw,1)
fprintf('Raw Rx Paths: Found %d duplictaes of %s \n',duplictae_raw.occurrences(i),duplictae_raw.rx_raw_path(i));
end
end
healthyDB = true;
end
function lastID = appendToTable(obj, tableName, newRow)
% appendToTable Appends a new row to the specified table
%
% Usage:
% appendToTable(tableName, newRow)
%
% Inputs:
% tableName: The name of the table to append data to.
% newRow: A MATLAB table or struct containing the new row to be appended.
% Check if the table exists in the fetched tables
if ~isfield(obj.tables, tableName)
error('Table %s does not exist in the database or has not been fetched.', tableName);
end
% Convert newRow to a table if it is a struct
if isstruct(newRow)
fields = fieldnames(newRow);
emptyFields = structfun(@isempty,newRow);
if sum(emptyFields)>0
newRow.(fields{emptyFields==1}) = NaN;
disp(['In Table: ',tableName,': ',fields{emptyFields==1},' was empty, is now NaN ',newRow.(fields{emptyFields==1})])
end
newRow = struct2table(newRow);
end
% Ensure the new row matches the structure of the existing table
existingTableStructure = obj.tables.(tableName);
% Perform data type checks and conversions
for colName = newRow.Properties.VariableNames
% Extract the value and its intended column type
value = newRow.(colName{1});
existingValue = existingTableStructure.(colName{1});
% If the value is a class object, convert it to JSON format
if isobject(value) && ~isdatetime(value) && ~isa(value,"string")
newRow.(colName{1}) = string(jsonencode(value));
% If the value is a character array, convert it to a string
elseif ischar(value)
newRow.(colName{1}) = string(value);
end
end
% Append the new row to the database table
try
sqlwrite(obj.conn, tableName, newRow);
% disp(['Successfully appended new row to the table ', tableName]);
catch e
error('Failed to append to the table %s: %s', tableName, e.message);
end
% Retrieve the measurement_id of the newly inserted row for linking other tables
result = fetch(obj.conn, 'SELECT last_insert_rowid()');
lastID = result{1, 1}; % Access the value directly from the table
end
function exists = checkIfRunExists(obj, table2check, column2check, value2check)
% checkIfRunExists Checks if a specific value exists in a specified column of a table
%
% Usage:
% exists = checkIfRunExists(table2check, column2check, value2check)
%
% Inputs:
% table2check: The name of the table to check for duplicates.
% column2check: The name of the column to check within the specified table.
% value2check: The value to check for in the specified column.
%
% Outputs:
% exists: Boolean indicating whether the value already exists in the table.
% Ensure the specified table and column exist in the database
if ~isfield(obj.tables, table2check)
error('Table %s does not exist in the database.', table2check);
end
% Ensure the specified column exists in the table structure
if ~isfield(obj.tables.(table2check), column2check)
error('Column %s does not exist in the table %s.', column2check, table2check);
end
% Construct the query to check for the value in the specified column
query = sprintf('SELECT COUNT(*) FROM %s WHERE %s = "%s"', table2check, column2check, value2check);
% Execute the query and pass the value2check to avoid SQL injection issues
try
result = fetch(obj.conn, query);
count = result{1, 1}; % Extract the count from the result
catch e
error('Failed to execute the duplicate check query: %s', e.message);
end
% If count is greater than 0, then the value exists in the table
exists = count > 0;
if exists
disp(['The value "', value2check, '" already exists in the column "', column2check, '" of the table "', table2check, '".']);
else
% disp(['The value "', value2check, '" does not exist in the column "', column2check, '" of the table "', table2check, '".']);
end
end
function addBEREntry(obj, berValue, occurrence, runID, ffe, dfe, mlse, pf, eqType, ffe_order, dfe_order, len_tr, mu_ffe, mu_dfe, mu_dc, comment)
% addBEREntry Adds a BER entry linked to an existing or new Equalizer entry.
% Usage:
% addBEREntry(runID, eq, ffe, dfe, mlse, pf, eqType, ffe_order, dfe_order, len_tr, mu_ffe, mu_dfe, mu_dc, berValue, comment)
if isempty(pf)
postfilter_taps = [];
else
postfilter_taps = pf.burg_coeff;
end
% Create equalizer data struct for searching and adding if necessary
equalizerData = struct( ...
'ffe', jsonencode(ffe), ...
'dfe', jsonencode(dfe), ...
'mlse', jsonencode(mlse), ...
'pf', jsonencode(pf), ...
'eq_type', string(eqType), ...
'ffe_order', jsonencode(ffe_order), ...
'dfe_order', jsonencode(dfe_order), ...
'postfilter_taps',jsonencode(postfilter_taps),...
'len_tr', len_tr, ...
'mu_ffe', jsonencode(mu_ffe), ...
'mu_dfe', mu_dfe, ...
'mu_dc', mu_dc, ...
'comment', comment ...
);
% Check if exact Equalizer and BER entries already exist in the DB ...
selectedFields = {'Runs.run_id','BERs.ber_id','Equalizer.eq_id','BERs.ber',['BERs.occurrence' ...
'']};
filterParams = obj.tables;
filterParams.Equalizer = equalizerData;
[dataTable,sql_query] = obj.queryDB(filterParams, selectedFields);
% get or insert Equalizer
if ~isempty(dataTable)
% Equalizer entry already exists, use the existing eq_id
cur_eq_id = dataTable.eq_id;
else
% Insert the new Equalizer entry
cur_eq_id = obj.appendToTable('Equalizer', equalizerData);
end
% skip if already here or insert BER entry
if ~isempty(dataTable)
% A BER entry with the same eq_id, run_id, and occurrence already exists
existingBERValue = dataTable.ber;
% Compare the existing BER value with the new BER value
if existingBERValue == berValue
fprintf('The BER entry %.2e || -- eq_id: %d -- run_id: %d -- occurrence: %d already exists. \n',berValue, cur_eq_id, runID, occurrence);
else
fprintf('Already found BER for EQ: %.2e ~= %.2e || -- eq_id: %d -- run_id: %d -- occurrence: %d already exists.\n', berValue, existingBERValue, cur_eq_id, runID, occurrence);
end
else
% No such BER entry exists, insert the new BER entry
berData = struct( ...
'run_id', runID, ...
'eq_id', cur_eq_id, ...
'ber', berValue, ...
'occurrence', occurrence ...
);
obj.appendToTable('BERs', berData);
end
end
function executeSQL(obj, query)
% This method executes an SQL statement using MATLAB's execute function.
execute(obj.conn, query);
end
function answer = fetch(obj,query)
answer = fetch(obj.conn,query);
end
function [result,query] = queryDB(obj, filterParams, selectedFields)
% getPathsWithFlexibleFilter Retrieves values from Runs table with flexible filtering
% and lets the user select which fields to include in the SELECT statement.
%
% Usage:
% [rxRawPaths, filteredValues] = getPathsWithFlexibleFilter(filterParams)
%
% Inputs:
% filterParams: A structure containing the parameters with their values.
% If left empty, two popup windows will prompt the user for input.
%
% Outputs:
% result: table with sql return
% query: this was send to SQL DB
arguments
obj
filterParams = [];
selectedFields = [];
end
% Step 1: Prompt the user to input filter parameters if not provided
if isempty(filterParams)
filterParams = obj.promptFilterParameters();
end
% Step 2: Prompt the user to select fields to include in the SELECT statement
if isempty(selectedFields)
selectedFields = obj.promptSelectFields();
else
if iscell(selectedFields)
elseif isstruct(selectedFields)
end
end
% Step 3: Construct the SQL query based on the inputs
query = obj.constructSQLQuery(filterParams, selectedFields);
% Step 4: Execute the query and handle results
result = obj.fetch(query);
end
function query = constructSQLQuery(obj, filterParams, selectedFields)
% constructSQLQuery Constructs the SQL query based on filter parameters and selected fields.
% Construct the SELECT clause dynamically based on user selection
selectClause = 'SELECT DISTINCT ';
for i = 1:numel(selectedFields)
fieldParts = strsplit(selectedFields{i}, '.');
tableName = fieldParts{1};
fieldName = fieldParts{2};
if isnumeric(obj.tables.(tableName).(fieldName))
selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', ''NaN'') AS ', fieldName];
else
selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', '''') AS ', fieldName];
end
if i < numel(selectedFields)
selectClause = [selectClause, ', '];
else
selectClause = [selectClause, ' '];
end
end
% Construct the FROM and WHERE clause
baseQuery = [selectClause, 'FROM Runs ' ...
'LEFT JOIN Configurations ON Runs.run_id = Configurations.run_id ' ...
'LEFT JOIN Measurements ON Runs.run_id = Measurements.run_id ' ...
'LEFT JOIN BERs ON Runs.run_id = BERs.run_id ' ...
'LEFT JOIN Equalizer ON BERs.eq_id = Equalizer.eq_id ' ...
'WHERE '];
% Loop through each table in filterParams
filterClauses = [];
tableNames_ = fieldnames(filterParams);
for t = 1:numel(tableNames_)
tableName = tableNames_{t};
tableParams = filterParams.(tableName);
% Loop through each parameter in the table
fieldNames = fieldnames(tableParams);
for i = 1:numel(fieldNames)
fieldName = fieldNames{i};
value = tableParams.(fieldName);
% Construct the full column name in the format "tableName.fieldName"
fullName = sprintf('%s.%s', tableName, fieldName);
% Handle different types of values for SQL query construction
if isempty(value)
% Skip this parameter if it is empty (include all values)
continue;
elseif isnumeric(value) && isnan(value)
% If value is NaN, use IS NULL in SQL
filterClause = sprintf('%s IS NULL', fullName);
elseif isnumeric(value) && ~isEnumeration(value)
filterClause = sprintf('%s = %f', fullName, value);
elseif islogical(value) || (isnumeric(value) && ismember(value, [0, 1])) && ~isEnumeration(value)
filterClause = sprintf('%s = %d', fullName, value);
elseif ischar(value) || isstring(value)
filterClause = sprintf('%s = ''%s''', fullName, char(value)); %nicht nach string suchen sondern nach chararray -> 'bla' statt "bla"
elseif isEnumeration(value)
filterClause = sprintf('%s = ''%s''', fullName, value);
else
error('Unsupported data type for field "%s".', fullName);
end
% Add the constructed filter clause to the list
filterClauses = [filterClauses, filterClause, ' AND '];
end
end
% Remove trailing ' AND ' from the filter clauses if any filters were added
if ~isempty(filterClauses)
filterClauses = filterClauses(1:end-5);
end
% Construct the final SQL query
if isempty(filterClauses)
query = [selectClause, 'FROM Runs ' ...
'LEFT JOIN Configurations ON Runs.run_id = Configurations.run_id ' ...
'LEFT JOIN Measurements ON Runs.run_id = Measurements.run_id ' ...
'LEFT JOIN BERs ON Runs.run_id = BERs.run_id'];
else
query = [baseQuery, filterClauses];
end
end
function selectedFields = promptSelectFields(obj)
% promptSelectFields Prompts the user to select fields from multiple tables to include in the SELECT statement using settingsdlg.
% Get all possible fields from all tables (excluding sqlite_sequence)
tableNames = fieldnames(obj.tables);
tableNames = setdiff(tableNames, {'sqlite_sequence'}); % Remove sqlite_sequence
% Prepare the inputs for settingsdlg
promptSettings = {};
allFieldsFullName = {};
convertedFieldNames = {};
for i = 1:numel(tableNames)
tableFields = fieldnames(obj.tables.(tableNames{i}));
for j = 1:numel(tableFields)
fieldName = tableFields{j};
fullName = sprintf('%s.%s', tableNames{i}, fieldName);
convertedName = strrep(fullName, '.', '_'); % Replace '.' with '_'
allFieldsFullName{end + 1} = fullName; % Add full name to the list
convertedFieldNames{end + 1} = convertedName; % Store the converted name
% Add the field name and checkbox setting to the prompt
promptSettings{end + 1} = {sprintf('Include %s', fullName), convertedName};
promptSettings{end + 1} = false; % Default: not selected
end
end
% Create the settings dialog
[settings, button] = settingsdlg(...
'title', 'Select Fields for the SQL Query', ...
'description', 'Check the boxes for the fields you want to include in the SELECT statement.', ...
promptSettings{:} ...
);
% If the user cancels, default to selecting all fields
if strcmp(button, 'cancel')
selectedFields = allFieldsFullName;
return;
end
% Parse user input into selectedFields
selectedFields = {};
for i = 1:numel(allFieldsFullName)
convertedName = convertedFieldNames{i};
if isfield(settings, convertedName) && settings.(convertedName) % Add to selectedFields if the checkbox was selected
selectedFields{end + 1} = allFieldsFullName{i}; %#ok<AGROW>
end
end
% If no fields are selected, default to selecting all fields
if isempty(selectedFields)
selectedFields = allFieldsFullName;
end
end
function filterParams = promptFilterParameters(obj)
% promptFilterParameters Prompts the user to enter filter parameters using the settingsdlg framework.
% Get all possible parameters from all tables (excluding sqlite_sequence)
tableNames_ = fieldnames(obj.tables);
tableNames_ = setdiff(tableNames_, {'sqlite_sequence'}); % Remove sqlite_sequence
% Prepare the inputs for settingsdlg with sections and separators
promptSettings = {};
allFieldsFullName = {};
convertedFieldNames = {};
for i = 1:numel(tableNames_)
% Add a separator for each table section
promptSettings{end + 1} = 'separator';
promptSettings{end + 1} = tableNames_{i};
% Get all fields from the current table
tableFields = fieldnames(obj.tables.(tableNames_{i}));
% Prepare each field to be added to the dialog
for j = 1:numel(tableFields)
fieldName = tableFields{j};
fullName = sprintf('%s.%s', tableNames_{i}, fieldName);
convertedName = strrep(fullName, '.', '_'); % Replace '.' with '_'
% Skip fields that do not have distinct values stored
if ~isfield(obj.distinctValues.(tableNames_{i}), fieldName)
continue;
end
% Get the distinct values for the field
distinctValues_ = obj.distinctValues.(tableNames_{i}).(fieldName);
% Prepare distinct values for dropdown
if isempty(distinctValues_)
% If there are no distinct values, use only an "All" entry
distinctValues_ = {'All'};
else
% Ensure distinctValues is a cell array of strings
if isnumeric(distinctValues_)
distinctValues_ = arrayfun(@(x) num2str(x), distinctValues_, 'UniformOutput', false);
elseif isstring(distinctValues_)
distinctValues_ = cellstr(distinctValues_);
elseif iscell(distinctValues_) && ~iscellstr(distinctValues_)
distinctValues_ = cellfun(@num2str, distinctValues_, 'UniformOutput', false);
end
% Add an "All" option at the beginning of the distinct values list
distinctValues_ = [{'All'}; distinctValues_];
end
allFieldsFullName{end + 1} = fullName; % Add full name to the list
convertedFieldNames{end + 1} = convertedName; % Store the converted name
% Add the field name and value setting to the prompt
promptSettings{end + 1} = {sprintf('%s', fullName), convertedName};
promptSettings{end + 1} = distinctValues_; % Add distinct values as dropdown options
end
end
% Create the settings dialog
[settings, button] = settingsdlg(...
'title', 'Input Parameters for Filtering', ...
'description', 'Enter the values for each field to filter. Select "All" to include all values.', ...
promptSettings{:} ...
);
% If the user cancels, return an empty struct
if strcmp(button, 'cancel')
filterParams = struct();
return;
end
% Parse user input into filterParams structure
filterParams = struct();
for i = 1:numel(allFieldsFullName)
value = settings.(convertedFieldNames{i});
% Split full name to get table and field names
fieldParts = strsplit(allFieldsFullName{i}, '.');
tableName = fieldParts{1};
fieldName = fieldParts{2};
% If the table does not exist in the filterParams struct, create it
if ~isfield(filterParams, tableName)
filterParams.(tableName) = struct();
end
% Assign values to the respective fields under each table
if strcmp(value, 'All')
filterParams.(tableName).(fieldName) = []; % Set to empty to include all values
elseif isnumeric(value) && isnan(value)
filterParams.(tableName).(fieldName) = NaN; % Use NaN to handle as NULL
else
filterParams.(tableName).(fieldName) = value; % Use the entered value
end
end
end
end
end