classdef DBHandler < handle % DBHANDLER Class to handle database queries % This class provides methods to interact with an SQLite database, including % inserting data, retrieving table names, and appending new rows. properties conn % Database connection object pathToDB % Path to the SQLite database tableNames % Cell array containing names of all tables in the database tables = struct(); % Structure containing MATLAB tables for each database table distinctValues end methods function obj = DBHandler(options) % DBHANDLER Constructor for the DBHandler class % Initializes the database connection and retrieves table and field names. % % Usage: % obj = DBHandler('pathToDB', 'path/to/database.db'); arguments options.pathToDB = ""; % Default value for pathToDB if not provided end % Assign values to class properties based on input arguments fn = fieldnames(options); for n = 1:numel(fn) try obj.(fn{n}) = options.(fn{n}); end end % Establish a connection to the SQLite database try obj.conn = sqlite(obj.pathToDB); catch e error('Failed to connect to the database: %s', e.message); end if obj.dbIsHealthy obj.refresh(); else error('DB seems to be corrupt') end end function obj = refresh(obj) % Get table names and the first rows of each table to understand the structure obj.getTableNames(); obj.getTables(); obj.getDistinctValues(); end function obj = getTableNames(obj) % Get all table names from the database try result = fetch(obj.conn, 'SELECT name FROM sqlite_master WHERE type="table"'); obj.tableNames = result.name; catch e error('Failed to retrieve table names: %s', e.message); end end function obj = getTables(obj) % Get a preview (first row) of each table to understand its structure for i = 1:numel(obj.tableNames) try tableName = obj.tableNames{i}; results = fetch(obj.conn, sprintf('SELECT * FROM %s WHERE 1 = 2', tableName)); % Matlab cant handle if there is a NULL in a returned % datarow... therefore do not return a row using the % above condition which is never true % results = sqlread(obj.conn, tableName, MaxRows=1); for l = 1:numel(results.Properties.VariableNames) varName = results.Properties.VariableNames{l}; obj.tables.(tableName).(varName) = []; % Store the preview as a reference end catch e warning('Failed to read the table %s: %s', tableName, e.message); end end end function obj = getDistinctValues(obj) % getDistinctValues Retrieves distinct values for each relevant field in all tables % excluding fields ending with "_id". Stores distinct values in the 'distinctValues' % property. % Initialize a structure to store distinct values for each table obj.distinctValues = struct(); % Iterate over each table in obj.tables tableNames_ = fieldnames(obj.tables); for i = 1:numel(tableNames_) tableName = tableNames_{i}; % Initialize a sub-struct to store distinct values for each field in the table obj.distinctValues.(tableName) = struct(); % Get all fields of the current table fieldNames = fieldnames(obj.tables.(tableName)); % Iterate over each field for j = 1:numel(fieldNames) fieldName = fieldNames{j}; % % Skip fields ending with '_id' as they don't contain useful distinct values % if endsWith(fieldName, '_id') % continue; % end % Construct SQL to get distinct values for the current field query = sprintf('SELECT DISTINCT %s FROM %s', fieldName, tableName); % Execute query and fetch distinct values try result = fetch(obj.conn, query); % Store the distinct values in the structure if ~isempty(result) distinctValues = table2array(result); else distinctValues = []; end obj.distinctValues.(tableName).(fieldName) = distinctValues; catch e % warning('Failed to retrieve distinct values for %s.%s: %s', tableName, fieldName, e.message); obj.distinctValues.(tableName).(fieldName) = []; end end end end function healthyDB = dbIsHealthy(obj) healthyDB = false; num_runs = obj.fetch('SELECT COUNT(*) AS total_runs FROM Runs'); num_configs = obj.fetch('SELECT COUNT(*) AS total_configurations FROM Configurations'); num_meas = obj.fetch('SELECT COUNT(*) AS total_measurements FROM Measurements'); assert((num_runs{1,1}==num_configs{1,1})&&(num_configs{1,1}==num_meas{1,1}),'Different num of entries per table'); %Check for any duplicate paths duplictae_raw = obj.fetch("SELECT COALESCE(Runs.rx_raw_path,'NaN') AS rx_raw_path, COUNT(*) AS occurrences FROM Runs GROUP BY rx_raw_path HAVING COUNT(*) > 1"); duplictae_sync = obj.fetch("SELECT COALESCE(Runs.rx_sync_path,'NaN') AS rx_sync_path, COUNT(*) AS occurrences FROM Runs GROUP BY rx_sync_path HAVING COUNT(*) > 1"); if size(duplictae_raw,2) > 1 for i = 1:size(duplictae_raw,2)-1 fprintf('Raw Rx Paths: Found %d duplictaes of %s \n',duplictae_raw.occurrences(i),duplictae_raw.rx_raw_path(i)); end end healthyDB = true; end function lastID = appendToTable(obj, tableName, newRow) % appendToTable Appends a new row to the specified table % % Usage: % appendToTable(tableName, newRow) % % Inputs: % tableName: The name of the table to append data to. % newRow: A MATLAB table or struct containing the new row to be appended. % Check if the table exists in the fetched tables if ~isfield(obj.tables, tableName) error('Table %s does not exist in the database or has not been fetched.', tableName); end % Convert newRow to a table if it is a struct if isstruct(newRow) fields = fieldnames(newRow); emptyFields = structfun(@isempty,newRow); if sum(emptyFields)>0 newRow.(fields{emptyFields==1}) = NaN; disp(['In Table: ',tableName,': ',fields{emptyFields==1},' was empty, is now NaN ',newRow.(fields{emptyFields==1})]) end newRow = struct2table(newRow); end % Ensure the new row matches the structure of the existing table existingTableStructure = obj.tables.(tableName); % Perform data type checks and conversions for colName = newRow.Properties.VariableNames % Extract the value and its intended column type value = newRow.(colName{1}); existingValue = existingTableStructure.(colName{1}); % If the value is a class object, convert it to JSON format if isobject(value) && ~isdatetime(value) && ~isa(value,"string") newRow.(colName{1}) = string(jsonencode(value)); % If the value is a character array, convert it to a string elseif ischar(value) newRow.(colName{1}) = string(value); end end % Append the new row to the database table try sqlwrite(obj.conn, tableName, newRow); % disp(['Successfully appended new row to the table ', tableName]); catch e error('Failed to append to the table %s: %s', tableName, e.message); end % Retrieve the measurement_id of the newly inserted row for linking other tables result = fetch(obj.conn, 'SELECT last_insert_rowid()'); lastID = result{1, 1}; % Access the value directly from the table end function exists = checkIfRunExists(obj, table2check, column2check, value2check) % checkIfRunExists Checks if a specific value exists in a specified column of a table % % Usage: % exists = checkIfRunExists(table2check, column2check, value2check) % % Inputs: % table2check: The name of the table to check for duplicates. % column2check: The name of the column to check within the specified table. % value2check: The value to check for in the specified column. % % Outputs: % exists: Boolean indicating whether the value already exists in the table. % Ensure the specified table and column exist in the database if ~isfield(obj.tables, table2check) error('Table %s does not exist in the database.', table2check); end % Ensure the specified column exists in the table structure if ~isfield(obj.tables.(table2check), column2check) error('Column %s does not exist in the table %s.', column2check, table2check); end % Construct the query to check for the value in the specified column query = sprintf('SELECT COUNT(*) FROM %s WHERE %s = "%s"', table2check, column2check, value2check); % Execute the query and pass the value2check to avoid SQL injection issues try result = fetch(obj.conn, query); count = result{1, 1}; % Extract the count from the result catch e error('Failed to execute the duplicate check query: %s', e.message); end % If count is greater than 0, then the value exists in the table exists = count > 0; if exists disp(['The value "', value2check, '" already exists in the column "', column2check, '" of the table "', table2check, '".']); else % disp(['The value "', value2check, '" does not exist in the column "', column2check, '" of the table "', table2check, '".']); end end function addBEREntry(obj, berValue, occurrence, runID, ffe, dfe, mlse, pf, eqType, ffe_order, dfe_order, len_tr, mu_ffe, mu_dfe, mu_dc, comment) % addBEREntry Adds a BER entry linked to an existing or new Equalizer entry. % Usage: % addBEREntry(runID, eq, ffe, dfe, mlse, pf, eqType, ffe_order, dfe_order, len_tr, mu_ffe, mu_dfe, mu_dc, berValue, comment) if isempty(pf) postfilter_taps = []; else postfilter_taps = pf.burg_coeff; end % Create equalizer data struct for searching and adding if necessary equalizerData = struct( ... 'ffe', jsonencode(ffe), ... 'dfe', jsonencode(dfe), ... 'mlse', jsonencode(mlse), ... 'pf', jsonencode(pf), ... 'eq_type', string(eqType), ... 'ffe_order', jsonencode(ffe_order), ... 'dfe_order', jsonencode(dfe_order), ... 'postfilter_taps',jsonencode(postfilter_taps),... 'len_tr', len_tr, ... 'mu_ffe', jsonencode(mu_ffe), ... 'mu_dfe', mu_dfe, ... 'mu_dc', mu_dc, ... 'comment', comment ... ); % Check if exact Equalizer and BER entries already exist in the DB ... selectedFields = {'Runs.run_id','BERs.ber_id','Equalizer.eq_id','BERs.ber',['BERs.occurrence' ... '']}; filterParams = obj.tables; filterParams.Equalizer = equalizerData; [dataTable,sql_query] = obj.queryDB(filterParams, selectedFields); % get or insert Equalizer if ~isempty(dataTable) % Equalizer entry already exists, use the existing eq_id cur_eq_id = dataTable.eq_id; else % Insert the new Equalizer entry cur_eq_id = obj.appendToTable('Equalizer', equalizerData); end % skip if already here or insert BER entry if ~isempty(dataTable) % A BER entry with the same eq_id, run_id, and occurrence already exists existingBERValue = dataTable.ber; % Compare the existing BER value with the new BER value if existingBERValue == berValue fprintf('The BER entry %.2e || -- eq_id: %d -- run_id: %d -- occurrence: %d already exists. \n',berValue, cur_eq_id, runID, occurrence); else fprintf('Already found BER for EQ: %.2e ~= %.2e || -- eq_id: %d -- run_id: %d -- occurrence: %d already exists.\n', berValue, existingBERValue, cur_eq_id, runID, occurrence); end else % No such BER entry exists, insert the new BER entry berData = struct( ... 'run_id', runID, ... 'eq_id', cur_eq_id, ... 'ber', berValue, ... 'occurrence', occurrence ... ); obj.appendToTable('BERs', berData); end end function answer = fetch(obj,query) answer = fetch(obj.conn,query); end function [result,query] = queryDB(obj, filterParams, selectedFields) % getPathsWithFlexibleFilter Retrieves values from Runs table with flexible filtering % and lets the user select which fields to include in the SELECT statement. % % Usage: % [rxRawPaths, filteredValues] = getPathsWithFlexibleFilter(filterParams) % % Inputs: % filterParams: A structure containing the parameters with their values. % If left empty, two popup windows will prompt the user for input. % % Outputs: % result: table with sql return % query: this was send to SQL DB arguments obj filterParams = []; selectedFields = []; end % Step 1: Prompt the user to input filter parameters if not provided if isempty(filterParams) filterParams = obj.promptFilterParameters(); end % Step 2: Prompt the user to select fields to include in the SELECT statement if isempty(selectedFields) selectedFields = obj.promptSelectFields(); else if iscell(selectedFields) elseif isstruct(selectedFields) end end % Step 3: Construct the SQL query based on the inputs query = obj.constructSQLQuery(filterParams, selectedFields); % Step 4: Execute the query and handle results result = obj.fetch(query); end function query = constructSQLQuery(obj, filterParams, selectedFields) % constructSQLQuery Constructs the SQL query based on filter parameters and selected fields. % Construct the SELECT clause dynamically based on user selection selectClause = 'SELECT DISTINCT '; for i = 1:numel(selectedFields) fieldParts = strsplit(selectedFields{i}, '.'); tableName = fieldParts{1}; fieldName = fieldParts{2}; if isnumeric(obj.tables.(tableName).(fieldName)) selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', ''NaN'') AS ', fieldName]; else selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', '''') AS ', fieldName]; end if i < numel(selectedFields) selectClause = [selectClause, ', ']; else selectClause = [selectClause, ' ']; end end % Construct the FROM and WHERE clause baseQuery = [selectClause, 'FROM Runs ' ... 'LEFT JOIN Configurations ON Runs.run_id = Configurations.run_id ' ... 'LEFT JOIN Measurements ON Runs.run_id = Measurements.run_id ' ... 'LEFT JOIN BERs ON Runs.run_id = BERs.run_id ' ... 'LEFT JOIN Equalizer ON BERs.eq_id = Equalizer.eq_id ' ... 'WHERE ']; % Loop through each table in filterParams filterClauses = []; tableNames_ = fieldnames(filterParams); for t = 1:numel(tableNames_) tableName = tableNames_{t}; tableParams = filterParams.(tableName); % Loop through each parameter in the table fieldNames = fieldnames(tableParams); for i = 1:numel(fieldNames) fieldName = fieldNames{i}; value = tableParams.(fieldName); % Construct the full column name in the format "tableName.fieldName" fullName = sprintf('%s.%s', tableName, fieldName); % Handle different types of values for SQL query construction if isempty(value) % Skip this parameter if it is empty (include all values) continue; elseif isnumeric(value) && isnan(value) % If value is NaN, use IS NULL in SQL filterClause = sprintf('%s IS NULL', fullName); elseif isnumeric(value) && ~isEnumeration(value) filterClause = sprintf('%s = %f', fullName, value); elseif islogical(value) || (isnumeric(value) && ismember(value, [0, 1])) && ~isEnumeration(value) filterClause = sprintf('%s = %d', fullName, value); elseif ischar(value) || isstring(value) filterClause = sprintf('%s = ''%s''', fullName, char(value)); %nicht nach string suchen sondern nach chararray -> 'bla' statt "bla" elseif isEnumeration(value) filterClause = sprintf('%s = ''%s''', fullName, value); else error('Unsupported data type for field "%s".', fullName); end % Add the constructed filter clause to the list filterClauses = [filterClauses, filterClause, ' AND ']; end end % Remove trailing ' AND ' from the filter clauses if any filters were added if ~isempty(filterClauses) filterClauses = filterClauses(1:end-5); end % Construct the final SQL query if isempty(filterClauses) query = [selectClause, 'FROM Runs ' ... 'LEFT JOIN Configurations ON Runs.run_id = Configurations.run_id ' ... 'LEFT JOIN Measurements ON Runs.run_id = Measurements.run_id ' ... 'LEFT JOIN BERs ON Runs.run_id = BERs.run_id']; else query = [baseQuery, filterClauses]; end end function selectedFields = promptSelectFields(obj) % promptSelectFields Prompts the user to select fields from multiple tables to include in the SELECT statement using settingsdlg. % Get all possible fields from all tables (excluding sqlite_sequence) tableNames = fieldnames(obj.tables); tableNames = setdiff(tableNames, {'sqlite_sequence'}); % Remove sqlite_sequence % Prepare the inputs for settingsdlg promptSettings = {}; allFieldsFullName = {}; convertedFieldNames = {}; for i = 1:numel(tableNames) tableFields = fieldnames(obj.tables.(tableNames{i})); for j = 1:numel(tableFields) fieldName = tableFields{j}; fullName = sprintf('%s.%s', tableNames{i}, fieldName); convertedName = strrep(fullName, '.', '_'); % Replace '.' with '_' allFieldsFullName{end + 1} = fullName; % Add full name to the list convertedFieldNames{end + 1} = convertedName; % Store the converted name % Add the field name and checkbox setting to the prompt promptSettings{end + 1} = {sprintf('Include %s', fullName), convertedName}; promptSettings{end + 1} = false; % Default: not selected end end % Create the settings dialog [settings, button] = settingsdlg(... 'title', 'Select Fields for the SQL Query', ... 'description', 'Check the boxes for the fields you want to include in the SELECT statement.', ... promptSettings{:} ... ); % If the user cancels, default to selecting all fields if strcmp(button, 'cancel') selectedFields = allFieldsFullName; return; end % Parse user input into selectedFields selectedFields = {}; for i = 1:numel(allFieldsFullName) convertedName = convertedFieldNames{i}; if isfield(settings, convertedName) && settings.(convertedName) % Add to selectedFields if the checkbox was selected selectedFields{end + 1} = allFieldsFullName{i}; %#ok end end % If no fields are selected, default to selecting all fields if isempty(selectedFields) selectedFields = allFieldsFullName; end end function filterParams = promptFilterParameters(obj) % promptFilterParameters Prompts the user to enter filter parameters using the settingsdlg framework. % Get all possible parameters from all tables (excluding sqlite_sequence) tableNames_ = fieldnames(obj.tables); tableNames_ = setdiff(tableNames_, {'sqlite_sequence'}); % Remove sqlite_sequence % Prepare the inputs for settingsdlg with sections and separators promptSettings = {}; allFieldsFullName = {}; convertedFieldNames = {}; for i = 1:numel(tableNames_) % Add a separator for each table section promptSettings{end + 1} = 'separator'; promptSettings{end + 1} = tableNames_{i}; % Get all fields from the current table tableFields = fieldnames(obj.tables.(tableNames_{i})); % Prepare each field to be added to the dialog for j = 1:numel(tableFields) fieldName = tableFields{j}; fullName = sprintf('%s.%s', tableNames_{i}, fieldName); convertedName = strrep(fullName, '.', '_'); % Replace '.' with '_' % Skip fields that do not have distinct values stored if ~isfield(obj.distinctValues.(tableNames_{i}), fieldName) continue; end % Get the distinct values for the field distinctValues_ = obj.distinctValues.(tableNames_{i}).(fieldName); % Prepare distinct values for dropdown if isempty(distinctValues_) % If there are no distinct values, use only an "All" entry distinctValues_ = {'All'}; else % Ensure distinctValues is a cell array of strings if isnumeric(distinctValues_) distinctValues_ = arrayfun(@(x) num2str(x), distinctValues_, 'UniformOutput', false); elseif isstring(distinctValues_) distinctValues_ = cellstr(distinctValues_); elseif iscell(distinctValues_) && ~iscellstr(distinctValues_) distinctValues_ = cellfun(@num2str, distinctValues_, 'UniformOutput', false); end % Add an "All" option at the beginning of the distinct values list distinctValues_ = [{'All'}; distinctValues_]; end allFieldsFullName{end + 1} = fullName; % Add full name to the list convertedFieldNames{end + 1} = convertedName; % Store the converted name % Add the field name and value setting to the prompt promptSettings{end + 1} = {sprintf('%s', fullName), convertedName}; promptSettings{end + 1} = distinctValues_; % Add distinct values as dropdown options end end % Create the settings dialog [settings, button] = settingsdlg(... 'title', 'Input Parameters for Filtering', ... 'description', 'Enter the values for each field to filter. Select "All" to include all values.', ... promptSettings{:} ... ); % If the user cancels, return an empty struct if strcmp(button, 'cancel') filterParams = struct(); return; end % Parse user input into filterParams structure filterParams = struct(); for i = 1:numel(allFieldsFullName) value = settings.(convertedFieldNames{i}); % Split full name to get table and field names fieldParts = strsplit(allFieldsFullName{i}, '.'); tableName = fieldParts{1}; fieldName = fieldParts{2}; % If the table does not exist in the filterParams struct, create it if ~isfield(filterParams, tableName) filterParams.(tableName) = struct(); end % Assign values to the respective fields under each table if strcmp(value, 'All') filterParams.(tableName).(fieldName) = []; % Set to empty to include all values elseif isnumeric(value) && isnan(value) filterParams.(tableName).(fieldName) = NaN; % Use NaN to handle as NULL else filterParams.(tableName).(fieldName) = value; % Use the entered value end end end end end