silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Many changes in DBHandler

Browse Source 
new general processing structure
just before splitting off the projects folder from this repo
This commit is contained in:
Silas Oettinghaus
2025-05-13 10:24:09 +02:00
parent 727c3d9364
commit 9ce23c4a10
38 changed files with 2440 additions and 1103 deletions
Download Patch File Download Diff File Expand all files Collapse all files

406
Classes/DataBaseHandler/DBHandler.m
View File

@@ -83,7 +83,7 @@ classdef DBHandler < handle
result = fetch(obj.conn, 'SELECT name FROM sqlite_master WHERE type="table"');
obj.tableNames = result.name;
end
catch e
error('Failed to retrieve table names: %s', e.message);
end
@@ -169,14 +169,6 @@ classdef DBHandler < handle
function healthyDB = dbIsHealthy(obj)
healthyDB = false;
num_runs = obj.fetch('SELECT COUNT(*) AS total_runs FROM Runs');
num_configs = obj.fetch('SELECT COUNT(*) AS total_configurations FROM Configurations');
num_meas = obj.fetch('SELECT COUNT(*) AS total_measurements FROM Measurements');
assert((num_runs{1,1}==num_configs{1,1})&&(num_configs{1,1}==num_meas{1,1}),'Different num of entries per table');
%Check for any duplicate paths
duplictae_raw = obj.fetch("SELECT COALESCE(Runs.rx_raw_path,'NaN') AS rx_raw_path, COUNT(*) AS occurrences FROM Runs GROUP BY rx_raw_path HAVING COUNT(*) > 1");
duplictae_sync = obj.fetch("SELECT COALESCE(Runs.rx_sync_path,'NaN') AS rx_sync_path, COUNT(*) AS occurrences FROM Runs GROUP BY rx_sync_path HAVING COUNT(*) > 1");
@@ -190,7 +182,6 @@ classdef DBHandler < handle
end
function lastID = appendToTable(obj, tableName, newRow)
% appendToTable Appends a new row to the specified table
%
@@ -206,48 +197,52 @@ classdef DBHandler < handle
error('Table %s does not exist in the database or has not been fetched.', tableName);
end
% Convert newRow to a table if it is a struct
% Handle struct preprocessing before table conversion
if isstruct(newRow)
fields = fieldnames(newRow);
emptyFields = structfun(@isempty,newRow);
% Handle empty fields
emptyFields = structfun(@isempty, newRow);
if sum(emptyFields) > 0
emptyFieldNames = fields(emptyFields); % use () to get a cell array
emptyFieldNames = fields(emptyFields);
for idx = 1:numel(emptyFieldNames)
newRow.(emptyFieldNames{idx}) = NaN;
end
end
% Convert arrays to JSON strings
for i = 1:length(fields)
fieldValue = newRow.(fields{i});
if isnumeric(fieldValue) && length(fieldValue) > 1
newRow.(fields{i}) = jsonencode(fieldValue);
end
end
% Convert to table
newRow = struct2table(newRow);
end
% Ensure the new row matches the structure of the existing table
existingTableStructure = obj.tables.(tableName);
% Perform data type checks and conversions
% Perform remaining data type checks and conversions
for colName = newRow.Properties.VariableNames
% Extract the value and its intended column type
value = newRow.(colName{1});
existingValue = existingTableStructure.(colName{1});
% If the value is a class object, convert it to JSON format
if isobject(value) && ~isdatetime(value) && ~isa(value,"string")
if iscell(value) && ~isempty(value)
if ischar(value{1}) || isstring(value{1})
newRow.(colName{1}) = value{1};
end
elseif isobject(value) && ~isdatetime(value) && ~isa(value, "string")
newRow.(colName{1}) = string(jsonencode(value));
% If the value is a character array, convert it to a string
elseif ischar(value)
newRow.(colName{1}) = string(value);
elseif isdatetime(value)
% newRow.(colName{1}) = string(value);
end
end
% Parameters for retry logic
maxRetries = 50;
basePause = 0.05; % seconds
attempt = 0;
success = false;
@@ -257,36 +252,40 @@ classdef DBHandler < handle
newRow_ = obj.convertTableToCellStrings(newRow);
end
sqlwrite(obj.conn, tableName, newRow);
success = true; % Write successful
success = true;
catch e
if contains(e.message, 'database is locked') || contains(e.message, 'cannot rollback transaction')
attempt = attempt + 1;
pauseTime = basePause * (1 + rand()); % Add random jitter to reduce collision chance
fprintf('Database locked. Retry %d/%d after %.3f seconds...\n', attempt, maxRetries, pauseTime);
pauseTime = basePause * (1 + rand());
fprintf('Database locked. Retry %d/%d after %.3f seconds...\n', ...
attempt, maxRetries, pauseTime);
pause(pauseTime);
else
fprintf('Error details:\n');
fprintf('Column values:\n');
disp(newRow);
error('Failed to append to the table %s: %s', tableName, e.message);
end
end
end
if ~success
error('Failed to append to table %s after %d retries due to database lock.', tableName, maxRetries);
error('Failed to append to table %s after %d retries due to database lock.', ...
tableName, maxRetries);
end
% Retrieve the measurement_id of the newly inserted row for linking other tables
% Retrieve the ID of the newly inserted row
if obj.type == "mysql"
queue = "SELECT LAST_INSERT_ID()";
query = "SELECT LAST_INSERT_ID()";
elseif obj.type == "sqlite"
queue = "SELECT last_insert_rowid()";
query = "SELECT last_insert_rowid()";
end
result = fetch(obj.conn, queue);
lastID = result{1, 1}; % Access the value directly from the table
result = fetch(obj.conn, query);
lastID = result{1, 1};
end
function exists = checkIfRunExists(obj, table2check, column2check, value2check)
function [exists, count] = checkIfRunExists(obj, table2check, column2check, value2check)
% checkIfRunExists Checks if a specific value exists in a specified column of a table
%
% Usage:
@@ -311,7 +310,11 @@ classdef DBHandler < handle
end
% Construct the query to check for the value in the specified column
query = sprintf('SELECT COUNT(*) FROM %s WHERE %s = "%s"', table2check, column2check, value2check);
if isnumeric(value2check)
query = sprintf('SELECT COUNT(*) FROM %s WHERE %s = %d', table2check, column2check, value2check);
else
query = sprintf('SELECT COUNT(*) FROM %s WHERE %s = "%s"', table2check, column2check, value2check);
end
% Execute the query and pass the value2check to avoid SQL injection issues
try
@@ -325,12 +328,21 @@ classdef DBHandler < handle
exists = count > 0;
if exists
disp(['The value "', value2check, '" already exists in the column "', column2check, '" of the table "', table2check, '".']);
% disp(['The value "', num2str(value2check), '" already exists in the column "', column2check, '" of the table "', table2check, '".']);
else
% disp(['The value "', value2check, '" does not exist in the column "', column2check, '" of the table "', table2check, '".']);
end
end
function hashStr = calcHash(~,object)
jsonStr = jsonencode(object);
md = java.security.MessageDigest.getInstance('MD5');
md.update(uint8(jsonStr));
hashBytes = typecast(md.digest, 'uint8');
hashStr = lower(dec2hex(hashBytes)');
hashStr = lower(strtrim(hashStr(:)')); % Convert to a lowercase string
end
function resultID = addProcessingResult(obj, run_id, resultData, eqParamsData)
% addProcessingResult Adds a processing result and links it to an EqualizerParameters entry.
@@ -348,18 +360,19 @@ classdef DBHandler < handle
% resultID: The result_id of the newly inserted ProcessingResults entry.
% 1. Compute hash for equalizer parameters
jsonStr = jsonencode(eqParamsData);
md = java.security.MessageDigest.getInstance('MD5');
md.update(uint8(jsonStr));
hashBytes = typecast(md.digest, 'uint8');
hashStr = lower(dec2hex(hashBytes)');
hashStr = lower(strtrim(hashStr(:)')); % Convert to a lowercase string
% jsonStr = jsonencode(eqParamsData);
% md = java.security.MessageDigest.getInstance('MD5');
% md.update(uint8(jsonStr));
% hashBytes = typecast(md.digest, 'uint8');
% hashStr = lower(dec2hex(hashBytes)');
% hashStr = lower(strtrim(hashStr(:)')); % Convert to a lowercase string
hashStr = obj.calcHash(eqParamsData);
% Add hash to equalizer parameters
eqParamsData.config_hash = hashStr;
eqParamsData.hash = hashStr;
% 2. Check if an equalizer configuration with the same hash exists
queryStr = sprintf('SELECT eq_id FROM EqualizerParameters WHERE config_hash = ''%s''', eqParamsData.config_hash);
queryStr = sprintf('SELECT eq_id FROM Equalizer WHERE hash = ''%s''', eqParamsData.hash);
existingEntry = obj.fetch(queryStr);
if ~isempty(existingEntry)
@@ -367,27 +380,30 @@ classdef DBHandler < handle
eq_id = existingEntry{1,1};
else
% Insert the new equalizer configuration and get its eq_id
eq_id = obj.appendToTable('EqualizerParameters', eqParamsData);
eqParamsData = eqParamsData.toStruct;
eq_id = obj.appendToTable('Equalizer', eqParamsData);
end
% 3. Add the equalizer configuration reference and run_id to resultData
resultData.eqParam_id = eq_id;
resultData = resultData.toStruct;
resultData.eq_id = eq_id;
resultData.run_id = run_id;
% 4. Compute hash for the processing result
tempResultData = rmfield(resultData, 'date_of_processing');
resultJsonStr = jsonencode(tempResultData);
md2 = java.security.MessageDigest.getInstance('MD5'); % Create a new MD5 instance
md2.update(uint8(resultJsonStr));
resultHashBytes = typecast(md2.digest, 'uint8');
resultHashStr = lower(dec2hex(resultHashBytes)');
resultHashStr = lower(strtrim(resultHashStr(:)')); % Convert to a lowercase string
% resultJsonStr = jsonencode(tempResultData);
% md2 = java.security.MessageDigest.getInstance('MD5'); % Create a new MD5 instance
% md2.update(uint8(resultJsonStr));
% resultHashBytes = typecast(md2.digest, 'uint8');
% resultHashStr = lower(dec2hex(resultHashBytes)');
% resultHashStr = lower(strtrim(resultHashStr(:)')); % Convert to a lowercase string
resultHashStr = obj.calcHash(tempResultData);
% Add the result hash to resultData
resultData.result_hash = resultHashStr;
resultData.hash = resultHashStr;
% 5. Check if an identical processing result already exists
queryStr2 = sprintf('SELECT result_id FROM Results WHERE result_hash = ''%s''', resultData.result_hash);
queryStr2 = sprintf('SELECT result_id FROM Results WHERE hash = ''%s''', resultData.hash);
existingResult = obj.fetch(queryStr2);
if ~isempty(existingResult)
@@ -397,7 +413,66 @@ classdef DBHandler < handle
return;
end
% 6. Insert the processing result
% 6. check if obj.tables.Results matches Metricstruct
% Fields to exclude from comparison
excludeFields = {'result_id', 'run_id', 'eq_id'};
% 7. Chack for new fields in Metric struct and append to
% database if necessary
ms=Metricstruct;
metricFields = setdiff(fieldnames(ms), excludeFields);
tableFields = setdiff(fieldnames(obj.tables.Results), excludeFields);
% Check matches and find missing fields
matches = all(ismember(metricFields, tableFields));
if ~matches
missingFields = setdiff(metricFields, tableFields);
% If there are missing fields, add them to the SQL table
if ~isempty(missingFields)
for i = 1:length(missingFields)
fieldName = missingFields{i};
% Determine SQL data type based on MATLAB class
fieldValue = ms.(fieldName);
if isnumeric(fieldValue)
if isinteger(fieldValue)
sqlType = 'INTEGER';
else
sqlType = 'REAL';
end
elseif ischar(fieldValue) || isstring(fieldValue)
sqlType = 'TEXT';
elseif isdatetime(fieldValue)
sqlType = 'DATETIME';
elseif iscell(fieldValue) || isstruct(fieldValue) || islogical(fieldValue)
sqlType = 'TEXT'; % Store as JSON
else
sqlType = 'TEXT'; % Default to TEXT for unknown types
end
% Create ALTER TABLE query
queryStr = sprintf('ALTER TABLE Results ADD COLUMN %s %s', fieldName, sqlType);
try
% Execute the query
obj.fetch(queryStr);
fprintf('Added field "%s" of type %s to Results table\n', fieldName, sqlType);
catch ME
fprintf('Error adding field "%s": %s\n', fieldName, ME.message);
end
end
else
fprintf('No missing fields to add.\n');
end
end
% 8. Insert the processing result
resultID = obj.appendToTable('Results', resultData);
end
@@ -486,8 +561,6 @@ classdef DBHandler < handle
end
function executeSQL(obj, query)
% This method executes an SQL statement using MATLAB's execute function.
execute(obj.conn, query);
@@ -518,22 +591,6 @@ classdef DBHandler < handle
selectedFields = [];
end
% Step 1: Prompt the user to input filter parameters if not provided
if isempty(filterParams)
filterParams = obj.promptFilterParameters();
end
% Step 2: Prompt the user to select fields to include in the SELECT statement
if isempty(selectedFields)
selectedFields = obj.promptSelectFields();
else
if iscell(selectedFields)
elseif isstruct(selectedFields)
end
end
% Step 3: Construct the SQL query based on the inputs
query = obj.constructSQLQuery(filterParams, selectedFields);
@@ -619,16 +676,20 @@ classdef DBHandler < handle
end
end
function query = constructSQLQuery(obj, filterParams, selectedFields)
% constructSQLQuery Constructs the SQL query based on filter parameters and selected fields.
%
% If selectedFields is provided as a struct, it is converted to a cell array.
% The conversion takes the field names and creates entries like:
% {'selectedFields.fieldName'} for each field.
% Input check for selectedFields: if it's a struct, convert it to a cell array.
if isstruct(selectedFields)
arguments
obj
filterParams
selectedFields
end
% Step 1: Handle selectedFields conversion
if isempty(selectedFields) || (ischar(selectedFields) && strcmpi(selectedFields, 'all'))
selectedFields = obj.getTableFieldNames('Runs'); % Default to Runs table
elseif isstruct(selectedFields)
% Convert struct to cell array of 'Table.field' format
newFields = {};
tableNames = fieldnames(selectedFields);
for t = 1:numel(tableNames)
@@ -643,39 +704,10 @@ classdef DBHandler < handle
selectedFields = newFields;
end
% Construct the SELECT clause dynamically based on user selection.
% (Assuming that when provided as a cell array, each entry is of the form
% 'TableName.fieldName' or, in our conversion case, 'selectedFields.fieldName'.)
selectClause = 'SELECT DISTINCT ';
for i = 1:numel(selectedFields)
fieldParts = strsplit(selectedFields{i}, '.');
% If the field comes from the struct conversion, its first part is 'selectedFields'
% and the actual field name is in the second part.
if strcmp(fieldParts{1}, 'selectedFields')
tableName = fieldParts{1}; % not used for type checking below
fieldName = fieldParts{2};
else
tableName = fieldParts{1};
fieldName = fieldParts{2};
end
% Step 2: Generate COALESCE string for SELECT clause
selectClause = obj.generateCoalesceString(selectedFields);
% Decide on COALESCE depending on the field type.
% If the table is known in obj.tables and the field is numeric, use 'NaN'.
if isfield(obj.tables, tableName) && isfield(obj.tables.(tableName), fieldName) && isnumeric(obj.tables.(tableName).(fieldName))
selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', ''NaN'') AS ', fieldName];
else
selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', '''') AS ', fieldName];
end
if i < numel(selectedFields)
selectClause = [selectClause, ', '];
else
selectClause = [selectClause, ' '];
end
end
% --- Adaptive FROM Clause ---
% Step 3: Generate FROM clause with joins
mainTable = 'Runs';
fromClause = ['FROM ', mainTable, ' '];
@@ -693,58 +725,90 @@ classdef DBHandler < handle
if isfield(obj.tables.(tableName), 'run_id')
% Direct join to Runs
normalJoins = [normalJoins, 'LEFT JOIN ', tableName, ' ON ', mainTable, '.run_id = ', tableName, '.run_id '];
normalJoins = [normalJoins, 'LEFT JOIN ', tableName, ...
' ON ', mainTable, '.run_id = ', tableName, '.run_id '];
elseif isfield(obj.tables.(tableName), 'eq_id')
% Equalizer depends on Results, collect this join separately
equalizerJoin = ['LEFT JOIN ', tableName, ' ON Results.eqParam_id = ', tableName, '.eq_id '];
equalizerJoin = ['LEFT JOIN ', tableName, ...
' ON Results.eq_id = ', tableName, '.eq_id '];
end
end
% Combine joins: normal joins first, Equalizer last
fromClause = [fromClause, normalJoins, equalizerJoin];
% Step 4: Generate WHERE clause if filter parameters exist
if ~isempty(filterParams)
whereClause = obj.generateWhereClause(filterParams);
if ~isempty(whereClause)
query = [selectClause, ' ', fromClause, 'WHERE ', whereClause];
else
query = [selectClause, ' ', fromClause];
end
else
query = [selectClause, ' ', fromClause];
end
end
% --- WHERE Clause Construction ---
baseQuery = [selectClause, ' ', fromClause, 'WHERE '];
function whereClause = generateWhereClause(obj, filterParams)
filterClauses = [];
tableNames_ = fieldnames(filterParams);
for t = 1:numel(tableNames_)
tableName = tableNames_{t};
% If filterParams is a DbFilterParameter object, get its internal structure
if isa(filterParams, 'QueryFilter')
filterParams = filterParams.toStruct();
end
% Now proceed with the structure
tableNames = fieldnames(filterParams);
for t = 1:numel(tableNames)
tableName = tableNames{t};
tableParams = filterParams.(tableName);
fieldNames = fieldnames(tableParams);
for i = 1:numel(fieldNames)
fieldName = fieldNames{i};
value = tableParams.(fieldName);
fullName = sprintf('%s.%s', tableName, fieldName);
% Handle various types of values for SQL query construction
if isempty(value)
% Skip empty values
if isempty(value) || (isa(value, 'QueryFilter') && isempty(value.value))
continue;
elseif isnumeric(value) && isnan(value)
filterClause = sprintf('%s IS NULL', fullName);
elseif isnumeric(value) && ~isEnumeration(value)
filterClause = sprintf('%s = %f', fullName, value);
elseif islogical(value) || (isnumeric(value) && ismember(value, [0, 1])) && ~isEnumeration(value)
filterClause = sprintf('%s = %d', fullName, value);
elseif ischar(value) || isstring(value)
filterClause = sprintf('%s = ''%s''', fullName, char(value));
elseif isEnumeration(value)
filterClause = sprintf('%s = ''%s''', fullName, value);
else
error('Unsupported data type for field "%s".', fullName);
end
filterClauses = [filterClauses, filterClause, ' AND '];
% Handle Filter class
if isa(value, 'SqlFilter')
if isnumeric(value.value)
filterClause = sprintf('%s %s %f', ...
fullName, value.operator, value.value);
elseif ischar(value.value) || isstring(value.value)
filterClause = sprintf('%s %s ''%s''', ...
fullName, value.operator, char(value.value));
else
continue; % Skip unsupported types
end
filterClauses = [filterClauses, filterClause, ' AND '];
else
% Handle direct values (legacy support)
if isnumeric(value) && isnan(value)
filterClause = sprintf('%s IS NULL', fullName);
elseif isnumeric(value)
filterClause = sprintf('%s = %f', fullName, value);
elseif ischar(value) || isstring(value)
filterClause = sprintf('%s = ''%s''', fullName, char(value));
else
continue; % Skip unsupported types
end
filterClauses = [filterClauses, filterClause, ' AND '];
end
end
end
% Remove trailing ' AND ' if any filters were added.
% Remove trailing ' AND ' if any filters were added
if ~isempty(filterClauses)
filterClauses = filterClauses(1:end-5);
query = [selectClause, ' ', fromClause, 'WHERE ', filterClauses];
whereClause = filterClauses(1:end-5);
else
query = [selectClause, ' ', fromClause];
whereClause = '';
end
end
@@ -980,5 +1044,59 @@ classdef DBHandler < handle
end
function fieldNames = getTableFieldNames(obj, tableName)
% Returns all field names for a given table as a cell array in the format {'TableName.fieldName'}
if isfield(obj.tables, tableName)
% Get raw field names
rawFields = fieldnames(obj.tables.(tableName));
% Create cell array with table name prefix
fieldNames = cellfun(@(x) [tableName, '.', x], ...
rawFields, ...
'UniformOutput', false);
else
error('Table "%s" not found in obj.tables.', tableName);
end
end
function coalesceStr = generateCoalesceString(obj, selectedFields)
% Generates a COALESCE string for selected fields
% Input:
% selectedFields: cell array of strings in format {'Table.field'}
% e.g., {'Runs.run_id', 'Runs.bitrate'}
% Output:
% coalesceStr: string with COALESCE statements
arguments
obj
selectedFields cell
end
% Initialize cell array to store each COALESCE statement
coalesceStatements = cell(length(selectedFields), 1);
% Generate COALESCE statement for each field
for i = 1:length(selectedFields)
% Split table and field name
parts = strsplit(selectedFields{i}, '.');
if length(parts) ~= 2
error('Field name must be in format "Table.field": %s', selectedFields{i});
end
tableName = parts{1};
fieldName = parts{2};
% Generate COALESCE statement
coalesceStatements{i} = sprintf('COALESCE(%s.%s, ''NaN'') AS %s ', ...
tableName, fieldName, fieldName);
end
% Join with comma, newline and MATLAB string continuation
coalesceStr = strjoin(coalesceStatements, [', ' sprintf('\n ')]);
% Add initial newline and spacing for formatting
coalesceStr = [sprintf('SELECT DISTINCT \n ') coalesceStr];
end
end
end