silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

High Speed Auswertung und Database code

Browse Source
This commit is contained in:
sioe
2024-11-13 10:49:49 +01:00
parent ca769f2b3d
commit 553ed19b9f
20 changed files with 2931 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

514
Classes/DataBaseHandler/DBHandler.m Normal file
View File

@@ -0,0 +1,514 @@
classdef DBHandler < handle
% DBHANDLER Class to handle database queries
% This class provides methods to interact with an SQLite database, including
% inserting data, retrieving table names, and appending new rows.
properties
conn % Database connection object
pathToDB % Path to the SQLite database
tableNames % Cell array containing names of all tables in the database
tables = struct(); % Structure containing MATLAB tables for each database table
distinctValues
end
methods
function obj = DBHandler(options)
% DBHANDLER Constructor for the DBHandler class
% Initializes the database connection and retrieves table and field names.
%
% Usage:
% obj = DBHandler('pathToDB', 'path/to/database.db');
arguments
options.pathToDB = ""; % Default value for pathToDB if not provided
end
% Assign values to class properties based on input arguments
fn = fieldnames(options);
for n = 1:numel(fn)
try
obj.(fn{n}) = options.(fn{n});
end
end
% Establish a connection to the SQLite database
try
obj.conn = sqlite(obj.pathToDB);
catch e
error('Failed to connect to the database: %s', e.message);
end
% Get table names and the first rows of each table to understand the structure
obj.getTableNames();
obj.getTables();
obj.getDistinctValues();
end
function obj = getTableNames(obj)
% Get all table names from the database
try
result = fetch(obj.conn, 'SELECT name FROM sqlite_master WHERE type="table"');
obj.tableNames = result.name;
catch e
error('Failed to retrieve table names: %s', e.message);
end
end
function obj = getTables(obj)
% Get a preview (first row) of each table to understand its structure
for i = 1:numel(obj.tableNames)
try
tableName = obj.tableNames{i};
results = fetch(obj.conn, sprintf('SELECT * FROM %s WHERE 1 = 2', tableName));
% Matlab cant handle if there is a NULL in a returned
% datarow... therefore do not return a row using the
% above condition which is never true
% results = sqlread(obj.conn, tableName, MaxRows=1);
for l = 1:numel(results.Properties.VariableNames)
varName = results.Properties.VariableNames{l};
obj.tables.(tableName).(varName) = []; % Store the preview as a reference
end
catch e
warning('Failed to read the table %s: %s', tableName, e.message);
end
end
end
function obj = getDistinctValues(obj)
% getDistinctValues Retrieves distinct values for each relevant field in all tables
% excluding fields ending with "_id". Stores distinct values in the 'distinctValues'
% property.
% Initialize a structure to store distinct values for each table
obj.distinctValues = struct();
% Iterate over each table in obj.tables
tableNames = fieldnames(obj.tables);
for i = 1:numel(tableNames)
tableName = tableNames{i};
% Initialize a sub-struct to store distinct values for each field in the table
obj.distinctValues.(tableName) = struct();
% Get all fields of the current table
fieldNames = fieldnames(obj.tables.(tableName));
% Iterate over each field
for j = 1:numel(fieldNames)
fieldName = fieldNames{j};
% Skip fields ending with '_id' as they don't contain useful distinct values
if endsWith(fieldName, '_id')
continue;
end
% Construct SQL to get distinct values for the current field
query = sprintf('SELECT DISTINCT %s FROM %s', fieldName, tableName);
% Execute query and fetch distinct values
try
result = fetch(obj.conn, query);
% Store the distinct values in the structure
if ~isempty(result)
distinctValues = table2array(result);
else
distinctValues = [];
end
obj.distinctValues.(tableName).(fieldName) = distinctValues;
catch e
warning('Failed to retrieve distinct values for %s.%s: %s', tableName, fieldName, e.message);
obj.distinctValues.(tableName).(fieldName) = [];
end
end
end
% Display the distinct values (optional, for debugging purposes)
disp('Distinct values for each field:');
disp(obj.distinctValues);
end
function lastID = appendToTable(obj, tableName, newRow)
% appendToTable Appends a new row to the specified table
%
% Usage:
% appendToTable(tableName, newRow)
%
% Inputs:
% tableName: The name of the table to append data to.
% newRow: A MATLAB table or struct containing the new row to be appended.
% Check if the table exists in the fetched tables
if ~isfield(obj.tables, tableName)
error('Table %s does not exist in the database or has not been fetched.', tableName);
end
% Convert newRow to a table if it is a struct
if isstruct(newRow)
fields = fieldnames(newRow);
emptyFields = structfun(@isempty,newRow);
if sum(emptyFields)>0
newRow.(fields{emptyFields==1}) = NaN;
disp(['In Table: ',tableName,': ',fields{emptyFields==1},' was empty, is now NaN ',newRow.(fields{emptyFields==1})])
end
newRow = struct2table(newRow);
end
% Ensure the new row matches the structure of the existing table
existingTableStructure = obj.tables.(tableName);
% Perform data type checks and conversions
for colName = newRow.Properties.VariableNames
% Extract the value and its intended column type
value = newRow.(colName{1});
existingValue = existingTableStructure.(colName{1});
% If the value is a class object, convert it to JSON format
if isobject(value) && ~isdatetime(value) && ~isa(value,"string")
newRow.(colName{1}) = string(jsonencode(value));
% If the value is a character array, convert it to a string
elseif ischar(value)
newRow.(colName{1}) = string(value);
end
end
% Append the new row to the database table
try
sqlwrite(obj.conn, tableName, newRow);
% disp(['Successfully appended new row to the table ', tableName]);
catch e
error('Failed to append to the table %s: %s', tableName, e.message);
end
% Retrieve the measurement_id of the newly inserted row for linking other tables
result = fetch(obj.conn, 'SELECT last_insert_rowid()');
lastID = result{1, 1}; % Access the value directly from the table
end
function exists = checkIfRunExists(obj, table2check, column2check, value2check)
% checkIfRunExists Checks if a specific value exists in a specified column of a table
%
% Usage:
% exists = checkIfRunExists(table2check, column2check, value2check)
%
% Inputs:
% table2check: The name of the table to check for duplicates.
% column2check: The name of the column to check within the specified table.
% value2check: The value to check for in the specified column.
%
% Outputs:
% exists: Boolean indicating whether the value already exists in the table.
% Ensure the specified table and column exist in the database
if ~isfield(obj.tables, table2check)
error('Table %s does not exist in the database.', table2check);
end
% Ensure the specified column exists in the table structure
if ~isfield(obj.tables.(table2check), column2check)
error('Column %s does not exist in the table %s.', column2check, table2check);
end
% Construct the query to check for the value in the specified column
query = sprintf('SELECT COUNT(*) FROM %s WHERE %s = "%s"', table2check, column2check, value2check);
% Execute the query and pass the value2check to avoid SQL injection issues
try
result = fetch(obj.conn, query);
count = result{1, 1}; % Extract the count from the result
catch e
error('Failed to execute the duplicate check query: %s', e.message);
end
% If count is greater than 0, then the value exists in the table
exists = count > 0;
if exists
disp(['The value "', value2check, '" already exists in the column "', column2check, '" of the table "', table2check, '".']);
else
% disp(['The value "', value2check, '" does not exist in the column "', column2check, '" of the table "', table2check, '".']);
end
end
function answer = fetch(obj,query)
answer = fetch(obj.conn,query);
end
function result = getPathsWithFlexibleFilter(obj, filterParams, selectedFields)
% getPathsWithFlexibleFilter Retrieves values from Runs table with flexible filtering
% and lets the user select which fields to include in the SELECT statement.
%
% Usage:
% [rxRawPaths, filteredValues] = getPathsWithFlexibleFilter(filterParams)
%
% Inputs:
% filterParams: A structure containing the parameters with their values.
% If left empty, two popup windows will prompt the user for input.
%
% Outputs:
% rxRawPaths: Cell array of values from the Runs table matching the criteria.
% filteredValues: Table of distinct values for parameters included in filterParams.
arguments
obj
filterParams = [];
selectedFields = [];
end
% Step 1: Prompt the user to input filter parameters if not provided
if isempty(filterParams)
filterParams = obj.promptFilterParameters();
end
% Step 2: Prompt the user to select fields to include in the SELECT statement
if isempty(selectedFields)
selectedFields = obj.promptSelectFields();
else
if iscell(selectedFields)
elseif isstruct(selectedFields)
end
end
% Step 3: Construct the SQL query based on the inputs
query = obj.constructSQLQuery(filterParams, selectedFields);
% Step 4: Execute the query and handle results
result = obj.fetch(query);
end
function query = constructSQLQuery(obj, filterParams, selectedFields)
% constructSQLQuery Constructs the SQL query based on filter parameters and selected fields.
% Construct the SELECT clause dynamically based on user selection
selectClause = 'SELECT DISTINCT ';
for i = 1:numel(selectedFields)
fieldParts = strsplit(selectedFields{i}, '.');
tableName = fieldParts{1};
fieldName = fieldParts{2};
if isnumeric(obj.tables.(tableName).(fieldName))
selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', -1) AS ', fieldName];
else
selectClause = [selectClause, 'COALESCE(', selectedFields{i}, ', '''') AS ', fieldName];
end
if i < numel(selectedFields)
selectClause = [selectClause, ', '];
else
selectClause = [selectClause, ' '];
end
end
% Construct the FROM and WHERE clause
baseQuery = [selectClause, 'FROM Runs ' ...
'LEFT JOIN Configurations ON Runs.run_id = Configurations.run_id ' ...
'LEFT JOIN Measurements ON Runs.run_id = Measurements.run_id ' ...
'LEFT JOIN BERs ON Runs.run_id = BERs.run_id ' ...
'LEFT JOIN Equalizer ON BERs.eq_id = Equalizer.eq_id ' ...
'WHERE '];
% Loop through each table in filterParams
filterClauses = [];
tableNames = fieldnames(filterParams);
for t = 1:numel(tableNames)
tableName = tableNames{t};
tableParams = filterParams.(tableName);
% Loop through each parameter in the table
fieldNames = fieldnames(tableParams);
for i = 1:numel(fieldNames)
fieldName = fieldNames{i};
value = tableParams.(fieldName);
% Construct the full column name in the format "tableName.fieldName"
fullName = sprintf('%s.%s', tableName, fieldName);
% Handle different types of values for SQL query construction
if isempty(value)
% Skip this parameter if it is empty (include all values)
continue;
elseif isnumeric(value) && isnan(value)
% If value is NaN, use IS NULL in SQL
filterClause = sprintf('%s IS NULL', fullName);
elseif isnumeric(value)
filterClause = sprintf('%s = %f', fullName, value);
elseif islogical(value) || (isnumeric(value) && ismember(value, [0, 1]))
filterClause = sprintf('%s = %d', fullName, value);
elseif ischar(value) || isstring(value)
filterClause = sprintf('%s = "%s"', fullName, value);
else
error('Unsupported data type for field "%s".', fullName);
end
% Add the constructed filter clause to the list
filterClauses = [filterClauses, filterClause, ' AND '];
end
end
% Remove trailing ' AND ' from the filter clauses if any filters were added
if ~isempty(filterClauses)
filterClauses = filterClauses(1:end-5);
end
% Construct the final SQL query
if isempty(filterClauses)
query = [selectClause, 'FROM Runs ' ...
'LEFT JOIN Configurations ON Runs.run_id = Configurations.run_id ' ...
'LEFT JOIN Measurements ON Runs.run_id = Measurements.run_id ' ...
'LEFT JOIN BERs ON Runs.run_id = BERs.run_id'];
else
query = [baseQuery, filterClauses];
end
end
function selectedFields = promptSelectFields(obj)
% promptSelectFields Prompts the user to select fields from multiple tables to include in the SELECT statement using settingsdlg.
% Get all possible fields from all tables (excluding sqlite_sequence)
tableNames = fieldnames(obj.tables);
tableNames = setdiff(tableNames, {'sqlite_sequence'}); % Remove sqlite_sequence
% Prepare the inputs for settingsdlg
promptSettings = {};
allFieldsFullName = {};
convertedFieldNames = {};
for i = 1:numel(tableNames)
tableFields = fieldnames(obj.tables.(tableNames{i}));
for j = 1:numel(tableFields)
fieldName = tableFields{j};
fullName = sprintf('%s.%s', tableNames{i}, fieldName);
convertedName = strrep(fullName, '.', '_'); % Replace '.' with '_'
allFieldsFullName{end + 1} = fullName; % Add full name to the list
convertedFieldNames{end + 1} = convertedName; % Store the converted name
% Add the field name and checkbox setting to the prompt
promptSettings{end + 1} = {sprintf('Include %s', fullName), convertedName};
promptSettings{end + 1} = false; % Default: not selected
end
end
% Create the settings dialog
[settings, button] = settingsdlg(...
'title', 'Select Fields for the SQL Query', ...
'description', 'Check the boxes for the fields you want to include in the SELECT statement.', ...
promptSettings{:} ...
);
% If the user cancels, default to selecting all fields
if strcmp(button, 'cancel')
selectedFields = allFieldsFullName;
return;
end
% Parse user input into selectedFields
selectedFields = {};
for i = 1:numel(allFieldsFullName)
convertedName = convertedFieldNames{i};
if isfield(settings, convertedName) && settings.(convertedName) % Add to selectedFields if the checkbox was selected
selectedFields{end + 1} = allFieldsFullName{i}; %#ok<AGROW>
end
end
% If no fields are selected, default to selecting all fields
if isempty(selectedFields)
selectedFields = allFieldsFullName;
end
end
function filterParams = promptFilterParameters(obj)
% promptFilterParameters Prompts the user to enter filter parameters using the settingsdlg framework.
% Get all possible parameters from all tables (excluding sqlite_sequence)
tableNames = fieldnames(obj.tables);
tableNames = setdiff(tableNames, {'sqlite_sequence'}); % Remove sqlite_sequence
% Prepare the inputs for settingsdlg with sections and separators
promptSettings = {};
allFieldsFullName = {};
convertedFieldNames = {};
for i = 1:numel(tableNames)
% Add a separator for each table section
promptSettings{end + 1} = 'separator';
promptSettings{end + 1} = tableNames{i};
% Get all fields from the current table
tableFields = fieldnames(obj.tables.(tableNames{i}));
% Prepare each field to be added to the dialog
for j = 1:numel(tableFields)
fieldName = tableFields{j};
fullName = sprintf('%s.%s', tableNames{i}, fieldName);
convertedName = strrep(fullName, '.', '_'); % Replace '.' with '_'
allFieldsFullName{end + 1} = fullName; % Add full name to the list
convertedFieldNames{end + 1} = convertedName; % Store the converted name
% Add the field name and value setting to the prompt
promptSettings{end + 1} = {sprintf('%s', fullName), convertedName};
promptSettings{end + 1} = [];
end
end
% Create the settings dialog
[settings, button] = settingsdlg(...
'title', 'Input Parameters for Filtering', ...
'description', 'Enter the values for each field to filter. Leave empty to include all values. Type NaN for NULL.', ...
promptSettings{:} ...
);
% If the user cancels, return an empty struct
if strcmp(button, 'cancel')
filterParams = struct();
return;
end
% Parse user input into filterParams structure
filterParams = struct();
for i = 1:numel(allFieldsFullName)
value = settings.(convertedFieldNames{i});
% Split full name to get table and field names
fieldParts = strsplit(allFieldsFullName{i}, '.');
tableName = fieldParts{1};
fieldName = fieldParts{2};
% If the table does not exist in the filterParams struct, create it
if ~isfield(filterParams, tableName)
filterParams.(tableName) = struct();
end
% Assign values to the respective fields under each table
if isempty(value)
filterParams.(tableName).(fieldName) = []; % Set to empty to include all values
elseif isnumeric(value) && isnan(value)
filterParams.(tableName).(fieldName) = NaN; % Use NaN to handle as NULL
else
filterParams.(tableName).(fieldName) = value; % Use the entered value
end
end
end
end
end