Many changes in DBHandler
new general processing structure just before splitting off the projects folder from this repo
This commit is contained in:
Silas Oettinghaus
40
Functions/Job_Processing/configureEqualizers_remove.m
Normal file
40
Functions/Job_Processing/configureEqualizers_remove.m
Normal file
@@ -0,0 +1,40 @@
|
||||
function [eq_, pf_, mlse_, mlse_db_, eq_post] = configureEqualizers(M, len_tr, vnle_order, dfe_order, mu_dc, mu_ffe, mu_dfe, pf_ncoeffs)
|
||||
% CONFIGUREEQUALIZERS Creates and configures equalizer objects
|
||||
%
|
||||
% Inputs:
|
||||
% M - PAM level
|
||||
% len_tr - Training length
|
||||
% vnle_order - Array with orders for VNLE [order1, order2, order3]
|
||||
% dfe_order - Array with orders for DFE
|
||||
% mu_dc - DC adaptation rate
|
||||
% mu_ffe - Array with adaptation rates for FFE [mu1, mu2, mu3]
|
||||
% mu_dfe - Adaptation rate for DFE
|
||||
% pf_ncoeffs - Number of coefficients for postfilter
|
||||
%
|
||||
% Outputs:
|
||||
% eq_ - Configured EQ object
|
||||
% pf_ - Configured Postfilter object
|
||||
% mlse_ - Configured MLSE_viterbi object
|
||||
% mlse_db_ - Configured MLSE_viterbi object for duobinary
|
||||
% eq_post - Configured FFE object for post-processing
|
||||
|
||||
% Configure main equalizer
|
||||
eq_ = EQ("Ne", vnle_order, "Nb", dfe_order, ...
|
||||
"training_length", len_tr, "training_loops", 5, "dd_loops", 5, ...
|
||||
"K", 2, "DCmu", mu_dc, "DDmu", [mu_ffe mu_dfe], ...
|
||||
"DFEmu", 0.005, "FFEmu", 0, "plotfinal", 0, "ideal_dfe", 1);
|
||||
|
||||
% Configure postfilter
|
||||
pf_ = Postfilter("ncoeff", pf_ncoeffs, "useBurg", 1);
|
||||
|
||||
% Configure MLSE objects
|
||||
mlse_ = MLSE_viterbi("duobinary_output", 0, 'M', M, ...
|
||||
'trellis_states', PAMmapper(M,0).levels);
|
||||
mlse_db_ = MLSE_viterbi("DIR", [1,1], "duobinary_output", 0, ...
|
||||
"M", M, "trellis_states", PAMmapper(M,0).levels);
|
||||
|
||||
% Configure post-FFE
|
||||
eq_post = FFE("epochs_tr", 5, "epochs_dd", 5, "len_tr", 4096*2, ...
|
||||
"mu_dd", 1e-4, "mu_tr", 0, "order", 2001, ...
|
||||
"sps", 1, "decide", 0);
|
||||
end
|
||||
106
Functions/Job_Processing/loadSignalData.m
Normal file
106
Functions/Job_Processing/loadSignalData.m
Normal file
@@ -0,0 +1,106 @@
|
||||
function [Bits, Symbols, Scpe_cell, found_sync] = loadSignalData(dataTable, options)
|
||||
% LOADSIGNALDATA Loads and synchronizes signal data from storage
|
||||
%
|
||||
% Inputs:
|
||||
% dataTable - Table with file paths and configuration
|
||||
% options - Struct with storage_path and max_occurences
|
||||
%
|
||||
% Outputs:
|
||||
% Symbols_mapped - Mapped symbols from bits
|
||||
% Symbols - Original symbols
|
||||
% Scpe_cell - Cell array of synchronized signals
|
||||
% found_sync - Boolean indicating if synchronization was successful
|
||||
|
||||
found_sync = 0;
|
||||
tempLocalStorage = 1;
|
||||
|
||||
% Part A: Check and load from local storage if available
|
||||
if tempLocalStorage == 1
|
||||
local_filename = sprintf('sync_data_run_%s.mat', num2str(dataTable.run_id));
|
||||
if exist(local_filename, 'file')
|
||||
% Load from local storage and return
|
||||
load(local_filename, 'Bits', 'Symbols', 'Scpe_cell');
|
||||
found_sync = 1;
|
||||
end
|
||||
end
|
||||
|
||||
% If not locally saved, load from storage
|
||||
if ~found_sync
|
||||
% Load transmitted bits
|
||||
Bits = load([options.storage_path, char(dataTable.tx_bits_path)]);
|
||||
Bits = Bits.Bits;
|
||||
|
||||
% Map bits to symbols
|
||||
M = double(dataTable.pam_level);
|
||||
fsym = dataTable.symbolrate;
|
||||
Symbols_mapped = PAMmapper(M,0).map(Bits);
|
||||
Symbols_mapped.fs = fsym;
|
||||
|
||||
% Load original symbols
|
||||
Symbols = load([options.storage_path, char(dataTable.tx_symbols_path)]);
|
||||
Symbols = Symbols.Symbols;
|
||||
|
||||
found_sync = 0;
|
||||
Scpe_cell = {};
|
||||
|
||||
% Try to load pre-synchronized data
|
||||
try
|
||||
Scpe_load = load([options.storage_path, char(dataTable.rx_sync_path)]);
|
||||
Scpe_cell = Scpe_load.S;
|
||||
[~,~,~,found_sync] = Scpe_cell{2}.tsynch("reference", Symbols, "fs_ref", fsym, "debug_plots", 1);
|
||||
catch
|
||||
% Continue to next method if this fails
|
||||
end
|
||||
end
|
||||
|
||||
% If not found, try with raw data
|
||||
if ~found_sync
|
||||
try
|
||||
Scpe_sig_raw = load([options.storage_path, char(dataTable.rx_raw_path(1))]);
|
||||
Scpe_sig_raw = Scpe_sig_raw.Scpe_sig_raw;
|
||||
Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in", Scpe_sig_raw.fs, "fs_out", 2*fsym);
|
||||
[~, Scpe_cell, ~, found_sync] = Scpe_sig_resampled.tsynch("reference", Symbols, "fs_ref", fsym, "debug_plots", 1);
|
||||
catch
|
||||
% Continue to next method if this fails
|
||||
end
|
||||
end
|
||||
|
||||
% Last attempt with mapped symbols
|
||||
if ~found_sync && exist('Scpe_sig_raw', 'var')
|
||||
if length(Symbols_mapped.signal) ~= sum(Symbols_mapped.signal == Symbols.signal)
|
||||
[~, Scpe_cell, ~, found_sync] = Scpe_sig_raw.tsynch("reference", Symbols_mapped, "fs_ref", fsym, "debug_plots", 0);
|
||||
end
|
||||
end
|
||||
|
||||
% Part B: Save to local storage if data was loaded and synced
|
||||
if tempLocalStorage == 1 && found_sync
|
||||
local_filename = sprintf('sync_data_run_%s.mat', num2str(dataTable.run_id));
|
||||
|
||||
% Create directory if it doesn't exist
|
||||
if ~exist('temp_sync_data', 'dir')
|
||||
mkdir('temp_sync_data');
|
||||
end
|
||||
|
||||
% List existing files and remove oldest if more than N
|
||||
max_local_files = 10; % Store up to N files
|
||||
files = dir('sync_data_run_*.mat');
|
||||
if length(files) >= max_local_files
|
||||
% Sort by date
|
||||
[~, idx] = sort([files.datenum]);
|
||||
% Delete oldest file
|
||||
delete(files(idx(1)).name);
|
||||
end
|
||||
|
||||
% Save current data
|
||||
save(local_filename, 'Bits', 'Symbols', 'Scpe_cell');
|
||||
end
|
||||
|
||||
% Limit number of occurrences
|
||||
if found_sync
|
||||
record_realizations = min(options.max_occurences, length(Scpe_cell));
|
||||
Scpe_cell = Scpe_cell(1:record_realizations);
|
||||
else
|
||||
warning('Could not synchronize the received signal with the stored symbols!');
|
||||
end
|
||||
|
||||
end
|
||||
26
Functions/Job_Processing/preprocessSignal.m
Normal file
26
Functions/Job_Processing/preprocessSignal.m
Normal file
@@ -0,0 +1,26 @@
|
||||
function Scpe_sig = preprocessSignal(Scpe_sig, Symbols, fsym)
|
||||
% PREPROCESSSIGNAL Performs standard preprocessing on a signal
|
||||
%
|
||||
% Inputs:
|
||||
% Scpe_sig - Input signal
|
||||
% Symbols - Reference symbols for synchronization
|
||||
% fsym - Symbol frequency
|
||||
%
|
||||
% Outputs:
|
||||
% Scpe_sig - Preprocessed signal
|
||||
|
||||
% Resample to 2x symbol rate
|
||||
Scpe_sig = Scpe_sig.resample("fs_out", 2*fsym);
|
||||
|
||||
% Synchronize with reference
|
||||
[Scpe_sig, ~] = Scpe_sig.tsynch("reference", Symbols, "fs_ref", fsym, "debug_plots", 0);
|
||||
|
||||
% Apply Gaussian filter
|
||||
Scpe_sig = Filter('filtdegree', 4, "f_cutoff", Symbols.fs.*0.6, ...
|
||||
"fs", Scpe_sig.fs, "filterType", filtertypes.gaussian, ...
|
||||
"active", true).process(Scpe_sig);
|
||||
|
||||
% Remove DC offset
|
||||
Scpe_sig = Scpe_sig - mean(Scpe_sig.signal);
|
||||
|
||||
end
|
||||
47
Functions/Job_Processing/printResults.m
Normal file
47
Functions/Job_Processing/printResults.m
Normal file
@@ -0,0 +1,47 @@
|
||||
function printResults(run_id, M, Symbols, vnle_pf_package, dbtgt_package, occ)
|
||||
% PRINTRESULTS Prints formatted results from equalization
|
||||
%
|
||||
% Inputs:
|
||||
% run_id - Run identifier
|
||||
% M - PAM level
|
||||
% Symbols - Symbol data with fs property
|
||||
% vnle_pf_package - VNLE+PF results package
|
||||
% dbtgt_package - DB target results package (optional)
|
||||
% occ - Occurrence index
|
||||
|
||||
% Print header
|
||||
fprintf("==== EQUALIZATION RUN-ID %d | PAM-%d | %.2f GBd ====\n\n", run_id, M, Symbols.fs.*1e-9);
|
||||
|
||||
% VNLE Results
|
||||
if ~isempty(vnle_pf_package)
|
||||
vnle_result = vnle_pf_package{occ}.resultsVNLE;
|
||||
mlse_result = vnle_pf_package{occ}.resultsMLSE;
|
||||
|
||||
fprintf(">> VNLE Results:\n");
|
||||
fprintf(" BER: %.2e\n", vnle_result.BER);
|
||||
fprintf(" BER (pre-code): %.2e\n", vnle_result.BER_precoded);
|
||||
fprintf(" SNR: %.2f dB\n", vnle_result.SNR);
|
||||
fprintf(" GMI: %.4f\n", vnle_result.GMI);
|
||||
fprintf(" Linerate: %.2f Gbps\n", Symbols.fs .* floor(log2(M)*10)/10 .*1e-9);
|
||||
fprintf(" AIR: %.2f Gbps\n", vnle_result.AIR.*1e-9);
|
||||
fprintf("\n");
|
||||
|
||||
% MLSE Results
|
||||
fprintf(">> MLSE Results:\n");
|
||||
fprintf(" BER: %.2e\n", mlse_result.BER);
|
||||
fprintf(" BER (pre-code): %.2e\n", mlse_result.BER_precoded);
|
||||
fprintf(" Channel Alpha: %.2f\n", mlse_result.Alpha);
|
||||
fprintf("\n");
|
||||
end
|
||||
|
||||
% DB Target Results
|
||||
if ~isempty(dbtgt_package)
|
||||
dbtgt = dbtgt_package{occ}.resultsDBtgt;
|
||||
fprintf(">> DB Target Results:\n");
|
||||
fprintf(" BER: %.2e\n", dbtgt.BER);
|
||||
fprintf(" BER (pre-code): %.2e\n", dbtgt.BER_precoded);
|
||||
fprintf("\n");
|
||||
end
|
||||
|
||||
fprintf("- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \n\n");
|
||||
end
|
||||
11
Functions/Job_Processing/queryRunid.m
Normal file
11
Functions/Job_Processing/queryRunid.m
Normal file
@@ -0,0 +1,11 @@
|
||||
function dataTable = queryRunid(run_id, database)
|
||||
|
||||
% Query database for run configuration
|
||||
filterParams = database.tables;
|
||||
filterParams.Runs = struct('run_id', run_id);
|
||||
|
||||
[dataTable, ~] = database.queryDB(filterParams, database.getTableFieldNames('Runs'));
|
||||
[~, uniqueIdx] = unique(dataTable.run_id); % Get unique run_id indices
|
||||
dataTable = dataTable(uniqueIdx, :); % Extract unique configurations for each run_id
|
||||
|
||||
end
|
||||
213
Functions/Job_Processing/submitJobs.m
Normal file
213
Functions/Job_Processing/submitJobs.m
Normal file
@@ -0,0 +1,213 @@
|
||||
function [results,wh] = submitJobs(run_ids, dsp_options, submit_mode, submit_options)
|
||||
% SUBMITJOBS Submits dsp_runid jobs for processing
|
||||
%
|
||||
% Inputs:
|
||||
% run_ids - Single run ID or array of Run IDs for processing
|
||||
% dsp_options - Options for dsp_runid function
|
||||
% submit_mode - Execution mode: 'parallel' or 'linear'
|
||||
% options - Optional parameters
|
||||
%
|
||||
% Outputs:
|
||||
% results - Cell array of results from each job
|
||||
% wh - Updated DataStorage object
|
||||
|
||||
% USAGE:
|
||||
% % === SETTINGS ===
|
||||
%
|
||||
% dsp_options.append_to_db = 1;
|
||||
% dsp_options.max_occurences = 15;
|
||||
% dsp_options.database_path = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor\';
|
||||
% dsp_options.database_name = 'silas_labor_newdsp_newstructure.db';
|
||||
% dsp_options.storage_path = 'Z:\2024\sioe_labor\';
|
||||
% dsp_options.parameters = struct();
|
||||
% dsp_options.parameters.mu_dc = [0.005];
|
||||
%
|
||||
% % === Get Run ID's ===
|
||||
% db = DBHandler("pathToDB", [dsp_options.database_path, dsp_options.database_name], "type", "sqlite");
|
||||
% fp = QueryFilter();
|
||||
% % fp.where('Runs', 'run_id','EQUALS', 5108);
|
||||
% fp.where('Runs', 'is_mpi','EQUALS', 0);
|
||||
% fp.where('Runs', 'fiber_length','EQUALS', 1);
|
||||
% fp.where('Runs', 'wavelength','EQUALS', 1310);
|
||||
% fp.where('Runs', 'db_mode','EQUALS', 0);
|
||||
% fp.where('Runs', 'rop_attenuation','EQUALS', 0);
|
||||
% fp.where('Runs', 'pam_level','EQUALS', 4);
|
||||
% fp.where('Runs', 'bitrate','EQUALS', 360e9);
|
||||
% fp.where('Runs', 'power_pd_in','GREATER_THAN', 7);
|
||||
% [dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
|
||||
|
||||
% % === Initialize DataStorage ===
|
||||
% wh = DataStorage(dsp_options.parameters);
|
||||
% wh.addStorage("ffe_package");
|
||||
% wh.addStorage("mlse_package");
|
||||
% wh.addStorage("vnle_package");
|
||||
% wh.addStorage("dbtgt_package");
|
||||
% wh.addStorage("dbenc_package");
|
||||
%
|
||||
% % === RUN IT ===
|
||||
% [results,wh] = submitJobs(dataTable.run_id(:), dsp_options, "parallel", 'wh', wh, 'waitbar', true);
|
||||
% wh.getStoValue('ffe_package',0.005);
|
||||
% wh.getStoValue('mlse_package',0.005);
|
||||
|
||||
|
||||
arguments
|
||||
run_ids int32 = 0
|
||||
dsp_options struct = struct();
|
||||
submit_mode processingMode = processingMode.serial
|
||||
submit_options.waitbar (1,1) logical = true
|
||||
submit_options.wh = DataStorage(struct()); % Optional DataStorage object
|
||||
end
|
||||
|
||||
% Ensure run_ids is a row vector
|
||||
run_ids = run_ids(:)';
|
||||
|
||||
% Get number of jobs per run_id
|
||||
nJobsPerRunId = submit_options.wh.getLastLinIndice;
|
||||
nRunIds = length(run_ids);
|
||||
totalJobs = nJobsPerRunId * nRunIds;
|
||||
|
||||
% Initialize results array for all jobs
|
||||
results = cell(nJobsPerRunId, nRunIds);
|
||||
futures = cell(totalJobs, 1);
|
||||
|
||||
% Initialize waitbar if requested
|
||||
if submit_options.waitbar
|
||||
h = waitbar(0, 'Processing Jobs...');
|
||||
cleanupObj = onCleanup(@() delete(h));
|
||||
end
|
||||
|
||||
% Process jobs based on mode
|
||||
if submit_mode == processingMode.parallel
|
||||
setupParallelPool();
|
||||
|
||||
% Submit jobs to parallel pool
|
||||
jobCounter = 0;
|
||||
for r = 1:nRunIds
|
||||
for k = 1:nJobsPerRunId
|
||||
jobCounter = jobCounter + 1;
|
||||
optionalVars = buildOptionalVars(k, submit_options.wh);
|
||||
|
||||
% Submit job with proper arguments
|
||||
futures{jobCounter} = parfeval(@dsp_runid, 1, run_ids(r), ...
|
||||
"database_name", dsp_options.database_name, ...
|
||||
"append_to_db", dsp_options.append_to_db, ...
|
||||
"database_path", dsp_options.database_path, ...
|
||||
"max_occurences", dsp_options.max_occurences, ...
|
||||
"storage_path", dsp_options.storage_path, ...
|
||||
"parameters", optionalVars);
|
||||
|
||||
fprintf('[RunID %d, Job %d] Submitted to pool.\n', run_ids(r), k);
|
||||
end
|
||||
end
|
||||
|
||||
if submit_options.waitbar
|
||||
setupParallelWaitbar(futures, totalJobs, h);
|
||||
end
|
||||
|
||||
% Fetch results
|
||||
jobCounter = 0;
|
||||
for r = 1:nRunIds
|
||||
for k = 1:nJobsPerRunId
|
||||
jobCounter = jobCounter + 1;
|
||||
try
|
||||
results{k,r} = fetchOutputs(futures{jobCounter});
|
||||
fprintf('[RunID %d, Job %d] Completed successfully.\n', run_ids(r), k);
|
||||
storeResult(results{k,r}, k, submit_options.wh);
|
||||
catch ME
|
||||
handleError(ME, k, run_ids(r));
|
||||
results{k,r} = ME;
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
elseif submit_mode == processingMode.serial
|
||||
% Process jobs sequentially
|
||||
jobCounter = 0;
|
||||
for r = 1:nRunIds
|
||||
for k = 1:nJobsPerRunId
|
||||
jobCounter = jobCounter + 1;
|
||||
optionalVars = buildOptionalVars(k, submit_options.wh);
|
||||
|
||||
try
|
||||
fprintf('[RunID %d, Job %d] Running in linear mode...\n', run_ids(r), k);
|
||||
|
||||
results{k,r} = dsp_runid(run_ids(r),...
|
||||
"database_name", dsp_options.database_name,...
|
||||
"append_to_db", dsp_options.append_to_db,...
|
||||
"database_path", dsp_options.database_path,...
|
||||
"max_occurences", dsp_options.max_occurences,...
|
||||
"storage_path", dsp_options.storage_path,...
|
||||
"parameters", optionalVars);
|
||||
|
||||
fprintf('[RunID %d, Job %d] Completed successfully.\n', run_ids(r), k);
|
||||
storeResult(results{k,r}, k, submit_options.wh);
|
||||
catch ME
|
||||
handleError(ME, k, run_ids(r));
|
||||
results{k,r} = ME;
|
||||
end
|
||||
|
||||
% Update waitbar if requested
|
||||
if submit_options.waitbar
|
||||
waitbar(jobCounter/totalJobs, h, sprintf('Completed %d/%d jobs', jobCounter, totalJobs));
|
||||
end
|
||||
end
|
||||
end
|
||||
else
|
||||
error('')
|
||||
end
|
||||
|
||||
wh = submit_options.wh;
|
||||
|
||||
% Helper functions remain the same except for handleError
|
||||
% Modified handleError function to include run_id
|
||||
function handleError(ME, jobIndex, run_id)
|
||||
fprintf('[RunID %d, Job %d] ERROR [%s]: %s\n', run_id, jobIndex, ME.identifier, ME.message);
|
||||
for st = ME.stack'
|
||||
fprintf(' %s:%d (%s)\n', st.file, st.line, st.name);
|
||||
end
|
||||
fprintf('Full report:\n%s\n', getReport(ME,'extended'));
|
||||
end
|
||||
|
||||
% Other helper functions remain unchanged
|
||||
function setupParallelPool()
|
||||
curpool = gcp('nocreate');
|
||||
if isempty(curpool)
|
||||
parpool;
|
||||
else
|
||||
if ~isempty(curpool.FevalQueue.QueuedFutures) || ~isempty(curpool.FevalQueue.RunningFutures)
|
||||
oldq = length(curpool.FevalQueue.QueuedFutures) + length(curpool.FevalQueue.RunningFutures);
|
||||
curpool.FevalQueue.cancelAll;
|
||||
fprintf('Canceled %d unfetched jobs from old queue.\n', oldq);
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function optionalVars = buildOptionalVars(jobIndex, wh)
|
||||
optionalVars = struct();
|
||||
if ~isempty(wh.getDimension)
|
||||
[parametervalues, parameternames] = wh.getPhysIndicesByLinIndex(jobIndex);
|
||||
for pidx = 1:numel(parameternames)
|
||||
optionalVars.(parameternames{pidx}) = parametervalues{pidx};
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function setupParallelWaitbar(futures, totalJobs, h)
|
||||
updateWaitbar = @(~) waitbar(sum(cellfun(@(f) strcmp(f.State, 'finished'), futures))/totalJobs, h, ...
|
||||
sprintf('Completed %d/%d jobs', sum(cellfun(@(f) strcmp(f.State, 'finished'), futures)), totalJobs));
|
||||
|
||||
futureArray = [futures{:}];
|
||||
afterEach(futureArray, updateWaitbar, 0);
|
||||
end
|
||||
|
||||
function storeResult(result, jobIndex, wh)
|
||||
if ~isempty(wh)
|
||||
wh.addValueToStorageByLinIdx(result.ffe_package, 'ffe_package', jobIndex);
|
||||
wh.addValueToStorageByLinIdx(result.mlse_package, 'mlse_package', jobIndex);
|
||||
wh.addValueToStorageByLinIdx(result.vnle_package, 'vnle_package', jobIndex);
|
||||
wh.addValueToStorageByLinIdx(result.dbtgt_package, 'dbtgt_package', jobIndex);
|
||||
wh.addValueToStorageByLinIdx(result.dbenc_package, 'dbenc_package', jobIndex);
|
||||
end
|
||||
end
|
||||
|
||||
end
|
||||
Reference in New Issue
Block a user