silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

restructure and organize

Browse Source
This commit is contained in:
Silas Oettinghaus
2026-06-22 22:58:58 +02:00
parent c4f75b7ec4
commit 33ec5b3116
36 changed files with 1914 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
Functions/EQ_recipes/dsp_recipe_minimal.m Normal file
View File

@@ -0,0 +1,36 @@
function output = dsp_recipe_minimal(Scpe_sig_raw, Symbols, Tx_bits, options)
%DSP_RECIPE_MINIMAL Minimal example recipe for the DSP job framework.
% This recipe intentionally performs only light preprocessing and records
% summary values. It demonstrates the recipe interface without running a
% full equalizer chain.
arguments
Scpe_sig_raw
Symbols
Tx_bits
options.fsym
options.M
options.duob_mode
options.dataTable table
options.userParameters struct = struct()
options.debug_plots (1,1) logical = false
end
Scpe_sig = preprocessSignal(Scpe_sig_raw, Symbols, options.fsym, ...
"mode", "auto", ...
"debug_plots", options.debug_plots);
eq_ffe = FFE("epochs_tr",5,"epochs_dd",5,"len_tr",4096*2,"mu_dd",1e-1,"mu_tr",0.4,"order",25,...
"sps",2,"decide",0,"optmize_mus",0,"dd_mode",options.userParameters.dd_mode,"adaption_technique","nlms","mu_dc",1.021e-05);
ffe_results = ffe(eq_ffe, options.M, Scpe_sig, Symbols, Tx_bits, ...
"precode_mode", options.duob_mode, ...
'showAnalysis', options.debug_plots, ...
"postFFE", [], ...
"eth_style_symbol_mapping", 0);
ffe_results.config.equalizer_structure = "ffe";
ffe_results.metrics.print("description",'FFE');
output.ffe_package = ffe_results;
end

0
Functions/EQ_blocks/dsp_scope_signal.m → Functions/EQ_recipes/dsp_scope_signal.m
View File

78
Functions/EQ_recipes/mpi_recipe_dev.m Normal file
View File

@@ -0,0 +1,78 @@
function output = mpi_recipe_dev(Scpe_sig_raw, Symbols, Tx_bits, options)
%mpi_recipe_dev Minimal example recipe for the DSP job framework.
% This recipe intentionally performs only light preprocessing and records
% summary values. It demonstrates the recipe interface without running a
% full equalizer chain.
arguments
Scpe_sig_raw
Symbols
Tx_bits
options.fsym
options.M
options.duob_mode
options.dataTable table
options.userParameters struct = struct()
options.debug_plots (1,1) logical = false
end
Scpe_sig = preprocessSignal(Scpe_sig_raw, Symbols, options.fsym, ...
"mode", "auto", ...
"debug_plots", options.debug_plots);
mu_dc = 0; % 1e-5
dc_buffer_len = 0;
ffe_buffer_len = 0;
smoothing_buffer_length = options.userParameters.smoothing_length;
smoothing_buffer_update = 1;
eq_settings = { ...
"epochs_tr", 5, ...
"epochs_dd", 5, ...
"len_tr", 4096*2, ...
"mu_dd", 1e-1, ...
"mu_tr", 0.4, ...
"order", 25, ...
"sps", 2, ...
"decide", 0, ...
"optmize_mus", 0, ...
"dd_mode", 1, ...
"adaption_technique", "nlms", ...
"mu_dc", mu_dc};
eq_ffe = FFE(eq_settings{:});
% showLevelScatter(Scpe_sig_raw, Symbols, ...
% "fsym", options.fsym, ...
% "fignum", options.dataTable.run_id, ...
% "normalize", true);
%%
% tic
% ffe_results = ffe(eq_ffe, options.M, Scpe_sig, Symbols, Tx_bits, ...
% "precode_mode", options.duob_mode, ...
% 'showAnalysis', options.debug_plots, ...
% "postFFE", [], ...
% "eth_style_symbol_mapping", 0);
% toc
% ffe_results.metrics.print("description",'FFE');
% output.ffe_package = ffe_results;
%%
eq_ffe_dcr = FFE_DCremoval_adaptive_mu(eq_settings{:}, ...
"dc_buffer_len",dc_buffer_len, ...
"ffe_buffer_len",ffe_buffer_len,...
"smoothing_buffer_length",smoothing_buffer_length,...
"smoothing_buffer_update",smoothing_buffer_update);
tic
ffe_results_dcr = ffe(eq_ffe_dcr, options.M, Scpe_sig, Symbols, Tx_bits, ...
"precode_mode", options.duob_mode, ...
'showAnalysis', options.debug_plots, ...
"postFFE", [], ...
"eth_style_symbol_mapping", 0);
toc
ffe_results_dcr.metrics.print("description",'FFE DCR');
output.ffe_dcr_package = ffe_results_dcr;
end

455
Functions/EQ_visuals/showLevelScatter.m
View File

@@ -1,133 +1,370 @@
function [symbols_for_lvl,avg_for_lvl] = showLevelScatter(eq_signal,ref_symbols,options)
function [symbols_for_lvl, avg_for_lvl, info] = showLevelScatter(rxInput, refSymbols, options)
%SHOWLEVELSCATTER Plot received samples separated by reference PAM level.
% Supports plain numeric vectors, Signal objects, synchronized scope cell
% arrays, and raw unsynchronized Signal input. Raw Signal input is
% synchronized to refSymbols and stitched before plotting.
arguments
eq_signal
ref_symbols
options.fignum (1,1) double = NaN % Default to NaN if not provided
options.displayname (1,:) char = '' % Default to an empty string if not provided
options.f_sym =1e6;
rxInput
refSymbols
options.fignum (1,1) double = NaN
options.displayname (1,:) char = ''
options.f_sym double = []
options.fsym double = []
options.syncFs (1,1) double = 0
options.shiftFs (1,1) double = 0
options.shifts double = []
options.maxOccurences (1,1) double = Inf
options.normalize (1,1) logical = false
options.debug_plots (1,1) logical = false
options.showPlot (1,1) logical = true
options.clear (1,1) logical = true
options.windowLength (1,1) double {mustBePositive, mustBeInteger} = 500
options.xLimits double = []
options.yLimits (1,2) double = [-3 3]
options.showStdAnnotations (1,1) logical = true
end
plot_shit = 1;
fsym = resolveSymbolRate(rxInput, refSymbols, options);
[symbols_for_lvl, avg_for_lvl, xAxisUs, info] = prepareLevelScatterData(rxInput, refSymbols, fsym, options);
if isa(eq_signal,'Signal')
options.f_sym = eq_signal.fs;
eq_signal = eq_signal.signal;
assert(~isempty(options.f_sym),'No fsym given');
if options.showPlot
plotLevelScatter(symbols_for_lvl, avg_for_lvl, refSymbols, xAxisUs, options);
end
if isa(ref_symbols,'Signal')
ref_symbols = ref_symbols.signal;
end
function fsym = resolveSymbolRate(rxInput, refSymbols, options)
if ~isempty(options.fsym)
fsym = options.fsym;
elseif ~isempty(options.f_sym)
fsym = options.f_sym;
elseif isa(refSymbols, "Signal") && ~isempty(refSymbols.fs)
fsym = refSymbols.fs;
elseif isa(rxInput, "Signal") && ~isempty(rxInput.fs)
fsym = rxInput.fs;
elseif iscell(rxInput) && ~isempty(rxInput) && isa(rxInput{1}, "Signal") && ~isempty(rxInput{1}.fs)
fsym = rxInput{1}.fs;
else
fsym = 1e6;
end
end
if plot_shit
% Determine the figure number to use or create a new figure
if isnan(options.fignum)
fig = figure; % Create a new figure and get its handle
function [symbols_for_lvl, avg_for_lvl, xAxisUs, info] = prepareLevelScatterData(rxInput, refSymbols, fsym, options)
refSignal = numericSignal(refSymbols);
info = defaultInfo(fsym);
if iscell(rxInput)
[symbols_for_lvl, avg_for_lvl, info] = prepareCellInput(rxInput, refSymbols, fsym, options, info);
elseif isa(rxInput, "Signal")
[symbols_for_lvl, avg_for_lvl, info] = prepareSignalInput(rxInput, refSymbols, fsym, options, info);
else
rxSymbols = numericSignal(rxInput);
[symbols_for_lvl, avg_for_lvl] = levelScatterForOneSequence(rxSymbols, refSignal, options.windowLength);
end
xAxisUs = ((1:size(avg_for_lvl, 2)) / fsym) * 1e6;
end
function info = defaultInfo(fsym)
info = struct();
info.found_sync = true;
info.startSamples = 1;
info.shifts = [];
info.fsym = fsym;
info.shiftFs = fsym;
info.varianceByLevel = [];
end
function [symbols_for_lvl, avg_for_lvl, info] = prepareSignalInput(rxSignal, refSymbols, fsym, options, info)
rxAtSymbolRate = rxSignal.resample("fs_in", rxSignal.fs, "fs_out", fsym);
refSignal = numericSignal(refSymbols);
if numel(rxAtSymbolRate.signal) == numel(refSignal)
rxSymbols = rxAtSymbolRate.signal;
if options.normalize
rxSymbols = normalizeNumericRms(rxSymbols);
end
[symbols_for_lvl, avg_for_lvl] = levelScatterForOneSequence(rxSymbols, refSignal, options.windowLength);
info.varianceByLevel = var(symbols_for_lvl, 0, 2, "omitnan");
return
end
[scopeCell, shifts, shiftFs, foundSync] = synchronizeRawSignal(rxSignal, refSymbols, fsym, options);
info.found_sync = foundSync;
info.shifts = shifts;
info.shiftFs = shiftFs;
if isempty(scopeCell)
symbols_for_lvl = [];
avg_for_lvl = [];
warning("showLevelScatter:NoScopeCells", ...
"No synchronized scope signal occurrences available.");
return
end
[symbols_for_lvl, avg_for_lvl, startSamples] = stitchScopeCells(scopeCell, refSymbols, fsym, shifts, shiftFs, options);
info.startSamples = startSamples;
info.varianceByLevel = var(symbols_for_lvl, 0, 2, "omitnan");
end
function [symbols_for_lvl, avg_for_lvl, info] = prepareCellInput(scopeCell, refSymbols, fsym, options, info)
scopeCell = scopeCell(:);
if isempty(scopeCell)
symbols_for_lvl = [];
avg_for_lvl = [];
info.found_sync = false;
warning("showLevelScatter:NoScopeCells", ...
"No synchronized scope signal occurrences available.");
return
end
shiftFs = options.shiftFs;
if shiftFs <= 0
shiftFs = fsym;
end
[symbols_for_lvl, avg_for_lvl, startSamples] = stitchScopeCells(scopeCell, refSymbols, fsym, options.shifts, shiftFs, options);
info.found_sync = true;
info.shifts = options.shifts;
info.shiftFs = shiftFs;
info.startSamples = startSamples;
info.varianceByLevel = var(symbols_for_lvl, 0, 2, "omitnan");
end
function [scopeCell, shifts, shiftFs, foundSync] = synchronizeRawSignal(rxSignal, refSymbols, fsym, options)
syncFs = options.syncFs;
if syncFs <= 0
syncFs = 2*fsym;
end
syncSignal = rxSignal.resample("fs_in", rxSignal.fs, "fs_out", syncFs);
if options.normalize
syncSignal = syncSignal.normalize("mode", "rms");
end
[~, scopeCell, ~, foundSync, shifts] = syncSignal.tsynch( ...
"reference", refSymbols, ...
"fs_ref", fsym, ...
"debug_plots", options.debug_plots);
if options.shiftFs > 0
shiftFs = options.shiftFs;
else
shiftFs = syncFs;
end
end
function [symbols_for_lvl, avg_for_lvl, startSamples] = stitchScopeCells(scopeCell, refSymbols, fsym, shifts, shiftFs, options)
recordOccurrences = min(numel(scopeCell), options.maxOccurences);
scopeCell = scopeCell(1:recordOccurrences);
refSignal = numericSignal(refSymbols);
startSamples = getStartSamples(shifts, recordOccurrences, shiftFs, fsym, numel(refSignal));
levelScatter = cell(1, recordOccurrences);
levelAverage = cell(1, recordOccurrences);
for occurrenceIdx = 1:recordOccurrences
occurrence = scopeCell{occurrenceIdx};
if isa(occurrence, "Signal")
occurrence = occurrence.resample("fs_out", fsym);
occurrence = occurrence.signal;
end
[levelScatter{occurrenceIdx}, levelAverage{occurrenceIdx}] = ...
levelScatterForOneSequence(occurrence, refSignal, options.windowLength);
end
numLevels = size(levelScatter{1}, 1);
traceLength = max(startSamples(:).' + cellfun(@(x) size(x, 2), levelScatter) - 1);
symbols_for_lvl = NaN(numLevels, traceLength);
avg_for_lvl = NaN(numLevels, traceLength);
for occurrenceIdx = 1:recordOccurrences
writeIdx = startSamples(occurrenceIdx):(startSamples(occurrenceIdx) + size(levelScatter{occurrenceIdx}, 2) - 1);
symbols_for_lvl(:, writeIdx) = levelScatter{occurrenceIdx};
avg_for_lvl(:, writeIdx) = levelAverage{occurrenceIdx};
end
end
function startSamples = getStartSamples(shifts, recordOccurrences, shiftFs, fsym, symbolLength)
if isempty(shifts)
startSamples = ((0:recordOccurrences-1) .* symbolLength) + 1;
return
end
shifts = shifts(:);
if numel(shifts) ~= recordOccurrences
positiveShifts = shifts(shifts >= 0);
if numel(positiveShifts) >= recordOccurrences
shifts = positiveShifts(1:recordOccurrences);
else
fig = figure(options.fignum); % Use the specified figure number
warning("showLevelScatter:ShiftCountMismatch", ...
"Shift count does not match scope cell count. Using sequential stitching.");
startSamples = ((0:recordOccurrences-1) .* symbolLength) + 1;
return
end
end
startSamples = round((shifts(1:recordOccurrences) - shifts(1)) ./ shiftFs .* fsym) + 1;
end
function [symbols_for_lvl, avg_for_lvl] = levelScatterForOneSequence(rxSymbols, refSymbols, windowLength)
rxSymbols = numericSignal(rxSymbols);
refSymbols = numericSignal(refSymbols);
assert(numel(rxSymbols) == numel(refSymbols), ...
'showLevelScatter:LengthMismatch', ...
'rxInput and refSymbols must have the same number of samples after resampling/synchronization.');
levels = unique(refSymbols);
[symbols_for_lvl, levels] = splitByReferenceLevels(rxSymbols, refSymbols, levels);
avg_for_lvl = NaN(numel(levels), numel(refSymbols));
for levelIdx = 1:numel(levels)
levelMask = ~isnan(symbols_for_lvl(levelIdx, :));
levelSamples = symbols_for_lvl(levelIdx, levelMask);
if isempty(levelSamples)
continue
end
smoothWindowLength = min(windowLength, numel(levelSamples));
avg_for_lvl(levelIdx, levelMask) = movmean(levelSamples, smoothWindowLength, 'Endpoints', 'shrink');
avg_for_lvl(levelIdx, :) = interpolateMissingLevelAverage(avg_for_lvl(levelIdx, :));
end
end
function [symbols_for_lvl, levels] = splitByReferenceLevels(rxSymbols, refSymbols, levels)
supportedPamLevels = [2 4 6 8 16];
if ismember(numel(levels), supportedPamLevels)
[symbols_for_lvl, levels] = PAMmapper(numel(levels), 0).splitByReferenceLevels( ...
rxSymbols, refSymbols, ...
"levels", levels);
else
symbols_for_lvl = NaN(numel(levels), numel(refSymbols));
for levelIdx = 1:numel(levels)
levelMask = refSymbols == levels(levelIdx);
symbols_for_lvl(levelIdx, levelMask) = rxSymbols(levelMask);
end
end
end
function plotLevelScatter(symbols_for_lvl, avg_for_lvl, refSymbols, xAxisUs, options)
if isempty(symbols_for_lvl)
return
end
if isnan(options.fignum)
figure;
else
figure(options.fignum);
if options.clear
clf;
end
end
hold on
numLevels = size(symbols_for_lvl, 1);
cols = cbrewer2("Paired", 2*numLevels);
rx_symbols = eq_signal; %./ rms(eq_signal);
correct_symbols = ref_symbols;
f_sym = options.f_sym;
col = cbrewer2('Paired',numel(unique(correct_symbols))*2);
ccnt = -1;
levels = unique(correct_symbols);
symbols_for_lvl = NaN(numel(levels),length(correct_symbols));
start = 1;
ende = length(correct_symbols);
for l = 1:numel(levels)
ccnt = ccnt+2;
level_amplitude = levels(l);
symbols_for_lvl(l,correct_symbols==level_amplitude) = rx_symbols(correct_symbols==level_amplitude);
std_lvl(l) = std(symbols_for_lvl(l,:),'omitnan');
xax_in_sec = ((1:length(correct_symbols)) / f_sym) * 1e6;
if plot_shit
scatter(xax_in_sec(start:ende),symbols_for_lvl(l,start:ende),10,'.','MarkerFaceAlpha',0.5,'MarkerEdgeAlpha',0.5,'MarkerEdgeColor',col(ccnt,:));
hold on;
end
for levelIdx = 1:numLevels
scatter(xAxisUs, symbols_for_lvl(levelIdx, :), 10, ".", ...
"MarkerEdgeColor", cols((2*levelIdx)-1, :), ...
"MarkerEdgeAlpha", 0.5);
end
std_lvl = round(std_lvl,2);
ccnt = 0;
avg_for_lvl = NaN(numel(levels),length(correct_symbols));
% Add the windowed/ smoothed curves
for l = 1:numel(levels)
ccnt = ccnt+2;
level_amplitude = levels(l);
L = 500;
movmean = 1/L .* movsum(rx_symbols(correct_symbols==level_amplitude),[L/2,L/2], 'Endpoints', 'fill');
avg_for_lvl(l,correct_symbols==level_amplitude) = movmean;
nanx = isnan(avg_for_lvl(l,:));
t = 1:numel(avg_for_lvl(l,:));
avg_for_lvl(l,nanx) = interp1(t(~nanx), avg_for_lvl(l,~nanx), t(nanx));
xax_in_sec = ((1:length(correct_symbols)) / f_sym) * 1e6;
% xax_in_sec = 1:length(correct_symbols);
if plot_shit
plot(xax_in_sec(start:ende),avg_for_lvl(l,start:ende),'Color',col(ccnt,:));
end
hold on
for levelIdx = 1:numLevels
plot(xAxisUs, avg_for_lvl(levelIdx, :), ...
"LineWidth", 1, ...
"Color", cols(2*levelIdx, :));
end
if 0
annotation(fig,'textbox',...
[0.660523809523809 0.844444444444448 0.133523809523809 0.0603174603174607],...
'String',['\sigma = ',num2str(std_lvl(4))],...
'LineWidth',1.8,...
'LineStyle','none',...
'FontSize',12,...
'FitBoxToText','off');
% Create textbox
annotation(fig,'textbox',...
[0.667666666666665 0.642857142857147 0.133523809523809 0.0603174603174607],...
'String',['\sigma = ',num2str(std_lvl(3))],...
'LineWidth',1.8,...
'LineStyle','none',...
'FontSize',12,...
'FitBoxToText','off');
% Create textbox
annotation(fig,'textbox',...
[0.671238095238093 0.442857142857148 0.133523809523809 0.0603174603174608],...
'String',['\sigma = ',num2str(std_lvl(2))],...
'LineWidth',1.8,...
'LineStyle','none',...
'FontSize',12,...
'FitBoxToText','off');
% Create textbox
annotation(fig,'textbox',...
[0.670047619047616 0.265079365079371 0.133523809523809 0.0603174603174608],...
'String',['\sigma = ',num2str(std_lvl(1))],...
'LineWidth',1.8,...
'LineStyle','none',...
'FontSize',12,...
'FitBoxToText','off');
refValues = numericSignal(refSymbols);
yline(unique(refValues), "HandleVisibility", "off");
if options.showStdAnnotations
annotateLevelStd(symbols_for_lvl, avg_for_lvl, xAxisUs, options.xLimits);
end
if plot_shit
% yline(levels);
xlabel('Time in $\mu$s');
ylabel('Normalized Amplitude');
ylim([-3 3]);
xlim(getXLimits(options.xLimits, xAxisUs, symbols_for_lvl));
ylim(options.yLimits);
grid on
drawnow;
end
function xLimits = getXLimits(configuredLimits, xAxisUs, symbols_for_lvl)
if ~isempty(configuredLimits)
xLimits = configuredLimits;
return
end
filledColumns = any(~isnan(symbols_for_lvl), 1);
if ~any(filledColumns)
xLimits = [xAxisUs(1), xAxisUs(end)];
return
end
lastFilledColumn = find(filledColumns, 1, "last");
xMax = xAxisUs(lastFilledColumn);
xLimits = [0, 1.05*xMax];
end
function annotateLevelStd(symbols_for_lvl, avg_for_lvl, xAxisUs, configuredXLimits)
xLimits = getXLimits(configuredXLimits, xAxisUs, symbols_for_lvl);
xText = xLimits(1) + 0.96*diff(xLimits);
for levelIdx = 1:size(symbols_for_lvl, 1)
levelSamples = symbols_for_lvl(levelIdx, :);
levelStd = std(levelSamples, 0, 2, 'omitnan');
if isnan(levelStd)
continue
end
levelAverage = avg_for_lvl(levelIdx, :);
yText = median(levelAverage(~isnan(levelAverage)), 'omitnan');
if isnan(yText)
yText = median(levelSamples(~isnan(levelSamples)), 'omitnan');
end
label = ['std = ', sprintf('%.3f', levelStd)];
text(xText, yText, label, ...
'Interpreter', 'none', ...
'HorizontalAlignment', 'right', ...
'VerticalAlignment', 'middle', ...
'FontSize', 10, ...
'BackgroundColor', 'w', ...
'Margin', 2, ...
'EdgeColor', [0.8 0.8 0.8]);
end
end
function levelAverage = interpolateMissingLevelAverage(levelAverage)
validSamples = ~isnan(levelAverage);
if nnz(validSamples) == 0
return
elseif nnz(validSamples) == 1
levelAverage(:) = levelAverage(validSamples);
return
end
t = 1:numel(levelAverage);
levelAverage(~validSamples) = interp1(t(validSamples), levelAverage(validSamples), ...
t(~validSamples), 'linear', 'extrap');
end
function values = numericSignal(signalLike)
if isa(signalLike, "Signal")
values = signalLike.signal;
else
values = signalLike;
end
values = values(:).';
end
function values = normalizeNumericRms(values)
values = values ./ sqrt(mean(values.^2, "omitnan"));
end

40
Functions/EQ_visuals/showMpiLevelScatter.m Normal file
View File

@@ -0,0 +1,40 @@
function [sep_sig, avg_sig, info] = showMpiLevelScatter(scopeInput, Symbols, options)
%SHOWMPILEVELSCATTER Backward-compatible wrapper around showLevelScatter.
% Prefer showLevelScatter directly for new code. This wrapper keeps older
% MPI call sites working while sharing one plotting/synchronization path.
arguments
scopeInput
Symbols
options.fsym double = []
options.syncFs (1,1) double = 0
options.shiftFs (1,1) double = 0
options.shifts double = []
options.maxOccurences (1,1) double = Inf
options.normalize (1,1) logical = true
options.debug_plots (1,1) logical = false
options.fignum (1,1) double = NaN
options.clear (1,1) logical = true
options.showPlot (1,1) logical = true
options.xLimits double = []
options.yLimits (1,2) double = [-3 3]
options.showStdAnnotations (1,1) logical = true
options.windowLength (1,1) double {mustBePositive, mustBeInteger} = 500
end
[sep_sig, avg_sig, info] = showLevelScatter(scopeInput, Symbols, ...
"fsym", options.fsym, ...
"syncFs", options.syncFs, ...
"shiftFs", options.shiftFs, ...
"shifts", options.shifts, ...
"maxOccurences", options.maxOccurences, ...
"normalize", options.normalize, ...
"debug_plots", options.debug_plots, ...
"fignum", options.fignum, ...
"clear", options.clear, ...
"showPlot", options.showPlot, ...
"xLimits", options.xLimits, ...
"yLimits", options.yLimits, ...
"showStdAnnotations", options.showStdAnnotations, ...
"windowLength", options.windowLength);
end

5
Functions/EQ_blocks/dsp_runid.m → Functions/Job_Processing/dsp_runid.m
View File

@@ -4,6 +4,7 @@ arguments
run_id
options.append_to_db = 0;
options.max_occurences = 4;
options.start_occurence = 1;
options.userParameters = struct();
options.database_type
options.dataBase
@@ -36,8 +37,8 @@ try
dspInput = loadDspInputFromFilePaths(run_id, options);
end
options.max_occurences = min(options.max_occurences, length(dspInput.Scpe_cell));
for r = 1:options.max_occurences
num_occurences = length(dspInput.Scpe_cell);
for r = 1:num_occurences
%%%%%%%% CORE EQUALIZATION CALL (Scpe, Symbols, Bits, 'Options') %%%%%%%

196
Functions/Job_Processing/loadAndSyncRunSignals.m Normal file
View File

@@ -0,0 +1,196 @@
function [Bits, Symbols, Scpe_cell, found_sync] = loadAndSyncRunSignals(dataTable, options)
%LOADANDSYNCRUNSIGNALS Load and synchronize signal files for one run.
%
% Inputs:
% dataTable - one-row table with run metadata and signal file paths
% options - struct with storage_path, start_occurence and max_occurences
%
% Outputs:
% Bits - transmitted bit reference
% Symbols - transmitted symbol reference
% Scpe_cell - synchronized received signal occurrences
% found_sync - true when a valid synchronization was found
found_sync = 0;
tempLocalStorage = 1;
Scpe_cell = {};
loaded_from_cache = false;
storage_dir = fullfile(prefdir, 'temp_sync_data');
if tempLocalStorage == 1
local_filename = fullfile(storage_dir, sprintf('sync_data_run_%s.mat', num2str(dataTable.run_id)));
if exist(local_filename, 'file')
try
cacheData = load(local_filename, 'Bits', 'Symbols', 'Scpe_cell');
if isValidSyncCache(cacheData)
Bits = cacheData.Bits;
Symbols = cacheData.Symbols;
Scpe_cell = cacheData.Scpe_cell;
found_sync = 1;
loaded_from_cache = true;
else
warning('loadAndSyncRunSignals:InvalidSyncCache', ...
'Ignoring incomplete sync cache file "%s".', local_filename);
safeDelete(local_filename);
end
catch
safeDelete(local_filename);
found_sync = 0;
end
end
end
if ~found_sync
Bits = load(composeStoragePath(options.storage_path, dataTable.tx_bits_path));
Bits = Bits.Bits;
M = double(dataTable.pam_level);
fsym = dataTable.symbolrate;
Symbols_mapped = PAMmapper(M,0).map(Bits);
Symbols_mapped.fs = fsym;
Symbols = load(composeStoragePath(options.storage_path, dataTable.tx_symbols_path));
Symbols = Symbols.Symbols;
found_sync = 0;
try
Scpe_load = load(composeStoragePath(options.storage_path, dataTable.rx_sync_path));
Scpe_cell = Scpe_load.S;
[~,~,~,found_sync] = Scpe_cell{1}.tsynch("reference", Symbols, ...
"fs_ref", fsym, ...
"debug_plots", 0);
catch
% Continue with raw data if pre-synchronized data is unavailable.
end
end
if ~found_sync
try
Scpe_sig_raw = load(composeStoragePath(options.storage_path, dataTable.rx_raw_path));
Scpe_sig_raw = Scpe_sig_raw.Scpe_sig_raw;
Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in", Scpe_sig_raw.fs, "fs_out", 2*fsym);
[~, Scpe_cell, ~, found_sync] = Scpe_sig_resampled.tsynch("reference", Symbols, ...
"fs_ref", fsym, ...
"debug_plots", 0);
catch
% Continue to mapped-symbol fallback if raw data sync fails.
end
end
if ~found_sync && exist('Scpe_sig_raw', 'var')
if length(Symbols_mapped.signal) ~= sum(Symbols_mapped.signal == Symbols.signal)
[~, Scpe_cell, ~, found_sync] = Scpe_sig_raw.tsynch("reference", Symbols_mapped, ...
"fs_ref", fsym, ...
"debug_plots", 0);
end
end
if tempLocalStorage == 1 && found_sync && ~loaded_from_cache
if ~exist(storage_dir, 'dir')
mkdir(storage_dir);
end
max_local_files = 10;
files = dir(fullfile(storage_dir, 'sync_data_run_*.mat'));
if length(files) >= max_local_files
[~, fileOrder] = sort([files.datenum]);
delete(fullfile(storage_dir, files(fileOrder(1)).name));
end
writeSyncCache(local_filename, Bits, Symbols, Scpe_cell);
end
if found_sync
Scpe_cell = selectSyncedOccurrences(Scpe_cell, options);
else
warning('Could not synchronize the received signal with the stored symbols!');
end
end
function path = composeStoragePath(storagePath, relativePath)
relativePath = firstValue(relativePath);
path = char(string(storagePath) + string(relativePath));
end
function value = firstValue(value)
if iscell(value)
value = value{1};
elseif ~ischar(value) && ~isscalar(value)
value = value(1);
end
end
function Scpe_cell = selectSyncedOccurrences(Scpe_cell, options)
available_occurences = length(Scpe_cell);
start_occurence = floor(getOption(options, 'start_occurence', 1));
max_occurences = floor(getOption(options, 'max_occurences', available_occurences));
if available_occurences < 1
warning('loadAndSyncRunSignals:NoSyncedOccurrences', ...
'Synchronization reported success, but no synced occurrences are available.');
return
end
if start_occurence < 1
error('loadAndSyncRunSignals:InvalidStartOccurrence', ...
'start_occurence must be >= 1.');
end
if max_occurences < 1
Scpe_cell = Scpe_cell(1:0);
return
end
if start_occurence > available_occurences
warning('loadAndSyncRunSignals:StartOccurrenceTooHigh', ...
['Requested start_occurence %d, but only %d synced occurrences are available. ', ...
'Processing the last occurrence only.'], ...
start_occurence, available_occurences);
Scpe_cell = Scpe_cell(available_occurences);
return
end
stop_occurence = min(available_occurences, start_occurence + max_occurences - 1);
Scpe_cell = Scpe_cell(start_occurence:stop_occurence);
end
function value = getOption(options, name, defaultValue)
if isfield(options, name)
value = options.(name);
else
value = defaultValue;
end
end
function valid = isValidSyncCache(cacheData)
valid = isfield(cacheData, 'Bits') && ...
isfield(cacheData, 'Symbols') && ...
isfield(cacheData, 'Scpe_cell') && ...
iscell(cacheData.Scpe_cell) && ...
~isempty(cacheData.Scpe_cell);
end
function writeSyncCache(local_filename, Bits, Symbols, Scpe_cell)
cache_dir = fileparts(local_filename);
temp_filename = [tempname(cache_dir), '.mat'];
try
save(temp_filename, 'Bits', 'Symbols', 'Scpe_cell');
movefile(temp_filename, local_filename, 'f');
catch ME
safeDelete(temp_filename);
rethrow(ME);
end
end
function safeDelete(filename)
if exist(filename, 'file')
try
delete(filename);
catch
% Another parallel worker may already have removed or replaced it.
end
end
end

114
Functions/Job_Processing/loadAndSyncSignalDataFromDb.m
View File

@@ -1,114 +0,0 @@
function [Bits, Symbols, Scpe_cell, found_sync] = loadAndSyncSignalDataFromDb(dataTable, options)
% LOADSIGNALDATA Loads and synchronizes signal data from storage
%
% Inputs:d
% dataTable - Table with file paths and configuration
% options - Struct with storage_path and max_occurences
%
% Outputs:
% Symbols_mapped - Mapped symbols from bits
% Symbols - Original symbols
% Scpe_cell - Cell array of synchronized signals
% found_sync - Boolean indicating if synchronization was successful
found_sync = 0;
tempLocalStorage = 1;
% Define the fixed storage directory relative to the user's MATLAB preferences directory
storage_dir = fullfile(prefdir, 'temp_sync_data');
% Part A: Check and load from local storage if available-
if tempLocalStorage == 1
local_filename = fullfile(storage_dir, sprintf('sync_data_run_%s.mat', num2str(dataTable.run_id)));
if exist(local_filename, 'file')
% Load from local storage and return
try
load(local_filename, 'Bits', 'Symbols', 'Scpe_cell');
found_sync = 1;
return
catch
delete(local_filename);
end
end
end
% If not locally saved, load from storage
if ~found_sync
% Load transmitted bits
Bits = load(fullfile([options.storage_path, char(dataTable.tx_bits_path)]));
Bits = Bits.Bits;
% Map bits to symbols
M = double(dataTable.pam_level);
fsym = dataTable.symbolrate;
Symbols_mapped = PAMmapper(M,0).map(Bits);
Symbols_mapped.fs = fsym;
% Load original symbols
Symbols = load(fullfile([options.storage_path, char(dataTable.tx_symbols_path)]));
Symbols = Symbols.Symbols;
found_sync = 0;
Scpe_cell = {};
% Try to load pre-synchronized data
try
Scpe_load = load(fullfile([options.storage_path, char(dataTable.rx_sync_path)]));
Scpe_cell = Scpe_load.S;
[~,~,~,found_sync] = Scpe_cell{2}.tsynch("reference", Symbols, "fs_ref", fsym, "debug_plots", 1);
catch
% Continue to next method if this fails
end
end
% If not found, try with raw data
if ~found_sync
try
Scpe_sig_raw = load([options.storage_path, char(dataTable.rx_raw_path(1))]);
Scpe_sig_raw = Scpe_sig_raw.Scpe_sig_raw;
Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in", Scpe_sig_raw.fs, "fs_out", 2*fsym);
[~, Scpe_cell, ~, found_sync] = Scpe_sig_resampled.tsynch("reference", Symbols, "fs_ref", fsym, "debug_plots", 1);
catch
% Continue to next method if this fails
end
end
% Last attempt with mapped symbols
if ~found_sync && exist('Scpe_sig_raw', 'var')
if length(Symbols_mapped.signal) ~= sum(Symbols_mapped.signal == Symbols.signal)
[~, Scpe_cell, ~, found_sync] = Scpe_sig_raw.tsynch("reference", Symbols_mapped, "fs_ref", fsym, "debug_plots", 0);
end
end
% Part B: Save to local storage if data was loaded and synced
if tempLocalStorage == 1 && found_sync
% Create directory if it doesn't exist
if ~exist(storage_dir, 'dir')
mkdir(storage_dir);
end
% local_filename = fullfile(storage_dir, sprintf('sync_data_run_%s.mat', num2str(dataTable.run_id)));
% List existing files and remove oldest if more than N
max_local_files = 10; % Store up to N files
files = dir(fullfile(storage_dir, 'sync_data_run_*.mat'));
if length(files) >= max_local_files
% Sort by date
[~, idx] = sort([files.datenum]);
% Delete oldest file
delete(fullfile(storage_dir, files(idx(1)).name));
end
% Save current data
save(local_filename, 'Bits', 'Symbols', 'Scpe_cell');
end
% Limit number of occurrences
if found_sync
record_realizations = min(options.max_occurences, length(Scpe_cell));
Scpe_cell = Scpe_cell(1:record_realizations);
else
warning('Could not synchronize the received signal with the stored symbols!');
end
end

97
Functions/Job_Processing/loadDspInputFromFilePaths.m Normal file
View File

@@ -0,0 +1,97 @@
function dspInput = loadDspInputFromFilePaths(run_id, options)
%LOADDSPINPUTFROMFILEPATHS Load explicit signal files and prepare DSP input.
arguments
run_id
options struct
end
Tx_bits = load(textScalar(options.load_file_path.tx_bits_path));
Symbols = load(textScalar(options.load_file_path.tx_symbols_path));
Scpe_sig_raw = load(textScalar(options.load_file_path.rx_raw_path));
Tx_bits = Tx_bits.Bits;
Symbols = Symbols.Symbols;
Scpe_sig_raw = Scpe_sig_raw.Scpe_sig_raw;
fsym = Symbols.fs;
M = Symbols.logbook.ModifierCopy{1}.M;
duob_mode = Symbols.logbook.ModifierCopy{1}.duobinary_mode;
Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in", Scpe_sig_raw.fs, "fs_out", 2*fsym);
[~, Scpe_cell, ~, found_sync] = Scpe_sig_resampled.tsynch("reference", Symbols, ...
"fs_ref", fsym, ...
"debug_plots", 1);
if found_sync
Scpe_cell = selectSyncedOccurrences(Scpe_cell, options);
else
warning('Could not synchronize the received signal with the stored symbols!');
end
dspInput = struct();
dspInput.run_id = run_id;
dspInput.dataTable = table();
dspInput.Tx_bits = Tx_bits;
dspInput.Symbols = Symbols;
dspInput.Scpe_cell = Scpe_cell;
dspInput.found_sync = found_sync;
dspInput.fsym = fsym;
dspInput.M = M;
dspInput.duob_mode = duob_mode;
end
function value = textScalar(value)
value = firstValue(value);
value = char(string(value));
end
function value = firstValue(value)
if iscell(value)
value = value{1};
elseif ~ischar(value) && ~isscalar(value)
value = value(1);
end
end
function Scpe_cell = selectSyncedOccurrences(Scpe_cell, options)
available_occurences = length(Scpe_cell);
start_occurence = floor(getOption(options, 'start_occurence', 1));
max_occurences = floor(getOption(options, 'max_occurences', available_occurences));
if available_occurences < 1
warning('loadDspInputFromFilePaths:NoSyncedOccurrences', ...
'Synchronization reported success, but no synced occurrences are available.');
return
end
if start_occurence < 1
error('loadDspInputFromFilePaths:InvalidStartOccurrence', ...
'start_occurence must be >= 1.');
end
if max_occurences < 1
Scpe_cell = Scpe_cell(1:0);
return
end
if start_occurence > available_occurences
warning('loadDspInputFromFilePaths:StartOccurrenceTooHigh', ...
['Requested start_occurence %d, but only %d synced occurrences are available. ', ...
'Processing the last occurrence only.'], ...
start_occurence, available_occurences);
Scpe_cell = Scpe_cell(available_occurences);
return
end
stop_occurence = min(available_occurences, start_occurence + max_occurences - 1);
Scpe_cell = Scpe_cell(start_occurence:stop_occurence);
end
function value = getOption(options, name, defaultValue)
if isfield(options, name)
value = options.(name);
else
value = defaultValue;
end
end

40
Functions/Job_Processing/loadDspInputFromRunId.m Normal file
View File

@@ -0,0 +1,40 @@
function dspInput = loadDspInputFromRunId(run_id, database, options)
%LOADDSPINPUTFROMRUNID Query run metadata and prepare canonical DSP input.
arguments
run_id
database
options struct
end
dataTable = queryRunid(run_id, database);
% Load signal files referenced by the run metadata, verify/synchronize the
% received signal, optionally cache the sync result, and cap occurrences.
[Tx_bits, Symbols, Scpe_cell, found_sync] = loadAndSyncRunSignals(dataTable, options);
dspInput = struct();
dspInput.run_id = run_id;
dspInput.dataTable = dataTable;
dspInput.Tx_bits = Tx_bits;
dspInput.Symbols = Symbols;
dspInput.Scpe_cell = Scpe_cell;
dspInput.found_sync = found_sync;
dspInput.fsym = dataTable.symbolrate;
dspInput.M = double(dataTable.pam_level);
dspInput.duob_mode = parseDbMode(dataTable.db_mode);
end
function mode = parseDbMode(rawMode)
if isnumeric(rawMode)
mode = db_mode(rawMode);
return
end
if iscell(rawMode)
rawMode = rawMode{1};
end
mode = db_mode(strrep(string(rawMode), '"', ''));
end

36
Functions/Job_Processing/preprocessRunId.m
View File

@@ -1,36 +0,0 @@
function dspInput = preprocessRunId(run_id, database, options)
%PREPROCESSRUNID Load one run_id and prepare the canonical DSP input.
arguments
run_id
database
options struct
end
dataTable = queryRunid(run_id, database);
[Tx_bits, Symbols, Scpe_cell, found_sync] = loadAndSyncSignalDataFromDb(dataTable, options);
dspInput = struct();
dspInput.run_id = run_id;
dspInput.dataTable = dataTable;
dspInput.Tx_bits = Tx_bits;
dspInput.Symbols = Symbols;
dspInput.Scpe_cell = Scpe_cell;
dspInput.found_sync = found_sync;
dspInput.fsym = dataTable.symbolrate;
dspInput.M = double(dataTable.pam_level);
dspInput.duob_mode = parseDbMode(dataTable.db_mode);
end
function mode = parseDbMode(rawMode)
if isnumeric(rawMode)
mode = db_mode(rawMode);
return
end
if iscell(rawMode)
rawMode = rawMode{1};
end
mode = db_mode(strrep(string(rawMode), '"', ''));
end

80
Functions/Job_Processing/runBatch.m
View File

@@ -1,6 +1,6 @@
function results = runBatch(workerFcn, jobs, options)
function batchResults = runBatch(workerFcn, jobs, options)
%RUNBATCH Execute a list of jobs in serial or parallel.
% results = runBatch(workerFcn, jobs) executes each jobs(i).args cell
% batchResults = runBatch(workerFcn, jobs) executes each jobs(i).args cell
% with workerFcn and returns a cell array aligned with the input jobs.
%
% Supported job fields:
@@ -24,7 +24,7 @@ function results = runBatch(workerFcn, jobs, options)
mode = normalizeProcessingMode(options.mode);
jobs = normalizeJobs(jobs);
nJobs = numel(jobs);
results = cell(1, nJobs);
batchResults = cell(1, nJobs);
h = [];
if nJobs == 0
@@ -33,7 +33,7 @@ function results = runBatch(workerFcn, jobs, options)
if options.waitbar
h = waitbar(0, char(options.waitbarMessage));
cleanupWaitbar = onCleanup(@() closeWaitbar(h)); %#ok<NASGU>
cleanupWaitbar = onCleanup(@() closeWaitbar(h));
end
switch mode
@@ -41,36 +41,36 @@ function results = runBatch(workerFcn, jobs, options)
pool = setupParallelPool(options.numWorkers, options.idleTimeout, options.cancelExistingQueue);
futures = parallel.FevalFuture.empty(nJobs, 0);
for idx = 1:nJobs
fprintf('[%s] Submitted.\n', jobs(idx).label);
futures(idx) = parfeval(pool, workerFcn, 1, jobs(idx).args{:});
for jobIdx = 1:nJobs
fprintf('[%s] Submitted.\n', jobs(jobIdx).label);
futures(jobIdx) = parfeval(pool, workerFcn, 1, jobs(jobIdx).args{:});
end
consumedIdx = false(nJobs, 1);
for completedCount = 1:nJobs
try
[idx, value] = fetchNext(futures);
consumedIdx(idx) = true;
results{idx} = value;
fprintf('[%s] Completed (%d/%d).\n', jobs(idx).label, completedCount, nJobs);
[jobIdx, jobResult] = fetchNext(futures);
consumedIdx(jobIdx) = true;
batchResults{jobIdx} = jobResult;
fprintf('[%s] Completed (%d/%d).\n', jobs(jobIdx).label, completedCount, nJobs);
if ~isempty(options.resultHandler)
options.resultHandler(value, jobs(idx), idx);
options.resultHandler(jobResult, jobs(jobIdx), jobIdx);
end
catch fetchErr
idxErr = findErroredFuture(futures, consumedIdx);
if isempty(idxErr)
errorJobIdx = findErroredFuture(futures, consumedIdx);
if isempty(errorJobIdx)
rethrow(fetchErr);
end
consumedIdx(idxErr) = true;
errInfo = extractFutureError(futures(idxErr));
results{idxErr} = errInfo;
consumedIdx(errorJobIdx) = true;
errInfo = extractFutureError(futures(errorJobIdx));
batchResults{errorJobIdx} = errInfo;
if ~isempty(options.errorHandler)
options.errorHandler(errInfo, jobs(idxErr), idxErr);
options.errorHandler(errInfo, jobs(errorJobIdx), errorJobIdx);
else
defaultErrorHandler(errInfo, jobs(idxErr), idxErr);
defaultErrorHandler(errInfo, jobs(errorJobIdx), errorJobIdx);
end
end
@@ -78,27 +78,27 @@ function results = runBatch(workerFcn, jobs, options)
end
case processingMode.serial
for idx = 1:nJobs
for jobIdx = 1:nJobs
try
fprintf('[%s] Running.\n', jobs(idx).label);
value = feval(workerFcn, jobs(idx).args{:});
results{idx} = value;
fprintf('[%s] Completed (%d/%d).\n', jobs(idx).label, idx, nJobs);
fprintf('[%s] Running.\n', jobs(jobIdx).label);
jobResult = workerFcn(jobs(jobIdx).args{:});
batchResults{jobIdx} = jobResult;
fprintf('[%s] Completed (%d/%d).\n', jobs(jobIdx).label, jobIdx, nJobs);
if ~isempty(options.resultHandler)
options.resultHandler(value, jobs(idx), idx);
options.resultHandler(jobResult, jobs(jobIdx), jobIdx);
end
catch ME
results{idx} = ME;
batchResults{jobIdx} = ME;
if ~isempty(options.errorHandler)
options.errorHandler(ME, jobs(idx), idx);
options.errorHandler(ME, jobs(jobIdx), jobIdx);
else
defaultErrorHandler(ME, jobs(idx), idx);
defaultErrorHandler(ME, jobs(jobIdx), jobIdx);
end
end
updateWaitbar(options.waitbar, h, idx, nJobs);
updateWaitbar(options.waitbar, h, jobIdx, nJobs);
end
otherwise
@@ -128,23 +128,23 @@ function jobs = normalizeJobs(jobs)
return
end
for idx = 1:numel(jobs)
if ~isfield(jobs, 'args') || isempty(jobs(idx).args)
jobs(idx).args = {};
for jobIdx = 1:numel(jobs)
if ~isfield(jobs, 'args') || isempty(jobs(jobIdx).args)
jobs(jobIdx).args = {};
end
if ~iscell(jobs(idx).args)
error('runBatch:InvalidArgs', 'jobs(%d).args must be a cell array.', idx);
if ~iscell(jobs(jobIdx).args)
error('runBatch:InvalidArgs', 'jobs(%d).args must be a cell array.', jobIdx);
end
if ~isfield(jobs, 'label') || isempty(jobs(idx).label)
jobs(idx).label = sprintf('Job %d', idx);
if ~isfield(jobs, 'label') || isempty(jobs(jobIdx).label)
jobs(jobIdx).label = sprintf('Job %d', jobIdx);
else
jobs(idx).label = string(jobs(idx).label);
jobs(jobIdx).label = string(jobs(jobIdx).label);
end
if ~isfield(jobs, 'meta')
jobs(idx).meta = struct();
jobs(jobIdx).meta = struct();
end
end
end
@@ -189,9 +189,9 @@ function pool = setupParallelPool(numWorkers, idleTimeout, cancelExistingQueue)
end
end
function idxErr = findErroredFuture(futures, consumedIdx)
function errorJobIdx = findErroredFuture(futures, consumedIdx)
readMask = arrayfun(@(future) future.Read, futures).';
idxErr = find(readMask & ~consumedIdx, 1);
errorJobIdx = find(readMask & ~consumedIdx, 1);
end
function errInfo = extractFutureError(future)

120
Functions/Job_Processing/submitJobs.m
View File

@@ -9,9 +9,11 @@ function [results, wh] = submitJobs(run_ids, dsp_options, submit_mode, submit_op
% submit_options : struct with fields
% .waitbar (logical)
% .wh (DataStorage object)
% .storePackages (string array, optional package whitelist)
%
% results : cell(nJobsPerRunId, nRunIds)
% wh : updated DataStorage
% wh : updated DataStorage. For multiple run_ids, run_id is added as
% the first storage axis.
arguments
run_ids int32 = 0
@@ -19,15 +21,19 @@ arguments
submit_mode processingMode = processingMode.serial
submit_options.waitbar (1,1) logical = true
submit_options.wh = DataStorage(struct())
submit_options.storePackages string = string.empty()
end
% Normalize
run_ids = run_ids(:)';
nRunIds = numel(run_ids);
nJobsPerRunId = submit_options.wh.getLastLinIndice();
sweepWh = submit_options.wh;
validateNoRunIdSweepParameter(sweepWh);
nJobsPerRunId = sweepWh.getLastLinIndice();
totalJobs = nRunIds * nJobsPerRunId;
wh = submit_options.wh;
wh = buildStorageWarehouse(sweepWh, run_ids);
results = cell(nJobsPerRunId, nRunIds);
jobs = repmat(struct('args', {{}}, 'label', "", 'meta', struct()), 1, totalJobs);
@@ -35,14 +41,16 @@ jobCounter = 0;
for r = 1:nRunIds
for k = 1:nJobsPerRunId
jobCounter = jobCounter + 1;
optionalVars = buildOptionalVars(k, submit_options.wh);
userParameters = buildUserParameters(k, sweepWh);
jobOptions = dsp_options;
jobOptions.parameters = optionalVars;
jobOptions.userParameters = userParameters;
jobs(jobCounter).args = [{run_ids(r)}, structToNameValue(jobOptions)];
jobs(jobCounter).label = sprintf('RunID %d, Job %d', run_ids(r), k);
jobs(jobCounter).meta.runIndex = r;
jobs(jobCounter).meta.jobIndex = k;
jobs(jobCounter).meta.sweepJobIndex = k;
jobs(jobCounter).meta.storageJobIndex = buildStorageJobIndex(run_ids(r), userParameters, wh);
jobs(jobCounter).meta.run_id = run_ids(r);
end
end
@@ -74,16 +82,69 @@ end
fprintf('Full report:\n%s\n', getReport(ME,'extended'));
end
function optionalVars = buildOptionalVars(jobIndex, wh)
optionalVars = struct();
function userParameters = buildUserParameters(jobIndex, wh)
userParameters = struct();
if ~isempty(wh.getDimension())
[vals, names] = wh.getPhysIndicesByLinIndex(jobIndex);
for pi = 1:numel(names)
optionalVars.(names{pi}) = vals{pi};
userParameters.(names{pi}) = vals{pi};
end
end
end
function validateNoRunIdSweepParameter(wh)
if isfield(wh.inputParams, "run_id")
error("submitJobs:RunIdParameterConflict", ...
"Do not define userParameters.run_id. submitJobs manages run_id as a storage axis.");
end
end
function storageWh = buildStorageWarehouse(sweepWh, runIds)
if isscalar(runIds)
storageWh = sweepWh;
return
end
storageParameters = struct();
storageParameters.run_id = runIds;
sweepParameterNames = fieldnames(sweepWh.inputParams);
for parameterIdx = 1:numel(sweepParameterNames)
parameterName = sweepParameterNames{parameterIdx};
storageParameters.(parameterName) = sweepWh.inputParams.(parameterName);
end
storageWh = DataStorage(storageParameters);
end
function storageJobIndex = buildStorageJobIndex(runId, userParameters, wh)
storageParameterNames = wh.fn;
if isempty(storageParameterNames)
storageJobIndex = 1;
return
end
storageSubscripts = cell(1, numel(storageParameterNames));
for parameterIdx = 1:numel(storageParameterNames)
parameterName = char(storageParameterNames(parameterIdx));
if strcmp(parameterName, "run_id")
parameterValue = runId;
else
parameterValue = userParameters.(parameterName);
end
storageSubscripts{parameterIdx} = wh.getIndexByPhys(parameterName, parameterValue);
end
if isscalar(storageSubscripts)
storageJobIndex = storageSubscripts{1};
else
storageJobIndex = sub2ind(wh.getStorageSize(), storageSubscripts{:});
end
end
function nameValue = structToNameValue(options)
names = fieldnames(options);
nameValue = cell(1, 2*numel(names));
@@ -95,14 +156,41 @@ end
end
function storeResult(val, job, ~)
if ~isempty(wh)
jobIndex = job.meta.jobIndex;
wh.addValueToStorageByLinIdx(val.ffe_package, 'ffe_package', jobIndex);
wh.addValueToStorageByLinIdx(val.mlse_package, 'mlse_package', jobIndex);
wh.addValueToStorageByLinIdx(val.vnle_package, 'vnle_package', jobIndex);
wh.addValueToStorageByLinIdx(val.dbtgt_package,'dbtgt_package',jobIndex);
wh.addValueToStorageByLinIdx(val.dbenc_package,'dbenc_package',jobIndex);
wh.addValueToStorageByLinIdx(val.mlmlse_package,'mlmlse_package',jobIndex);
if isempty(wh) || ~isstruct(val)
return
end
storageJobIndex = job.meta.storageJobIndex;
resultFields = fieldnames(val);
for resultIdx = 1:numel(resultFields)
storageName = resultFields{resultIdx};
if ~shouldStore(storageName)
continue
end
if isempty(val.(storageName))
continue
end
ensureStorage(storageName);
wh.addValueToStorageByLinIdx(val.(storageName), storageName, storageJobIndex);
end
end
function tf = shouldStore(storageName)
if isempty(submit_options.storePackages)
tf = true;
return
end
tf = any(string(storageName) == submit_options.storePackages);
end
function ensureStorage(storageName)
if ~isfield(wh.sto, storageName)
wh.addStorage(storageName);
end
end