silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

restructure and organize

Browse Source
This commit is contained in:
Silas Oettinghaus
2026-06-22 22:58:58 +02:00
parent c4f75b7ec4
commit 33ec5b3116
36 changed files with 1914 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
projects/Advanced_DSP_for_400G_IMDD_experiments/Auswertung_JLT/final/FIGURE_EYES.m
View File

@@ -31,7 +31,7 @@ M = double(dataTable.pam_level);
duob_mode = db_mode(strrep(dataTable.db_mode,'"',''));
% Load and Sync signal data from DB
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncSignalDataFromDb(dataTable, dsp_options);
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncRunSignals(dataTable, dsp_options);
Scpe_sig_syncd = Scpe_cell{1};
Scpe_sig_syncd.eye(fsym,M,"fignum",rate.*1e-9*M+1,"displayname",' Eye of Signal');
@@ -276,4 +276,4 @@ ax1.Position = pos1;
ax2.Box = 'on'; % keep outline but no ticks on the right
ax1.Box = 'on';
ax2.View = [90 -90];
ax2.View = [90 -90];

2
projects/Advanced_DSP_for_400G_IMDD_experiments/Auswertung_JLT/final/FIGURE_Spectra.m
View File

@@ -82,7 +82,7 @@ for dbmode = 0%length(rates)
M = double(dataTable.pam_level);
% Load and Sync signal data from DB
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncSignalDataFromDb(dataTable, dsp_options);
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncRunSignals(dataTable, dsp_options);
% Preprocess signal
Scpe_sig = preprocessSignal(Scpe_cell{1}, Symbols, fsym);

6
projects/Advanced_DSP_for_400G_IMDD_experiments/Auswertung_JLT/final/generate_spectrum_plots.m
View File

@@ -30,7 +30,7 @@ fsym = dataTable.symbolrate;
M = double(dataTable.pam_level);
% Load and Sync signal data from DB
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncSignalDataFromDb(dataTable, dsp_options);
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncRunSignals(dataTable, dsp_options);
% Preprocess signal
Scpe_sig = preprocessSignal(Scpe_cell{1}, Symbols, fsym);
@@ -62,7 +62,7 @@ fsym = dataTable.symbolrate;
M = double(dataTable.pam_level);
% Load and Sync signal data from DB
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncSignalDataFromDb(dataTable, dsp_options);
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncRunSignals(dataTable, dsp_options);
% Preprocess signal
Scpe_sig = preprocessSignal(Scpe_cell{1}, Symbols, fsym);
@@ -117,4 +117,4 @@ matlab2tikz(outfile, ...
'legend columns=1' ...
});
end
end

12
projects/Advanced_DSP_for_400G_IMDD_experiments/Auswertung_JLT/run_dsp_from_db.m
View File

@@ -3,10 +3,10 @@ dsp_options.append_to_db = 0;
dsp_options.max_occurences = 1;
experiment = "highspeed_2024";
dsp_options.mode = "load_run_id"; % 'simulate' & 'load_files'
dsp_options.mode = "run_id"; % "run_id", "file_paths", or future "simulate"
dsp_options.load_file_path = struct();
if dsp_options.mode == "load_run_id"
if dsp_options.mode == "run_id"
if experiment == "highspeed_2024"
@@ -41,7 +41,7 @@ if dsp_options.mode == "load_run_id"
end
elseif dsp_options.mode == "load_files"
elseif dsp_options.mode == "file_paths"
dsp_options.load_file_path.tx_bits_path = "Z:\2025\ECOC Silas\ecoc_2025\mpi_opti_1000m_pam2 4 6 8\20250417_091513_PAM_4_R_112_bits.mat"';
dsp_options.load_file_path.tx_symbols_path = "Z:\2025\ECOC Silas\ecoc_2025\mpi_opti_1000m_pam2 4 6 8\20250417_091513_PAM_4_R_112_symbols.mat"';
@@ -74,10 +74,10 @@ fp.where('Runs', 'rop_attenuation','EQUAL', 0);
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
% === Set LOOPS & Initialize DataStorage ===
dsp_options.parameters = struct();
% dsp_options.parameters.pf_ncoeffs = [1,2];%s[0,logspace(-4,0,10)];
dsp_options.userParameters = struct();
dsp_options.userParameters.pf_ncoeffs = [1,2];%s[0,logspace(-4,0,10)];
wh = DataStorage(dsp_options.parameters);
wh = DataStorage(dsp_options.userParameters);
wh.addStorage("ffe_package");
wh.addStorage("mlse_package");
wh.addStorage("vnle_package");

4
projects/Advanced_DSP_for_400G_IMDD_experiments/a_minimal_example.m
View File

@@ -34,7 +34,7 @@ for id = run_ids
duob_mode = db_mode(strrep(dataTable(1,:).db_mode,'"',''));
% Load and Sync signal data from DB
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncSignalDataFromDb(dataTable(1,:), dsp_options);
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncRunSignals(dataTable(1,:), dsp_options);
% Preprocess signal
Scpe_sig = preprocessSignal(Scpe_cell{1}, Symbols, fsym);
@@ -159,4 +159,4 @@ ml_mlse_equalizer = ML_MLSE("epochs_tr",training_epochs,"epochs_dd",1, ...
[ml_mlse_results] = ml_mlse(ml_mlse_equalizer, M, Scpe_sig, Symbols, Tx_bits,"precode_mode",duob_mode);
ml_mlse_results.metrics.print("description",'ML pre Eq. + Viterbi')
ml_mlse_results.metrics.print("description",'ML pre Eq. + Viterbi')

8
projects/Advanced_DSP_for_400G_IMDD_experiments/auswertung MPI/dsp_run_id.m
View File

@@ -4,7 +4,7 @@ arguments
run_id
options.append_to_db = 0;
options.max_occurences = 4;
options.parameters = struct();
options.userParameters = struct();
end
basePath = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor\';
@@ -46,8 +46,8 @@ mu_dc = 0.00;
mu_ffe = [mu_ffe1 mu_ffe2 mu_ffe3];
vnle_order=[vnle_order1,vnle_order2,vnle_order3];
% Overwrite default parameters if given in options.parameters
paramStruct = options.parameters;
% Overwrite default parameters if given in options.userParameters
paramStruct = options.userParameters;
if ~isempty(paramStruct)
paramNames = fieldnames(paramStruct);
for i = 1:numel(paramNames)
@@ -191,4 +191,4 @@ output.vnle_dfe_package = vnle_dfe_package;
output.vnle_pf_package = vnle_pf_package;
output.dbtgt_package = dbtgt_package;
end
end

76
projects/Diss/MPI_revisit/investigate_mpi_algorithms.m Normal file
View File

@@ -0,0 +1,76 @@
% === DSP settings ===
dsp_options = struct();
dsp_options.mode = "run_id";
dsp_options.recipe = @mpi_recipe_dev;
dsp_options.append_to_db = false;
dsp_options.max_occurences = 3;
dsp_options.start_occurence = 1;
dsp_options.debug_plots = true;
dsp_options.database_type = "mysql";
dsp_options.dataBase = "labor";
dsp_options.storage_path = "W:\labdata\ECOC Silas\ecoc_2025";
dsp_options.server = "192.168.178.192";
dsp_options.port = 3306;
dsp_options.user = "silas";
dsp_options.password = "silas";
db = DBHandler("dataBase", [dsp_options.dataBase],...
"type", dsp_options.database_type,...
"server", dsp_options.server,...
"user", dsp_options.user, "password", dsp_options.password);
%%
fp = QueryFilter();
fp.where('Runs', 'symbolrate', 'EQUALS', 112e9);
fp.where('Runs', 'fiber_length', 'EQUALS', 0);
fp.where('Runs', 'interference_path_length', 'EQUALS', 1000);
fp.where('Runs', 'sir', 'EQUALS', 23);
% fp.where('Runs', 'db_mode', 'EQUALS', '"no_db"');
% fp.where('Runs', 'is_mpi', 'EQUALS', 1);
fp.where('Runs', 'pam_level', 'EQUALS', 4);
fp.where('Runs', 'wavelength', 'EQUALS', 1310);
fp.where('Runs', 'v_bias', 'EQUALS', 2.65);
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
[~, uniqueSirRows] = unique(dataTable.sir, "stable");
dataTable = dataTable(uniqueSirRows, :);
% sort rows by sir (ascending) and extract run_ids
[~, sortIdx] = sort(dataTable.sir, 'descend');
dataTable = dataTable(sortIdx, :);
run_ids = dataTable.run_id;
if isempty(run_ids)
error("run_minimal_recipe:MissingRunIds", ...
"Set run_ids to one or more known run IDs before running this example.");
end
%% === Warehouse setup ===
dsp_options.userParameters = struct();
dsp_options.userParameters.smoothing_length = logspace(1,5.5,5);
wh = DataStorage(dsp_options.userParameters);
%% === Run ===
% submitJobs returns the raw per-job results and the filled Warehouse. For a
% single run_id and remove_dc = [0, 1], results is a 2-by-1 cell array.
[results, wh] = submitJobs(run_ids, dsp_options, processingMode.parallel, ...
"wh", wh, ...
"waitbar", true);
%% Analyze
storageNames = fieldnames(wh.sto);
x_vars = dsp_options.userParameters.smoothing_length;
figure(2026);hold on
for i = 1:numel(run_ids)
disp(run_ids(i))
result = wh.getStoValue(storageNames{1},x_vars);
ber = cellfun(@(packageCell) packageCell{1}.metrics.BER, result);
plot(x_vars,ber);
beautifyBERplot("logscale",true,"setcolors",true,"setmarkers",true);
end

9
projects/ECOC_2025_MPI/auswertung_algorithms/run_offline_dsp.m
View File

@@ -7,8 +7,8 @@
% dsp_options.database_name = 'silas_labor_newdsp_newstructure.db';
% dsp_options.storage_path = 'Z:\2024\sioe_labor\';
%
% dsp_options.parameters = struct();
% dsp_options.parameters.mu_dc = [0.005];
% dsp_options.userParameters = struct();
% dsp_options.userParameters.mu_dc = [0.005];
%
% % === Get Run ID's ===
% db = DBHandler("pathToDB", [dsp_options.database_path, dsp_options.database_name], "type", "sqlite");
@@ -21,6 +21,7 @@ dsp_options.storage_path = 'W:\labdata\ECOC Silas\ECOC Silas\ecoc_2025\';
dsp_options.server = "192.168.178.192";% "134.245.243.254";
dsp_options.user = "silas";
dsp_options.password = "silas";
dsp_options.userParameters = struct();
db = DBHandler("dataBase", [dsp_options.dataBase],...
"type", dsp_options.database_type,...
@@ -40,7 +41,7 @@ fp.where('Runs', 'bitrate','EQUALS', 360e9);
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
% === Initialize DataStorage ===
wh = DataStorage(dsp_options.parameters);
wh = DataStorage(dsp_options.userParameters);
wh.addStorage("ffe_package");
wh.addStorage("mlse_package");
wh.addStorage("vnle_package");
@@ -131,4 +132,4 @@ ylabel(y_var, 'Interpreter', 'none');
legend('show', 'Location', 'best');
grid on;
title(sprintf('%s vs. %s', y_var, x_var), 'Interpreter', 'none');
hold off;
hold off;

8
projects/ECOC_2025_MPI/dsp_run_id.m
View File

@@ -4,7 +4,7 @@ arguments
run_id
options.append_to_db = 0;
options.max_occurences = 4;
options.parameters = struct();
options.userParameters = struct();
options.database_path
options.database_name
@@ -60,8 +60,8 @@ ffe_buffer_len = 1;
smoothing_buffer_length = 4096;
smoothing_buffer_update = 224;
% Overwrite default parameters if given in options.parameters
paramStruct = options.parameters;
% Overwrite default parameters if given in options.userParameters
paramStruct = options.userParameters;
if ~isempty(paramStruct)
paramNames = fieldnames(paramStruct);
for i = 1:numel(paramNames)
@@ -261,4 +261,4 @@ output.vnle_pf_package = vnle_pf_package;
output.vnle_package = vnle_package;
output.dbtgt_package = dbtgt_package;
end
end

8
projects/ECOC_2025_MPI/dsp_standalone.m
View File

@@ -52,7 +52,7 @@ params.mu_dc = 0.005;
futures_list = parallel.FevalFuture.empty();
for id = 1:length(dataTable.run_id)
run_id = dataTable.run_id(id);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'paramstruct',params);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'userParameters',params);
end
% ideal DC tracking
@@ -66,7 +66,7 @@ params.mu_dc = 0.005;
futures_list = parallel.FevalFuture.empty();
for id = 1:length(dataTable.run_id)
run_id = dataTable.run_id(id);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'paramstruct',params);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'userParameters',params);
end
% DC smoothing
@@ -79,7 +79,7 @@ params.mu_dc = 0.00;
futures_list = parallel.FevalFuture.empty();
for id = 1:length(dataTable.run_id)
run_id = dataTable.run_id(id);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'paramstruct',params);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'userParameters',params);
end
% only FFE
@@ -92,7 +92,7 @@ params.mu_dc = 0.00;
futures_list = parallel.FevalFuture.empty();
for id = 1:length(dataTable.run_id)
run_id = dataTable.run_id(id);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'paramstruct',params);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'userParameters',params);
end

2
projects/ECOC_2025_MPI/dsp_standalone_2.m
View File

@@ -45,7 +45,7 @@ params.mu_dc = 0.005;
futures_list = parallel.FevalFuture.empty();
for id = 1:length(dataTable.run_id)
run_id = dataTable.run_id(id);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'paramstruct',params);
[out, futures_list(id)] = submit_dsp(run_id, databasePath, database_name, savePath,"parallel",run_par,'max_occurences',num_occ,'userParameters',params);
end
% Extract all ber_mlse values from the vnle_pf_package using cellfun

48
projects/ECOC_2025_MPI/load_signal_standalone.m
View File

@@ -45,46 +45,12 @@ for i = 1:size(dataTable,1)
Scpe_sig_raw.plot("displayname",['Scope Signal (Run ID: ',num2str(dataTable_.run_id)],"fignum",dataTable_.run_id,"clear",0);
Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in",Scpe_sig_raw.fs,"fs_out",fsym);
[~,Scpe_cell,found_sync,test,shifts] = Scpe_sig_resampled.tsynch("reference",Symbols,"fs_ref",dataTable_.symbolrate,"debug_plots",0);
shifts_mus = shifts./Scpe_sig_resampled.fs .*1e6;
Scpe_sig_resampled = Scpe_sig_resampled.normalize("mode","rms");
% Scpe_sig_resampled.plot("displayname",['Scope Signal (Run ID: ',num2str(dataTable_.run_id)],"fignum",dataTable_.run_id,"clear",0);
hold on;
% xline(shifts_mus,'HandleVisibility','off');
shifts = shifts-shifts(1)+1;
sep_sig = NaN(M,length(Scpe_sig_resampled));
avg_sig = NaN(M,length(Scpe_sig_resampled));
for j = 1:size(Scpe_cell,1)
[sep_sig_,avg_sig_]=showLevelScatter(Scpe_cell{j}.resample("fs_out",Symbols.fs),Symbols,"fignum",400);
s = shifts(j);
sep_sig(:,s+1:s+length(sep_sig_)) = sep_sig_;
avg_sig(:,s+1:s+length(avg_sig_)) = avg_sig_;
end
disp(num2str(dataTable_.interference_path_length));
[sep_sig, avg_sig] = showLevelScatter(Scpe_sig_raw, Symbols, ...
"fsym", fsym, ...
"syncFs", 2*fsym, ...
"fignum", 400, ...
"normalize", true, ...
"debug_plots", false);
var(sep_sig,0,2,'omitnan')
xax_in_sec = ((1:length(avg_sig)) / fsym) * 1e6;
figure();hold on;
cols = cbrewer2('Paired',8);
for p = 1:size(avg_sig,1)
sc=scatter(xax_in_sec,sep_sig(p,:),1,'.','MarkerEdgeColor',cols((2*p)-1,:),'MarkerEdgeAlpha',0.2);
end
for p = 1:size(avg_sig,1)
sc=plot(xax_in_sec,avg_sig(p,:),'LineWidth',1,'Color',cols((2*p),:));
end
yline(unique(Symbols.signal),'HandleVisibility','off');
% xline(shifts./ fsym .*1e6,'HandleVisibility','off');
xlabel('time in $\mu$s');
ylabel('Normalized Amplitude');
xlim([0 25]);
ylim([-3 3]);
drawnow;
end
end

6
projects/ECOC_2025_MPI/submit_dsp.m
View File

@@ -12,7 +12,7 @@ arguments
savePath
options.parallel (1,1) logical = true
options.max_occurences = 1;
options.paramstruct = struct();
options.userParameters = struct();
end
if options.parallel
@@ -31,7 +31,7 @@ if options.parallel
'storage_path', savePath, ...
'append_to_db', 1, ...
'max_occurences', options.max_occurences, ...
'parameters', options.paramstruct ...
'userParameters', options.userParameters ...
);
output = [];
@@ -49,7 +49,7 @@ else
'storage_path', savePath, ...
'append_to_db', 1, ...
'max_occurences', options.max_occurences,...
'parameters', options.paramstruct ...
'userParameters', options.userParameters ...
);
future = []; % No future since it's synchronous

4
projects/MLSE_ML_based/experimental_data.m
View File

@@ -15,7 +15,7 @@ duob_mode = db_mode(strrep(dataTable.db_mode,'"',''));
% end
% Load and Sync signal data from DB
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncSignalDataFromDb(dataTable, dsp_options);
[Tx_bits, Symbols, Scpe_cell, ~] = loadAndSyncRunSignals(dataTable, dsp_options);
% Preprocess signal
Scpe_sig = preprocessSignal(Scpe_cell{1}, Symbols, fsym);
@@ -160,4 +160,4 @@ set(gca,'YScale','log');
xlabel('Epoch');
ylabel('BER');
title('PAM-4; L=3; SNR=20; AWGN Channel');
plot(1:200,ml_mlse_equalizer_adap.ber,'DisplayName','Adaptive mu');
plot(1:200,ml_mlse_equalizer_adap.ber,'DisplayName','Adaptive mu');

188
projects/minimal_dsp_recipe_example/dsp_architecture_overview.svg Normal file
View File

@@ -0,0 +1,188 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="980" viewBox="0 0 1400 980" role="img" aria-labelledby="title desc">
<title id="title">DSP Job Architecture Overview</title>
<desc id="desc">Architecture overview for the MATLAB DSP job framework: entry scripts define options and user parameters, submitJobs creates jobs, runBatch executes them, dsp_runid loads inputs and calls a recipe, and recipes use existing EQ kernels.</desc>
<defs>
<style>
.bg { fill: #f7f8fb; }
.title { font: 700 28px Arial, sans-serif; fill: #152033; }
.subtitle { font: 400 15px Arial, sans-serif; fill: #48566f; }
.sectionTitle { font: 700 16px Arial, sans-serif; fill: #1d2a3d; }
.boxTitle { font: 700 15px Arial, sans-serif; fill: #102033; }
.boxText { font: 400 12px Arial, sans-serif; fill: #314158; }
.smallText { font: 400 11px Arial, sans-serif; fill: #4e5f78; }
.noteText { font: 400 12px Arial, sans-serif; fill: #27364a; }
.laneLabel { font: 700 13px Arial, sans-serif; fill: #53627a; letter-spacing: 0.5px; }
.entry { fill: #e8f0ff; stroke: #4274c9; }
.job { fill: #eaf7f0; stroke: #3f9d68; }
.worker { fill: #fff3df; stroke: #d38a2f; }
.recipe { fill: #f3ecff; stroke: #8964c8; }
.kernel { fill: #eef2f5; stroke: #67798e; }
.sink { fill: #fdecee; stroke: #c65a6a; }
.note { fill: #ffffff; stroke: #c8d0db; }
.box { stroke-width: 1.5; rx: 8; ry: 8; }
.lane { fill: none; stroke: #d9dee8; stroke-width: 1; stroke-dasharray: 7 7; rx: 14; ry: 14; }
.arrow { stroke: #34445a; stroke-width: 2; fill: none; marker-end: url(#arrowHead); }
.thinArrow { stroke: #67798e; stroke-width: 1.5; fill: none; marker-end: url(#arrowHeadSmall); }
.dashedArrow { stroke: #67798e; stroke-width: 1.5; fill: none; stroke-dasharray: 6 5; marker-end: url(#arrowHeadSmall); }
</style>
<marker id="arrowHead" markerWidth="10" markerHeight="10" refX="8" refY="3" orient="auto" markerUnits="strokeWidth">
<path d="M0,0 L8,3 L0,6 Z" fill="#34445a"/>
</marker>
<marker id="arrowHeadSmall" markerWidth="8" markerHeight="8" refX="7" refY="3" orient="auto" markerUnits="strokeWidth">
<path d="M0,0 L7,3 L0,6 Z" fill="#67798e"/>
</marker>
</defs>
<rect class="bg" x="0" y="0" width="1400" height="980"/>
<text class="title" x="50" y="54">DSP Job Architecture Overview</text>
<text class="subtitle" x="50" y="80">Separation of entry scripts, job execution, input loading, recipe orchestration, existing EQ kernels, and result sinks.</text>
<rect class="lane" x="40" y="115" width="1320" height="120"/>
<text class="laneLabel" x="60" y="140">ENTRY / USER INTENT</text>
<rect class="lane" x="40" y="270" width="1320" height="145"/>
<text class="laneLabel" x="60" y="295">JOB CONSTRUCTION AND EXECUTION</text>
<rect class="lane" x="40" y="450" width="1320" height="180"/>
<text class="laneLabel" x="60" y="475">WORKER INPUT AND RECIPE CALL</text>
<rect class="lane" x="40" y="665" width="1320" height="170"/>
<text class="laneLabel" x="60" y="690">DSP RECIPE AND EXISTING ALGORITHM KERNELS</text>
<rect class="lane" x="40" y="865" width="1320" height="80"/>
<text class="laneLabel" x="60" y="890">RESULTS</text>
<!-- Entry layer -->
<rect class="box entry" x="75" y="155" width="270" height="62"/>
<text class="boxTitle" x="95" y="180">Entry script</text>
<text class="boxText" x="95" y="199">run_minimal_recipe.m or project script</text>
<rect class="box entry" x="395" y="145" width="300" height="82"/>
<text class="boxTitle" x="415" y="170">dsp_options</text>
<text class="boxText" x="415" y="190">mode, recipe, DB settings</text>
<text class="boxText" x="415" y="207">append_to_db, debug_plots</text>
<rect class="box entry" x="745" y="145" width="300" height="82"/>
<text class="boxTitle" x="765" y="170">userParameters</text>
<text class="boxText" x="765" y="190">User-defined sweep and overrides</text>
<text class="boxText" x="765" y="207">Example: dd_mode = [0, 1]</text>
<rect class="box entry" x="1095" y="145" width="220" height="82"/>
<text class="boxTitle" x="1115" y="170">run_ids</text>
<text class="boxText" x="1115" y="190">Measured or simulated run records</text>
<text class="boxText" x="1115" y="207">Usually selected by QueryFilter</text>
<!-- Job layer -->
<rect class="box job" x="95" y="320" width="285" height="72"/>
<text class="boxTitle" x="115" y="345">DataStorage wh</text>
<text class="boxText" x="115" y="365">Expands userParameters grid</text>
<text class="boxText" x="115" y="382">Defines optional result containers</text>
<rect class="box job" x="455" y="310" width="300" height="92"/>
<text class="boxTitle" x="475" y="335">submitJobs</text>
<text class="boxText" x="475" y="355">Builds one job per run_id and sweep index</text>
<text class="boxText" x="475" y="372">Injects concrete userParameters</text>
<text class="boxText" x="475" y="389">Stores matching output fields into wh</text>
<rect class="box job" x="850" y="310" width="285" height="92"/>
<text class="boxTitle" x="870" y="335">runBatch</text>
<text class="boxText" x="870" y="355">Generic executor</text>
<text class="boxText" x="870" y="372">Serial: workerFcn(args{:})</text>
<text class="boxText" x="870" y="389">Parallel: parfeval + fetchNext</text>
<!-- Worker layer -->
<rect class="box worker" x="95" y="505" width="280" height="92"/>
<text class="boxTitle" x="115" y="530">dsp_runid</text>
<text class="boxText" x="115" y="550">Worker for one job</text>
<text class="boxText" x="115" y="567">Normalizes input source</text>
<text class="boxText" x="115" y="584">Iterates synchronized occurrences</text>
<rect class="box worker" x="455" y="490" width="285" height="122"/>
<text class="boxTitle" x="475" y="515">Input source</text>
<text class="boxText" x="475" y="535">run_id: loadDspInputFromRunId</text>
<text class="boxText" x="475" y="552">file_paths: loadDspInputFromFilePaths</text>
<text class="boxText" x="475" y="569">loadAndSyncRunSignals</text>
<text class="boxText" x="475" y="586">Returns canonical dspInput struct</text>
<rect class="box worker" x="820" y="505" width="300" height="92"/>
<text class="boxTitle" x="840" y="530">Canonical dspInput</text>
<text class="boxText" x="840" y="550">Scpe_cell, Symbols, Tx_bits</text>
<text class="boxText" x="840" y="567">fsym, M, duob_mode, dataTable</text>
<text class="boxText" x="840" y="584">One occurrence passed to recipe</text>
<!-- Recipe layer -->
<rect class="box recipe" x="95" y="720" width="300" height="82"/>
<text class="boxTitle" x="115" y="745">Recipe function</text>
<text class="boxText" x="115" y="765">dsp_recipe_minimal or project recipe</text>
<text class="boxText" x="115" y="782">Orchestrates preprocessing and kernels</text>
<rect class="box recipe" x="455" y="710" width="285" height="102"/>
<text class="boxTitle" x="475" y="735">preprocessSignal</text>
<text class="boxText" x="475" y="755">Resample or matched filter</text>
<text class="boxText" x="475" y="772">Timing synchronization</text>
<text class="boxText" x="475" y="789">Optional filtering / debug plots</text>
<rect class="box kernel" x="820" y="690" width="465" height="135"/>
<text class="boxTitle" x="840" y="715">Existing EQ kernels</text>
<text class="boxText" x="840" y="735">ffe(eq, M, rx, symbols, bits, ...)</text>
<text class="boxText" x="840" y="752">ml_mlse(eq, M, rx, symbols, bits, ...)</text>
<text class="boxText" x="840" y="769">vnle_postfilter_mlse(eq, pf, mlse, M, rx, symbols, bits, ...)</text>
<text class="boxText" x="840" y="786">duobinary_target / duobinary_signaling</text>
<text class="boxText" x="840" y="806">Return result structs with .metrics and .config</text>
<!-- Results -->
<rect class="box sink" x="95" y="900" width="300" height="38"/>
<text class="boxTitle" x="115" y="924">results cell array</text>
<rect class="box sink" x="455" y="900" width="300" height="38"/>
<text class="boxTitle" x="475" y="924">Warehouse wh.sto fields</text>
<rect class="box sink" x="820" y="900" width="300" height="38"/>
<text class="boxTitle" x="840" y="924">Optional DB writes</text>
<!-- Arrows main path -->
<path class="arrow" d="M345 186 C365 186 375 186 395 186"/>
<path class="arrow" d="M695 186 C715 186 725 186 745 186"/>
<path class="arrow" d="M1045 186 C1065 186 1075 186 1095 186"/>
<path class="arrow" d="M210 217 C210 265 220 285 237 320"/>
<path class="arrow" d="M545 227 C545 265 565 285 605 310"/>
<path class="arrow" d="M895 227 C865 265 760 286 675 310"/>
<path class="arrow" d="M1205 227 C1140 270 990 295 755 330"/>
<path class="arrow" d="M380 356 C410 356 425 356 455 356"/>
<path class="arrow" d="M755 356 C790 356 815 356 850 356"/>
<path class="arrow" d="M992 402 C850 450 575 450 375 530"/>
<path class="arrow" d="M375 551 C405 551 425 551 455 551"/>
<path class="arrow" d="M740 551 C770 551 790 551 820 551"/>
<path class="arrow" d="M970 597 C875 655 520 675 395 742"/>
<path class="arrow" d="M395 761 C425 761 435 761 455 761"/>
<path class="arrow" d="M740 761 C770 761 790 761 820 761"/>
<!-- Result arrows -->
<path class="arrow" d="M1052 825 C1052 860 985 880 970 900"/>
<path class="arrow" d="M245 802 C245 845 245 875 245 900"/>
<path class="arrow" d="M597 812 C597 850 605 875 605 900"/>
<!-- Optional DB path -->
<path class="dashedArrow" d="M1120 505 C1240 540 1270 800 970 900"/>
<text class="smallText" x="1148" y="536">append_to_db = true</text>
<!-- Notes -->
<rect class="box note" x="1160" y="305" width="175" height="95"/>
<text class="sectionTitle" x="1175" y="330">Naming rule</text>
<text class="noteText" x="1175" y="350">userParameters:</text>
<text class="noteText" x="1175" y="367">only user sweep /</text>
<text class="noteText" x="1175" y="384">override values</text>
<rect class="box note" x="1160" y="505" width="175" height="95"/>
<text class="sectionTitle" x="1175" y="530">Key boundary</text>
<text class="noteText" x="1175" y="550">runBatch knows jobs,</text>
<text class="noteText" x="1175" y="567">not DSP.</text>
<text class="noteText" x="1175" y="584">Recipes know DSP.</text>
<rect class="box note" x="95" y="835" width="530" height="24"/>
<text class="smallText" x="110" y="852">Recipe output fields such as ffe_package or mlse_package are copied generically by dsp_runid and stored by submitJobs when wh contains matching storage.</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

130
projects/minimal_dsp_recipe_example/run_minimal_recipe.m Normal file
View File

@@ -0,0 +1,130 @@
% Minimal example for exchangeable DSP recipes.
%
% This script demonstrates the framework wiring only. Set run_ids to one or
% more known runs before executing it.
%
% Signal and parameter flow:
%
% 1) *This script* here entry script defines run selection and DSP options.
% - run_ids selects the measured/simulated records to process.
% - dsp_options contains worker configuration, database connection values,
% the recipe function handle, and the parameter sweep definition.
%
% 1.1) *DataStorage* expands dsp_options.userParameters into concrete jobs.
% Example:
% dsp_options.userParameters.dd_mode = [0, 1]
% creates two jobs for each run_id:
% job 1: userParameters.dd_mode = 0
% job 2: userParameters.dd_mode = 1
%
% 2) *submitJobs* builds the job list.
% For each run_id and each Warehouse index it forwards:
% dsp_runid(run_id, "recipe", @dsp_recipe_minimal, ...
% "userParameters", concreteParameterStruct, ...)
%
% 3) *runBatch* executes the job list.
% submitJobs delegates execution to runBatch. runBatch is the generic
% serial/parallel executor; it handles feval/parfeval, progress updates,
% result callbacks, and error callbacks, but it has no DSP, DB, Warehouse,
% or recipe-specific semantics.
%
% 4) *dsp_runid* is the worker.
% It does not own DSP algorithm choices. It only:
% - opens its own DB connection if mode/load/write requires it
% - loads metadata for the run_id
% - loads Tx_bits, Symbols, and received scope data
% - synchronizes/limits Scpe_cell occurrences
% - calls the selected recipe for each occurrence
%
% 5) The *recipe* is the project-specific DSP layer.
% dsp_recipe_minimal receives exactly:
% Scpe_sig_raw: one received scope/synchronized signal occurrence
% Symbols: transmitted symbols/reference
% Tx_bits: transmitted bit reference
% fsym, M, duob_mode: metadata from the run
% userParameters: one concrete parameter struct from the sweep
%
% 6) *Results* flow back as a standard output struct.
% submitJobs auto-creates wh.sto fields from non-empty recipe output
% fields. In this example, output.ffe_package creates wh.sto.ffe_package.
clearvars;
% === DSP settings ===
% The recipe handle is the only DSP-specific dispatch point in this script.
% Switch it to another function with the same signature to run another DSP
% chain without changing submitJobs, runBatch, DB loading, or Warehouse logic.
dsp_options = struct();
dsp_options.mode = "run_id";
dsp_options.recipe = @dsp_recipe_minimal;
dsp_options.append_to_db = false;
dsp_options.max_occurences = 1;
dsp_options.debug_plots = true;
dsp_options.database_type = "mysql";
mpi = 1;
if mpi
dsp_options.dataBase = "labor";
dsp_options.storage_path = "W:\labdata\ECOC Silas\ecoc_2025";
else
dsp_options.dataBase = "labor_highspeed";
dsp_options.storage_path = "W:\labdata\sioe_labor\";
end
dsp_options.server = "192.168.178.192";
dsp_options.port = 3306;
dsp_options.user = "silas";
dsp_options.password = "silas";
db = DBHandler("dataBase", [dsp_options.dataBase],...
"type", dsp_options.database_type,...
"server", dsp_options.server,...
"user", dsp_options.user, "password", dsp_options.password);
% Parameters are expanded by DataStorage and interpreted by the recipe.
% submitJobs treats this struct as transport data only; semantic meaning
% belongs to dsp_recipe_minimal or any replacement recipe.
dsp_options.userParameters = struct();
dsp_options.userParameters.dd_mode = [0,1];
fp = QueryFilter();
if mpi
fp.where('Runs', 'symbolrate', 'EQUALS', 112e9);
fp.where('Runs', 'fiber_length', 'EQUALS', 0);
fp.where('Runs', 'interference_path_length', 'EQUALS', 1000);
fp.where('Runs', 'sir', 'EQUALS', 52);
fp.where('Runs', 'db_mode', 'EQUALS', '"no_db"');
fp.where('Runs', 'is_mpi', 'EQUALS', 1);
fp.where('Runs', 'pam_level', 'EQUALS', 4);
fp.where('Runs', 'wavelength', 'EQUALS', 1310);
fp.where('Runs', 'v_bias', 'EQUALS', 2.65);
else
% fp.where('Runs', 'pam_level','EQUALS', 4);
% fp.where('Runs', 'bitrate','EQUALS', 360e9);
% fp.where('Runs', 'fiber_length','EQUALS', 2);
% fp.where('Runs', 'is_mpi','EQUALS', 0);
% fp.where('Runs', 'wavelength','EQUALS', 1310);
% fp.where('Runs', 'db_mode','EQUALS', 0);
% fp.where('Runs', 'rop_attenuation','EQUAL', 0);
end
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
run_ids = dataTable.run_id(1);
if isempty(run_ids)
error("run_minimal_recipe:MissingRunIds", ...
"Set run_ids to one or more known run IDs before running this example.");
end
%% === Warehouse setup ===
% DataStorage defines the parameter grid. Result containers are created by
% submitJobs from returned recipe output fields.
wh = DataStorage(dsp_options.userParameters);
%% === Run ===
% submitJobs returns the raw per-job results and the filled Warehouse. For a
% single run_id and dd_mode = [0, 1], results is a 2-by-1 cell array.
[results, wh] = submitJobs(run_ids, dsp_options, processingMode.serial, ...
"wh", wh, ...
"waitbar", true);

64
projects/minimal_dsp_recipe_example/run_minimal_runbatch_sweep.m Normal file
View File

@@ -0,0 +1,64 @@
% Minimal runBatch example without DSP, DB, or Warehouse.
%
% This script shows the generic execution layer directly:
%
% jobs -> runBatch -> workerFcn -> results
%
% runBatch only needs a function handle and a struct array of jobs. Each
% jobs(i).args cell is expanded into the worker function call.
clearvars;
% === Define a small calculation sweep ===
gainValues = [0.5, 1, 2];
offsetValues = [-1, 0, 1];
jobs = repmat(struct('args', {{}}, 'label', "", 'meta', struct()), ...
1, numel(gainValues)*numel(offsetValues));
jobIndex = 0;
for gi = 1:numel(gainValues)
for oi = 1:numel(offsetValues)
jobIndex = jobIndex + 1;
gain = gainValues(gi);
offset = offsetValues(oi);
jobs(jobIndex).args = {gain, offset};
jobs(jobIndex).label = sprintf("gain %.1f, offset %.1f", gain, offset);
jobs(jobIndex).meta.gainIndex = gi;
jobs(jobIndex).meta.offsetIndex = oi;
jobs(jobIndex).meta.gain = gain;
jobs(jobIndex).meta.offset = offset;
end
end
% === Execute ===
% Switch mode to processingMode.parallel to run the same jobs via parfeval.
results = runBatch(@minimalCalculationWorker, jobs, ...
"mode", processingMode.serial, ...
"waitbar", false, ...
"resultHandler", @printResult);
% Convert the cell result list into a table for convenient inspection.
resultTable = struct2table([results{:}].');
disp(resultTable);
function result = minimalCalculationWorker(gain, offset)
%MINIMALCALCULATIONWORKER Tiny worker used by this example.
inputSignal = 1:5;
outputSignal = gain .* inputSignal + offset;
result = struct();
result.gain = gain;
result.offset = offset;
result.meanOutput = mean(outputSignal);
result.maxOutput = max(outputSignal);
result.outputSignal = outputSignal;
end
function printResult(result, job, jobIndex)
fprintf('[%s | #%d] mean %.2f, max %.2f\n', ...
job.label, jobIndex, result.meanOutput, result.maxOutput);
end

BIN
projects/minimal_dsp_recipe_example/workerError.mat Normal file
View File

Binary file not shown.