silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

restructure and organize

Browse Source
This commit is contained in:
Silas Oettinghaus
2026-06-22 22:58:58 +02:00
parent c4f75b7ec4
commit 33ec5b3116
36 changed files with 1914 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

188
projects/minimal_dsp_recipe_example/dsp_architecture_overview.svg Normal file
View File

@@ -0,0 +1,188 @@
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="980" viewBox="0 0 1400 980" role="img" aria-labelledby="title desc">
<title id="title">DSP Job Architecture Overview</title>
<desc id="desc">Architecture overview for the MATLAB DSP job framework: entry scripts define options and user parameters, submitJobs creates jobs, runBatch executes them, dsp_runid loads inputs and calls a recipe, and recipes use existing EQ kernels.</desc>
<defs>
<style>
.bg { fill: #f7f8fb; }
.title { font: 700 28px Arial, sans-serif; fill: #152033; }
.subtitle { font: 400 15px Arial, sans-serif; fill: #48566f; }
.sectionTitle { font: 700 16px Arial, sans-serif; fill: #1d2a3d; }
.boxTitle { font: 700 15px Arial, sans-serif; fill: #102033; }
.boxText { font: 400 12px Arial, sans-serif; fill: #314158; }
.smallText { font: 400 11px Arial, sans-serif; fill: #4e5f78; }
.noteText { font: 400 12px Arial, sans-serif; fill: #27364a; }
.laneLabel { font: 700 13px Arial, sans-serif; fill: #53627a; letter-spacing: 0.5px; }
.entry { fill: #e8f0ff; stroke: #4274c9; }
.job { fill: #eaf7f0; stroke: #3f9d68; }
.worker { fill: #fff3df; stroke: #d38a2f; }
.recipe { fill: #f3ecff; stroke: #8964c8; }
.kernel { fill: #eef2f5; stroke: #67798e; }
.sink { fill: #fdecee; stroke: #c65a6a; }
.note { fill: #ffffff; stroke: #c8d0db; }
.box { stroke-width: 1.5; rx: 8; ry: 8; }
.lane { fill: none; stroke: #d9dee8; stroke-width: 1; stroke-dasharray: 7 7; rx: 14; ry: 14; }
.arrow { stroke: #34445a; stroke-width: 2; fill: none; marker-end: url(#arrowHead); }
.thinArrow { stroke: #67798e; stroke-width: 1.5; fill: none; marker-end: url(#arrowHeadSmall); }
.dashedArrow { stroke: #67798e; stroke-width: 1.5; fill: none; stroke-dasharray: 6 5; marker-end: url(#arrowHeadSmall); }
</style>
<marker id="arrowHead" markerWidth="10" markerHeight="10" refX="8" refY="3" orient="auto" markerUnits="strokeWidth">
<path d="M0,0 L8,3 L0,6 Z" fill="#34445a"/>
</marker>
<marker id="arrowHeadSmall" markerWidth="8" markerHeight="8" refX="7" refY="3" orient="auto" markerUnits="strokeWidth">
<path d="M0,0 L7,3 L0,6 Z" fill="#67798e"/>
</marker>
</defs>
<rect class="bg" x="0" y="0" width="1400" height="980"/>
<text class="title" x="50" y="54">DSP Job Architecture Overview</text>
<text class="subtitle" x="50" y="80">Separation of entry scripts, job execution, input loading, recipe orchestration, existing EQ kernels, and result sinks.</text>
<rect class="lane" x="40" y="115" width="1320" height="120"/>
<text class="laneLabel" x="60" y="140">ENTRY / USER INTENT</text>
<rect class="lane" x="40" y="270" width="1320" height="145"/>
<text class="laneLabel" x="60" y="295">JOB CONSTRUCTION AND EXECUTION</text>
<rect class="lane" x="40" y="450" width="1320" height="180"/>
<text class="laneLabel" x="60" y="475">WORKER INPUT AND RECIPE CALL</text>
<rect class="lane" x="40" y="665" width="1320" height="170"/>
<text class="laneLabel" x="60" y="690">DSP RECIPE AND EXISTING ALGORITHM KERNELS</text>
<rect class="lane" x="40" y="865" width="1320" height="80"/>
<text class="laneLabel" x="60" y="890">RESULTS</text>
<!-- Entry layer -->
<rect class="box entry" x="75" y="155" width="270" height="62"/>
<text class="boxTitle" x="95" y="180">Entry script</text>
<text class="boxText" x="95" y="199">run_minimal_recipe.m or project script</text>
<rect class="box entry" x="395" y="145" width="300" height="82"/>
<text class="boxTitle" x="415" y="170">dsp_options</text>
<text class="boxText" x="415" y="190">mode, recipe, DB settings</text>
<text class="boxText" x="415" y="207">append_to_db, debug_plots</text>
<rect class="box entry" x="745" y="145" width="300" height="82"/>
<text class="boxTitle" x="765" y="170">userParameters</text>
<text class="boxText" x="765" y="190">User-defined sweep and overrides</text>
<text class="boxText" x="765" y="207">Example: dd_mode = [0, 1]</text>
<rect class="box entry" x="1095" y="145" width="220" height="82"/>
<text class="boxTitle" x="1115" y="170">run_ids</text>
<text class="boxText" x="1115" y="190">Measured or simulated run records</text>
<text class="boxText" x="1115" y="207">Usually selected by QueryFilter</text>
<!-- Job layer -->
<rect class="box job" x="95" y="320" width="285" height="72"/>
<text class="boxTitle" x="115" y="345">DataStorage wh</text>
<text class="boxText" x="115" y="365">Expands userParameters grid</text>
<text class="boxText" x="115" y="382">Defines optional result containers</text>
<rect class="box job" x="455" y="310" width="300" height="92"/>
<text class="boxTitle" x="475" y="335">submitJobs</text>
<text class="boxText" x="475" y="355">Builds one job per run_id and sweep index</text>
<text class="boxText" x="475" y="372">Injects concrete userParameters</text>
<text class="boxText" x="475" y="389">Stores matching output fields into wh</text>
<rect class="box job" x="850" y="310" width="285" height="92"/>
<text class="boxTitle" x="870" y="335">runBatch</text>
<text class="boxText" x="870" y="355">Generic executor</text>
<text class="boxText" x="870" y="372">Serial: workerFcn(args{:})</text>
<text class="boxText" x="870" y="389">Parallel: parfeval + fetchNext</text>
<!-- Worker layer -->
<rect class="box worker" x="95" y="505" width="280" height="92"/>
<text class="boxTitle" x="115" y="530">dsp_runid</text>
<text class="boxText" x="115" y="550">Worker for one job</text>
<text class="boxText" x="115" y="567">Normalizes input source</text>
<text class="boxText" x="115" y="584">Iterates synchronized occurrences</text>
<rect class="box worker" x="455" y="490" width="285" height="122"/>
<text class="boxTitle" x="475" y="515">Input source</text>
<text class="boxText" x="475" y="535">run_id: loadDspInputFromRunId</text>
<text class="boxText" x="475" y="552">file_paths: loadDspInputFromFilePaths</text>
<text class="boxText" x="475" y="569">loadAndSyncRunSignals</text>
<text class="boxText" x="475" y="586">Returns canonical dspInput struct</text>
<rect class="box worker" x="820" y="505" width="300" height="92"/>
<text class="boxTitle" x="840" y="530">Canonical dspInput</text>
<text class="boxText" x="840" y="550">Scpe_cell, Symbols, Tx_bits</text>
<text class="boxText" x="840" y="567">fsym, M, duob_mode, dataTable</text>
<text class="boxText" x="840" y="584">One occurrence passed to recipe</text>
<!-- Recipe layer -->
<rect class="box recipe" x="95" y="720" width="300" height="82"/>
<text class="boxTitle" x="115" y="745">Recipe function</text>
<text class="boxText" x="115" y="765">dsp_recipe_minimal or project recipe</text>
<text class="boxText" x="115" y="782">Orchestrates preprocessing and kernels</text>
<rect class="box recipe" x="455" y="710" width="285" height="102"/>
<text class="boxTitle" x="475" y="735">preprocessSignal</text>
<text class="boxText" x="475" y="755">Resample or matched filter</text>
<text class="boxText" x="475" y="772">Timing synchronization</text>
<text class="boxText" x="475" y="789">Optional filtering / debug plots</text>
<rect class="box kernel" x="820" y="690" width="465" height="135"/>
<text class="boxTitle" x="840" y="715">Existing EQ kernels</text>
<text class="boxText" x="840" y="735">ffe(eq, M, rx, symbols, bits, ...)</text>
<text class="boxText" x="840" y="752">ml_mlse(eq, M, rx, symbols, bits, ...)</text>
<text class="boxText" x="840" y="769">vnle_postfilter_mlse(eq, pf, mlse, M, rx, symbols, bits, ...)</text>
<text class="boxText" x="840" y="786">duobinary_target / duobinary_signaling</text>
<text class="boxText" x="840" y="806">Return result structs with .metrics and .config</text>
<!-- Results -->
<rect class="box sink" x="95" y="900" width="300" height="38"/>
<text class="boxTitle" x="115" y="924">results cell array</text>
<rect class="box sink" x="455" y="900" width="300" height="38"/>
<text class="boxTitle" x="475" y="924">Warehouse wh.sto fields</text>
<rect class="box sink" x="820" y="900" width="300" height="38"/>
<text class="boxTitle" x="840" y="924">Optional DB writes</text>
<!-- Arrows main path -->
<path class="arrow" d="M345 186 C365 186 375 186 395 186"/>
<path class="arrow" d="M695 186 C715 186 725 186 745 186"/>
<path class="arrow" d="M1045 186 C1065 186 1075 186 1095 186"/>
<path class="arrow" d="M210 217 C210 265 220 285 237 320"/>
<path class="arrow" d="M545 227 C545 265 565 285 605 310"/>
<path class="arrow" d="M895 227 C865 265 760 286 675 310"/>
<path class="arrow" d="M1205 227 C1140 270 990 295 755 330"/>
<path class="arrow" d="M380 356 C410 356 425 356 455 356"/>
<path class="arrow" d="M755 356 C790 356 815 356 850 356"/>
<path class="arrow" d="M992 402 C850 450 575 450 375 530"/>
<path class="arrow" d="M375 551 C405 551 425 551 455 551"/>
<path class="arrow" d="M740 551 C770 551 790 551 820 551"/>
<path class="arrow" d="M970 597 C875 655 520 675 395 742"/>
<path class="arrow" d="M395 761 C425 761 435 761 455 761"/>
<path class="arrow" d="M740 761 C770 761 790 761 820 761"/>
<!-- Result arrows -->
<path class="arrow" d="M1052 825 C1052 860 985 880 970 900"/>
<path class="arrow" d="M245 802 C245 845 245 875 245 900"/>
<path class="arrow" d="M597 812 C597 850 605 875 605 900"/>
<!-- Optional DB path -->
<path class="dashedArrow" d="M1120 505 C1240 540 1270 800 970 900"/>
<text class="smallText" x="1148" y="536">append_to_db = true</text>
<!-- Notes -->
<rect class="box note" x="1160" y="305" width="175" height="95"/>
<text class="sectionTitle" x="1175" y="330">Naming rule</text>
<text class="noteText" x="1175" y="350">userParameters:</text>
<text class="noteText" x="1175" y="367">only user sweep /</text>
<text class="noteText" x="1175" y="384">override values</text>
<rect class="box note" x="1160" y="505" width="175" height="95"/>
<text class="sectionTitle" x="1175" y="530">Key boundary</text>
<text class="noteText" x="1175" y="550">runBatch knows jobs,</text>
<text class="noteText" x="1175" y="567">not DSP.</text>
<text class="noteText" x="1175" y="584">Recipes know DSP.</text>
<rect class="box note" x="95" y="835" width="530" height="24"/>
<text class="smallText" x="110" y="852">Recipe output fields such as ffe_package or mlse_package are copied generically by dsp_runid and stored by submitJobs when wh contains matching storage.</text>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 10 KiB

130
projects/minimal_dsp_recipe_example/run_minimal_recipe.m Normal file
View File

@@ -0,0 +1,130 @@
% Minimal example for exchangeable DSP recipes.
%
% This script demonstrates the framework wiring only. Set run_ids to one or
% more known runs before executing it.
%
% Signal and parameter flow:
%
% 1) *This script* here entry script defines run selection and DSP options.
% - run_ids selects the measured/simulated records to process.
% - dsp_options contains worker configuration, database connection values,
% the recipe function handle, and the parameter sweep definition.
%
% 1.1) *DataStorage* expands dsp_options.userParameters into concrete jobs.
% Example:
% dsp_options.userParameters.dd_mode = [0, 1]
% creates two jobs for each run_id:
% job 1: userParameters.dd_mode = 0
% job 2: userParameters.dd_mode = 1
%
% 2) *submitJobs* builds the job list.
% For each run_id and each Warehouse index it forwards:
% dsp_runid(run_id, "recipe", @dsp_recipe_minimal, ...
% "userParameters", concreteParameterStruct, ...)
%
% 3) *runBatch* executes the job list.
% submitJobs delegates execution to runBatch. runBatch is the generic
% serial/parallel executor; it handles feval/parfeval, progress updates,
% result callbacks, and error callbacks, but it has no DSP, DB, Warehouse,
% or recipe-specific semantics.
%
% 4) *dsp_runid* is the worker.
% It does not own DSP algorithm choices. It only:
% - opens its own DB connection if mode/load/write requires it
% - loads metadata for the run_id
% - loads Tx_bits, Symbols, and received scope data
% - synchronizes/limits Scpe_cell occurrences
% - calls the selected recipe for each occurrence
%
% 5) The *recipe* is the project-specific DSP layer.
% dsp_recipe_minimal receives exactly:
% Scpe_sig_raw: one received scope/synchronized signal occurrence
% Symbols: transmitted symbols/reference
% Tx_bits: transmitted bit reference
% fsym, M, duob_mode: metadata from the run
% userParameters: one concrete parameter struct from the sweep
%
% 6) *Results* flow back as a standard output struct.
% submitJobs auto-creates wh.sto fields from non-empty recipe output
% fields. In this example, output.ffe_package creates wh.sto.ffe_package.
clearvars;
% === DSP settings ===
% The recipe handle is the only DSP-specific dispatch point in this script.
% Switch it to another function with the same signature to run another DSP
% chain without changing submitJobs, runBatch, DB loading, or Warehouse logic.
dsp_options = struct();
dsp_options.mode = "run_id";
dsp_options.recipe = @dsp_recipe_minimal;
dsp_options.append_to_db = false;
dsp_options.max_occurences = 1;
dsp_options.debug_plots = true;
dsp_options.database_type = "mysql";
mpi = 1;
if mpi
dsp_options.dataBase = "labor";
dsp_options.storage_path = "W:\labdata\ECOC Silas\ecoc_2025";
else
dsp_options.dataBase = "labor_highspeed";
dsp_options.storage_path = "W:\labdata\sioe_labor\";
end
dsp_options.server = "192.168.178.192";
dsp_options.port = 3306;
dsp_options.user = "silas";
dsp_options.password = "silas";
db = DBHandler("dataBase", [dsp_options.dataBase],...
"type", dsp_options.database_type,...
"server", dsp_options.server,...
"user", dsp_options.user, "password", dsp_options.password);
% Parameters are expanded by DataStorage and interpreted by the recipe.
% submitJobs treats this struct as transport data only; semantic meaning
% belongs to dsp_recipe_minimal or any replacement recipe.
dsp_options.userParameters = struct();
dsp_options.userParameters.dd_mode = [0,1];
fp = QueryFilter();
if mpi
fp.where('Runs', 'symbolrate', 'EQUALS', 112e9);
fp.where('Runs', 'fiber_length', 'EQUALS', 0);
fp.where('Runs', 'interference_path_length', 'EQUALS', 1000);
fp.where('Runs', 'sir', 'EQUALS', 52);
fp.where('Runs', 'db_mode', 'EQUALS', '"no_db"');
fp.where('Runs', 'is_mpi', 'EQUALS', 1);
fp.where('Runs', 'pam_level', 'EQUALS', 4);
fp.where('Runs', 'wavelength', 'EQUALS', 1310);
fp.where('Runs', 'v_bias', 'EQUALS', 2.65);
else
% fp.where('Runs', 'pam_level','EQUALS', 4);
% fp.where('Runs', 'bitrate','EQUALS', 360e9);
% fp.where('Runs', 'fiber_length','EQUALS', 2);
% fp.where('Runs', 'is_mpi','EQUALS', 0);
% fp.where('Runs', 'wavelength','EQUALS', 1310);
% fp.where('Runs', 'db_mode','EQUALS', 0);
% fp.where('Runs', 'rop_attenuation','EQUAL', 0);
end
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
run_ids = dataTable.run_id(1);
if isempty(run_ids)
error("run_minimal_recipe:MissingRunIds", ...
"Set run_ids to one or more known run IDs before running this example.");
end
%% === Warehouse setup ===
% DataStorage defines the parameter grid. Result containers are created by
% submitJobs from returned recipe output fields.
wh = DataStorage(dsp_options.userParameters);
%% === Run ===
% submitJobs returns the raw per-job results and the filled Warehouse. For a
% single run_id and dd_mode = [0, 1], results is a 2-by-1 cell array.
[results, wh] = submitJobs(run_ids, dsp_options, processingMode.serial, ...
"wh", wh, ...
"waitbar", true);

64
projects/minimal_dsp_recipe_example/run_minimal_runbatch_sweep.m Normal file
View File

@@ -0,0 +1,64 @@
% Minimal runBatch example without DSP, DB, or Warehouse.
%
% This script shows the generic execution layer directly:
%
% jobs -> runBatch -> workerFcn -> results
%
% runBatch only needs a function handle and a struct array of jobs. Each
% jobs(i).args cell is expanded into the worker function call.
clearvars;
% === Define a small calculation sweep ===
gainValues = [0.5, 1, 2];
offsetValues = [-1, 0, 1];
jobs = repmat(struct('args', {{}}, 'label', "", 'meta', struct()), ...
1, numel(gainValues)*numel(offsetValues));
jobIndex = 0;
for gi = 1:numel(gainValues)
for oi = 1:numel(offsetValues)
jobIndex = jobIndex + 1;
gain = gainValues(gi);
offset = offsetValues(oi);
jobs(jobIndex).args = {gain, offset};
jobs(jobIndex).label = sprintf("gain %.1f, offset %.1f", gain, offset);
jobs(jobIndex).meta.gainIndex = gi;
jobs(jobIndex).meta.offsetIndex = oi;
jobs(jobIndex).meta.gain = gain;
jobs(jobIndex).meta.offset = offset;
end
end
% === Execute ===
% Switch mode to processingMode.parallel to run the same jobs via parfeval.
results = runBatch(@minimalCalculationWorker, jobs, ...
"mode", processingMode.serial, ...
"waitbar", false, ...
"resultHandler", @printResult);
% Convert the cell result list into a table for convenient inspection.
resultTable = struct2table([results{:}].');
disp(resultTable);
function result = minimalCalculationWorker(gain, offset)
%MINIMALCALCULATIONWORKER Tiny worker used by this example.
inputSignal = 1:5;
outputSignal = gain .* inputSignal + offset;
result = struct();
result.gain = gain;
result.offset = offset;
result.meanOutput = mean(outputSignal);
result.maxOutput = max(outputSignal);
result.outputSignal = outputSignal;
end
function printResult(result, job, jobIndex)
fprintf('[%s | #%d] mean %.2f, max %.2f\n', ...
job.label, jobIndex, result.meanOutput, result.maxOutput);
end

BIN
projects/minimal_dsp_recipe_example/workerError.mat Normal file
View File

Binary file not shown.