Files
imdd_silas/projects/HighSpeedExperiment_2024/auswertung/buildDBfromMeasurements.m
sioe e47a4dbbbe Many changes for 400G DSP
Minimal Example
...
2024-12-17 16:17:58 +01:00

555 lines
22 KiB
Matlab

% Connect to SQLite database
pathToDB = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor\silas_labor.db'; % Update the path as needed
db = DBHandler("pathToDB",pathToDB);
% main file path
sioe_labor_path = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor';
% Get list of all folders (including subfolders) within sioe_labor_path
folderList = dir(fullfile(sioe_labor_path, '**', '*'));
% Filter to include only directories and exclude '.' and '..'
folderNames = {folderList([folderList.isdir]).name};
folderPaths = {folderList([folderList.isdir]).folder}; % Get the full paths
folderPaths = folderPaths(~ismember(folderNames, {'.', '..'}));
folderNames = folderNames(~ismember(folderNames, {'.', '..'}));
% Combine folder names with paths
fullFolderPaths = flip(fullfile(folderPaths, folderNames));
% process only MPI?
only_mpi=0;
if only_mpi
fullFolderPaths = fullFolderPaths(contains(fullFolderPaths,"mpi"));
end
% Shorten folder paths to display only the folder name after the last filesep
displayFolderPaths = cellfun(@(x) x(find(x == filesep, 1, 'last') + 1 : end), fullFolderPaths, 'UniformOutput', false);
% Create checkbox settings for each folder path
numFolders = numel(fullFolderPaths);
checkboxSettings = cell(1, 2 * numFolders);
for i = 1:numFolders
checkboxSettings{2*i-1} = {displayFolderPaths{i}; sprintf('Folder_%d', i)};
checkboxSettings{2*i} = true; % Set false for unchecked by default
end
% Create settings dialog using settingsdlg
[settings, button] = settingsdlg( ...
'Description', 'Select Folders to Process:', ...
'title', 'Folder Selection', ...
checkboxSettings{:});
% Check if the user pressed OK
if strcmp(button, 'OK')
% Get all the field names from settings
allFields = fieldnames(settings);
% Determine which folders were selected
selectedFoldersIdx = cellfun(@(x) settings.(x), allFields);
% Get the list of selected folders
selectedFolders = fullFolderPaths(selectedFoldersIdx==1);
% Display the selected folders
fprintf('Selected Folders to Process:\n');
disp(selectedFolders);
else
fprintf('No folders selected.\n');
end
relativeFolderPaths = strrep(selectedFolders, sioe_labor_path, '');
disp(['Start to process ',num2str(numel(relativeFolderPaths)), ' folder in the directory']);
for f = 1:numel(selectedFolders)
folder = selectedFolders{n};
relfolder = relativeFolderPaths{f};
% Get list of all files in the specified folder and subfolders
fileList = dir(folder);
if isempty(fileList(~[fileList.isdir]))
continue
end
matches = regexp(folder, '\d+km', 'match');
% Check if a match was found
if ~isempty(matches)
length_from_foldername_km = matches{1}; % Extract the first match
length_from_foldername_km = strrep(length_from_foldername_km,'km','');
disp(['The length is: ', length_from_foldername_km]);
else
disp('No length information found in the folder name.');
end
% Loop through each file and rename if necessary
for i = 1:length(fileList)
oldName = fileList(i).name;
% Use regex to find and remove any prefix before the date string
newName = regexprep(oldName, '^[^\d]*(\d{8}_\d{6}.*)', '$1');
% Insert an underscore before "PAM" if missing
newName = regexprep(newName, '(\d{8}_\d{6})(PAM)', '$1_PAM');
% Rename the file only if a change was made
if ~strcmp(oldName, newName)
movefile(fullfile(fileList(i).folder, oldName), fullfile(fileList(i).folder, newName));
fprintf('Renamed: %s -> %s\n', oldName, newName);
end
end
% Get new list of all files in the specified folder and subfolders, we
% renamed files so we need to get the new filenames here to work on :-)
fileList = dir(folder);
% Initialize lists to store DataStorage objects based on size
big_wh_list = {}; % For large DataStorage objects
big_wh_filename = {}; % Corresponding filenames for large objects
small_wh_list = {}; % For small DataStorage objects
small_wh_filename = {}; % Corresponding filenames for small objects
% Loop through each file and categorize based on the presence of 'wh' in the filename
for i = 1:length(fileList)
fileName = fileList(i).name;
if contains(fileName, 'wh')
% Load DataStorage object from file
wh = load(fullfile(fileList(i).folder, fileName));
wh = wh.obj;
% Classify as big or small based on dimensions
if isa(wh, 'DataStorage')
if prod(wh.dim) > 2
big_wh_list{end+1} = wh;
big_wh_filename{end+1} = fileName;
else
small_wh_list{end+1} = wh;
small_wh_filename{end+1} = fileName;
end
end
end
end
% Aggregate unique parameters across all large DataStorage objects
params_merge = struct;
for c = 1:numel(big_wh_list)
fnames = fieldnames(big_wh_list{c}.parameter);
for fn = 1:numel(fnames)
% Initialize field if not already present
if ~isfield(params_merge, fnames{fn})
params_merge.(fnames{fn}) = [];
end
% Merge unique parameter values into params_merge
a = big_wh_list{c}.parameter.(fnames{fn}).values;
b = params_merge.(fnames{fn});
vals_to_add = setdiff(a, b); % New values in a that aren't in b
b = sort([b, vals_to_add]); % Combine and sort values
params_merge.(fnames{fn}) = b;
end
end
% Process each large DataStorage object
for w = 1:numel(big_wh_list)
wh = big_wh_list{w};
% Extract date and time for filename generation
datebody = regexp(big_wh_filename{w}, '^\d{8}_\d{6}', 'match', 'once');
% Get the total number of linear indices
totalIndices = wh.getLastLinIndice;
% Initialize the waitbar
h = waitbar(0, 'Processing DataStorage...');
% Loop over each linear index in DataStorage
for i = 1:wh.getLastLinIndice
% Update the waitbar with the current progress
waitbar(i / totalIndices, h, sprintf('Folder: %s...\n %d of %d', string(strrep(strrep(relfolder, '\', '/'),'_',' ')), i, totalIndices));
% Initialize record struct for each entry and flag for non-empty data
measurementStruct = struct();
recordIsFilled = false;
% Loop over each storage within DataStorage and gather data
storage_names = fieldnames(wh.sto);
for s = 1:length(storage_names)
% Retrieve physical values, parameter names, and stored value
[configStruct, stored_value] = wh.getPhysAndValueByLinIndex(storage_names{s}, i);
measurementStruct.(storage_names{s}) = stored_value;
if ~isempty(stored_value)
recordIsFilled = true; % Mark as filled if value is present
end
end
[configStruct.precomp_amp_max,configStruct.v_bias_for_pam] = getBias(configStruct.duobinary,configStruct.M);
isMPI = isfield(measurementStruct,'i_power');
% Process record if it contains *any* data
if recordIsFilled
if ~isMPI
% Generate filenames with conditionally formatted parameters
% Format the L parameter value (show decimal only if non-zero)
if configStruct.lambda == floor(configStruct.lambda)
L_str = sprintf('%.0f',configStruct.lambda); % No decimal part
else
L_str = sprintf('%.1f', configStruct.lambda); % Include one decimal place
end
% Synthesize filename base with placeholders for storage types
fbody_tx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%d', datebody, ...
configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, 0);
fbody_tx = strrep(fbody_tx, '.', '_'); % Replace decimal point with underscore
if configStruct.rop_atten == round(configStruct.rop_atten)
fbody_rx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%d', datebody, ...
configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, configStruct.rop_atten);
else
fbody_rx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%.1f', datebody, ...
configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, configStruct.rop_atten);
end
fbody_rx = strrep(fbody_rx, '.', '_');
elseif isMPI
fbody_tx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%d', datebody, ...
configStruct.M, configStruct.bitrate, configStruct.duobinary, 0);
fbody_tx = strrep(fbody_tx, '.', '_'); % Replace decimal point with underscore
if configStruct.interference_atten == round(configStruct.interference_atten)
fbody_rx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%d', datebody, ...
configStruct.M, configStruct.bitrate, configStruct.duobinary, configStruct.interference_atten);
else
fbody_rx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%.1f', datebody, ...
configStruct.M, configStruct.bitrate, configStruct.duobinary, configStruct.interference_atten);
end
fbody_rx = strrep(fbody_rx, '.', '_'); % Replace decimal point with underscore
end
% Check existence of different file types (bits, symbols, raw signal, rx signal)
% BIT SEQUENCE
fn_bits = [filesep, fbody_tx, '_bits.mat'];
fp_bits = fullfile([folder, fn_bits]);
if exist(fp_bits, "file") == 2
fn_bits_rel = [relfolder, fn_bits];
else
warning(['Bits not found at: ', fn_bits]);
end
% SYMBOL SEQUENCE
fn_symbols = [filesep, fbody_tx, '_symbols.mat'];
fp_symbols = fullfile([folder, fn_symbols]);
if exist(fp_symbols, "file") == 2
fn_symbols_rel = [relfolder, fn_symbols];
else
warning(['Symbols not found at: ', fn_symbols]);
end
% RAW RX SIGNAL
fn_rxraw = [filesep, fbody_rx, '_raw_signal.mat'];
fp_rxraw = fullfile([folder, fn_rxraw]);
missing_raw_flag = 1; % Initialize as missing
if exist(fp_rxraw, "file") == 2
fn_rxraw_rel = [relfolder, fn_rxraw];
missing_raw_flag = 0;
end
% SYNCHRONIZED RX SIGNAL
fn_rxtsynch = [filesep, fbody_rx, '_rx_signal.mat'];
fp_rxtsynch = fullfile([folder, fn_rxtsynch]);
fn_rxtsynch_rel = [];
if exist(fp_rxtsynch, "file") == 2
fn_rxtsynch_rel = [relfolder, fn_rxtsynch];
matObj = matfile([folder, fn_rxtsynch]);
% If RX signal actually contains raw signal, handle as necessary
if isprop(matObj, 'Scpe_sig_raw')
sig_rx = load([folder, fn_rxtsynch]);
if missing_raw_flag
% Save as raw signal if original raw signal is missing
Scpe_sig_raw = sig_rx.Scpe_sig_raw;
save([folder, fn_rxraw], "Scpe_sig_raw");
delete([folder, fn_rxtsynch]);
else
% Check if raw and rx signal files are identical, then delete duplicate
sig_raw = load([folder, fn_rxraw]);
if isequal(sig_raw, sig_rx)
delete([folder, fn_rxtsynch]);
end
end
end
elseif exist(fp_rxtsynch, "file") == 0
disp(['RX Signal not found at: ', fn_rxtsynch,' generate and save new one using symbols and raw signal']);
Scpe_sig_raw = load(fp_rxraw);
Scpe_sig_raw = Scpe_sig_raw.Scpe_sig_raw;
Symbols = load(fp_symbols);
Symbols = Symbols.Symbols;
%%%%%% Sample to 2x fsym %%%%%%
Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in",Scpe_sig_raw.fs,"fs_out",2*Symbols.fs);
%%%%%% Sync Rx signal with reference (S is a cell array with all occurences) %%%%%%
[Scpe_sig_syncd,S,isFlipped] = Scpe_sig_resampled.tsynch("reference",Symbols,"fs_ref",Symbols.fs);
%%%%% Plot and Save Routines: SAVE RECEIVED SIGNALS %%%%%%%%%%%%%%%%%%%%%%%%%
save(fp_rxtsynch,"S");
if exist(fp_rxtsynch, "file") == 0
warndlg('Something still wromg... No rx file here but we just generated it from raw signal');
end
fn_rxtsynch_rel = [relfolder, fn_rxtsynch];
elseif missing_raw_flag
warndlg('No RAW or RX signal found! This is bad!');
% look at measurementStruct and configStruct
end
else
disp('Record not filled?')
% look at measurementStruct and configStruct
end
% Call the duplicate check function
if ~isempty(fn_rxtsynch_rel)
exists = db.checkIfRunExists('Runs', 'rx_sync_path', fn_rxtsynch_rel);
end
if ~exists
% Table 1: Append to Runs
newRun = db.tables.Runs; % Get the existing table structure (an empty table)
newRun = struct(...
'run_id', NaN, ... % Auto-increment, leave empty
'date_of_run', datetime(datebody, 'InputFormat', 'yyyyMMdd_HHmmss'), ...
'tx_bits_path', fn_bits_rel, ...
'tx_symbols_path', fn_symbols_rel, ...
'rx_sync_path', fn_rxtsynch_rel, ...
'rx_raw_path', fn_rxraw_rel, ...
'filename', fbody_rx ...
);
% Append the new row to the Runs table and get the generated run ID
run_id = db.appendToTable('Runs', newRun);
if isMPI
assert(configStruct.interference_atten==measurementStruct.voa.value(4),'MPI attuation differs between voa state and desired config from simulation loop.');
interference_attenuation = configStruct.interference_atten;
interference_path_length = 2;
power_mpi_interference = measurementStruct.voa.power_state(4);
power_mpi_signal = measurementStruct.voa.power_state(3);
rop_attenuation = 0;
wavelength = 1310;
fiber_length = 1;
else
interference_attenuation = NaN;
interference_path_length = NaN;
power_mpi_interference = NaN;
power_mpi_signal = NaN;
rop_attenuation = configStruct.rop_atten;
wavelength = configStruct.lambda;
fiber_length = str2double(length_from_foldername_km);
end
% Table 2: Append to Configurations
newConfig = db.tables.Configurations; % Get the existing table structure (an empty table)
newConfig = struct(...
'configuration_id', NaN, ... % Auto-increment, leave empty
'run_id', run_id, ... % Foreign key from Runs
'unique_elab_id', "20241028-dea635ef776cd18270922ba0e52c65831ff7699f", ... % Set unique_elab_id as needed
'bitrate', configStruct.bitrate, ...
'symbolrate', floor(configStruct.bitrate * 1e-9 / log2(configStruct.M)) * 1e9, ... % Calculate symbolrate if available
'pam_level', configStruct.M, ...
'db_mode', configStruct.duobinary, ... % Assuming db_mode corresponds to duobinary mode
'v_bias', configStruct.v_bias_for_pam, ...
'v_awg', 2.7, ...
'precomp_amp', configStruct.precomp_amp_max, ...
'rop_attenuation', rop_attenuation, ...
'wavelength', wavelength, ...
'fiber_length', fiber_length, ...
'is_mpi', isMPI, ... % Set false for no MPI, change as needed
'interference_path_length', interference_path_length, ... % Set NaN if not applicable
'interference_attenuation', interference_attenuation ... % Set NaN if not applicable
);
% Append the new row to the Configurations table
db.appendToTable('Configurations', newConfig);
% Table 3: Append to Measurements
newMeas = db.tables.Measurements; % Get the existing table structure (an empty table)
newMeas = struct(...
'measurement_id', NaN, ... % Auto-increment, leave empty
'run_id', run_id, ... % Foreign key from Runs
'power_laser', measurementStruct.exfo.cur_power, ...
'power_rop', measurementStruct.rop, ...
'power_pd_in', measurementStruct.pd_in, ...
'power_mpi_interference', power_mpi_interference, ...
'power_mpi_signal', power_mpi_signal, ...
'voa_class', measurementStruct.voa, ...
'pdfa_class', measurementStruct.pdfa, ...
'laser_class', measurementStruct.exfo ...
);
% Append the new row to the Measurements table
db.appendToTable('Measurements', newMeas);
% % Table 4: Append to Bers
% [ber, structure, settings] = getBers(configStruct,measurementStruct);
% for t = 1:numel(ber)
%
% if iscell(ber(t))
% ber_ = ber(t);
% ber_ = ber_{1};
% else
% ber_ = ber(t);
% end
%
% if ber_~=-1
%
% newBer = struct(...
% 'ber_id', NaN,...
% 'run_id', run_id,...
% 'processing_structure', structure(t),...
% 'processing_settings', settings(t),...
% 'ber', jsonencode(ber_)...
% );
%
% db.appendToTable('BERs', newBer);
%
% end
%
% end
end
end
end
end
function [ber, structure, settings] = getBers(configStruct,measurementStruct)
if configStruct.duobinary == 0
structure(1) = "vnle";
settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1);
ber(1) = measurementStruct.ber_vnle;
structure(2) = "vnle -> remove DC from error ""Noi{s}.signal = Noi{s}.signal - mean(Noi{s}.signal);"" -> burg(error) -> pf -> mlse";
settings(2) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1);
ber(2) = measurementStruct.ber_vnle_mlse;
elseif configStruct.duobinary == 1
structure(1) = "tx: duobinary precode; rx: db target -> mlse -> modulo";
settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1);
ber(1) = measurementStruct.ber_db;
elseif configStruct.duobinary == 2
structure(1) = "tx: duobinary precode -> encode; rx: db target -> mlse as decoder -> modulo";
settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1);
ber(1) = measurementStruct.ber_db;
end
end
function [precomp_amp_max,v_bias_for_pam] = getBias(db,M)
if db == 1
ffe_only = 0;
postfilter_approach = 0;
db_channel_approach = 1;
db_coding_approach = 0;
db_precode = db_coding_approach || db_channel_approach;
if M == 4
pulsef=1;
precomp_amp_max = -50;
v_bias_for_pam = 2.3;
pulsef = 1;
elseif M == 6
pulsef=0;
precomp_amp_max = -50;
v_bias_for_pam = 2.3;
pulsef = 1;
elseif M == 8
pulsef=0;
precomp_amp_max = -50;
v_bias_for_pam=2.6;
pulsef = 0;
end
elseif db == 2
ffe_only = 0;
postfilter_approach = 0;
db_channel_approach = 0;
db_coding_approach = 1;
db_precode = db_coding_approach || db_channel_approach;
if M == 4
pulsef=1;
precomp_amp_max = -38;
v_bias_for_pam = 2.8;
pulsef = 1;
elseif M == 6
pulsef=0;
precomp_amp_max = -38;
v_bias_for_pam = 2.8;
pulsef = 1;
elseif M == 8
pulsef=0;
precomp_amp_max = -38;
v_bias_for_pam = 2.8;
pulsef = 1;
end
elseif db == 0
ffe_only = 0;
postfilter_approach = 1;
db_channel_approach = 0;
db_coding_approach = 0;
db_precode = db_coding_approach || db_channel_approach;
if M == 4
pulsef=1;
precomp_amp_max = -37;
v_bias_for_pam = 2.3;
pulsef = 1;
elseif M == 6
pulsef=0;
precomp_amp_max = -34;
v_bias_for_pam = 2.3;
pulsef = 1;
elseif M == 8
pulsef=0;
precomp_amp_max = -34;
v_bias_for_pam=2.6;
pulsef = 0;
end
end
end