% Connect to SQLite database pathToDB = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor\silas_labor.db'; % Update the path as needed db = DBHandler("pathToDB",pathToDB); % main file path sioe_labor_path = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor'; % Get list of all folders (including subfolders) within sioe_labor_path folderList = dir(fullfile(sioe_labor_path, '**', '*')); % Filter to include only directories and exclude '.' and '..' folderNames = {folderList([folderList.isdir]).name}; folderPaths = {folderList([folderList.isdir]).folder}; % Get the full paths folderPaths = folderPaths(~ismember(folderNames, {'.', '..'})); folderNames = folderNames(~ismember(folderNames, {'.', '..'})); % Combine folder names with paths fullFolderPaths = flip(fullfile(folderPaths, folderNames)); % process only MPI? only_mpi=0; if only_mpi fullFolderPaths = fullFolderPaths(contains(fullFolderPaths,"mpi")); end % Shorten folder paths to display only the folder name after the last filesep displayFolderPaths = cellfun(@(x) x(find(x == filesep, 1, 'last') + 1 : end), fullFolderPaths, 'UniformOutput', false); % Create checkbox settings for each folder path numFolders = numel(fullFolderPaths); checkboxSettings = cell(1, 2 * numFolders); for i = 1:numFolders checkboxSettings{2*i-1} = {displayFolderPaths{i}; sprintf('Folder_%d', i)}; checkboxSettings{2*i} = true; % Set false for unchecked by default end % Create settings dialog using settingsdlg [settings, button] = settingsdlg( ... 'Description', 'Select Folders to Process:', ... 'title', 'Folder Selection', ... checkboxSettings{:}); % Check if the user pressed OK if strcmp(button, 'OK') % Get all the field names from settings allFields = fieldnames(settings); % Determine which folders were selected selectedFoldersIdx = cellfun(@(x) settings.(x), allFields); % Get the list of selected folders selectedFolders = fullFolderPaths(selectedFoldersIdx==1); % Display the selected folders fprintf('Selected Folders to Process:\n'); disp(selectedFolders); else fprintf('No folders selected.\n'); end relativeFolderPaths = strrep(selectedFolders, sioe_labor_path, ''); disp(['Start to process ',num2str(numel(relativeFolderPaths)), ' folder in the directory']); for f = 1:numel(selectedFolders) folder = selectedFolders{n}; relfolder = relativeFolderPaths{f}; % Get list of all files in the specified folder and subfolders fileList = dir(folder); if isempty(fileList(~[fileList.isdir])) continue end matches = regexp(folder, '\d+km', 'match'); % Check if a match was found if ~isempty(matches) length_from_foldername_km = matches{1}; % Extract the first match length_from_foldername_km = strrep(length_from_foldername_km,'km',''); disp(['The length is: ', length_from_foldername_km]); else disp('No length information found in the folder name.'); end % Loop through each file and rename if necessary for i = 1:length(fileList) oldName = fileList(i).name; % Use regex to find and remove any prefix before the date string newName = regexprep(oldName, '^[^\d]*(\d{8}_\d{6}.*)', '$1'); % Insert an underscore before "PAM" if missing newName = regexprep(newName, '(\d{8}_\d{6})(PAM)', '$1_PAM'); % Rename the file only if a change was made if ~strcmp(oldName, newName) movefile(fullfile(fileList(i).folder, oldName), fullfile(fileList(i).folder, newName)); fprintf('Renamed: %s -> %s\n', oldName, newName); end end % Get new list of all files in the specified folder and subfolders, we % renamed files so we need to get the new filenames here to work on :-) fileList = dir(folder); % Initialize lists to store DataStorage objects based on size big_wh_list = {}; % For large DataStorage objects big_wh_filename = {}; % Corresponding filenames for large objects small_wh_list = {}; % For small DataStorage objects small_wh_filename = {}; % Corresponding filenames for small objects % Loop through each file and categorize based on the presence of 'wh' in the filename for i = 1:length(fileList) fileName = fileList(i).name; if contains(fileName, 'wh') % Load DataStorage object from file wh = load(fullfile(fileList(i).folder, fileName)); wh = wh.obj; % Classify as big or small based on dimensions if isa(wh, 'DataStorage') if prod(wh.dim) > 2 big_wh_list{end+1} = wh; big_wh_filename{end+1} = fileName; else small_wh_list{end+1} = wh; small_wh_filename{end+1} = fileName; end end end end % Aggregate unique parameters across all large DataStorage objects params_merge = struct; for c = 1:numel(big_wh_list) fnames = fieldnames(big_wh_list{c}.parameter); for fn = 1:numel(fnames) % Initialize field if not already present if ~isfield(params_merge, fnames{fn}) params_merge.(fnames{fn}) = []; end % Merge unique parameter values into params_merge a = big_wh_list{c}.parameter.(fnames{fn}).values; b = params_merge.(fnames{fn}); vals_to_add = setdiff(a, b); % New values in a that aren't in b b = sort([b, vals_to_add]); % Combine and sort values params_merge.(fnames{fn}) = b; end end % Process each large DataStorage object for w = 1:numel(big_wh_list) wh = big_wh_list{w}; % Extract date and time for filename generation datebody = regexp(big_wh_filename{w}, '^\d{8}_\d{6}', 'match', 'once'); % Get the total number of linear indices totalIndices = wh.getLastLinIndice; % Initialize the waitbar h = waitbar(0, 'Processing DataStorage...'); % Loop over each linear index in DataStorage for i = 1:wh.getLastLinIndice % Update the waitbar with the current progress waitbar(i / totalIndices, h, sprintf('Folder: %s...\n %d of %d', string(strrep(strrep(relfolder, '\', '/'),'_',' ')), i, totalIndices)); % Initialize record struct for each entry and flag for non-empty data measurementStruct = struct(); recordIsFilled = false; % Loop over each storage within DataStorage and gather data storage_names = fieldnames(wh.sto); for s = 1:length(storage_names) % Retrieve physical values, parameter names, and stored value [configStruct, stored_value] = wh.getPhysAndValueByLinIndex(storage_names{s}, i); measurementStruct.(storage_names{s}) = stored_value; if ~isempty(stored_value) recordIsFilled = true; % Mark as filled if value is present end end [configStruct.precomp_amp_max,configStruct.v_bias_for_pam] = getBias(configStruct.duobinary,configStruct.M); isMPI = isfield(measurementStruct,'i_power'); % Process record if it contains *any* data if recordIsFilled if ~isMPI % Generate filenames with conditionally formatted parameters % Format the L parameter value (show decimal only if non-zero) if configStruct.lambda == floor(configStruct.lambda) L_str = sprintf('%.0f',configStruct.lambda); % No decimal part else L_str = sprintf('%.1f', configStruct.lambda); % Include one decimal place end % Synthesize filename base with placeholders for storage types fbody_tx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%d', datebody, ... configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, 0); fbody_tx = strrep(fbody_tx, '.', '_'); % Replace decimal point with underscore if configStruct.rop_atten == round(configStruct.rop_atten) fbody_rx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%d', datebody, ... configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, configStruct.rop_atten); else fbody_rx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%.1f', datebody, ... configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, configStruct.rop_atten); end fbody_rx = strrep(fbody_rx, '.', '_'); elseif isMPI fbody_tx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%d', datebody, ... configStruct.M, configStruct.bitrate, configStruct.duobinary, 0); fbody_tx = strrep(fbody_tx, '.', '_'); % Replace decimal point with underscore if configStruct.interference_atten == round(configStruct.interference_atten) fbody_rx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%d', datebody, ... configStruct.M, configStruct.bitrate, configStruct.duobinary, configStruct.interference_atten); else fbody_rx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%.1f', datebody, ... configStruct.M, configStruct.bitrate, configStruct.duobinary, configStruct.interference_atten); end fbody_rx = strrep(fbody_rx, '.', '_'); % Replace decimal point with underscore end % Check existence of different file types (bits, symbols, raw signal, rx signal) % BIT SEQUENCE fn_bits = [filesep, fbody_tx, '_bits.mat']; fp_bits = fullfile([folder, fn_bits]); if exist(fp_bits, "file") == 2 fn_bits_rel = [relfolder, fn_bits]; else warning(['Bits not found at: ', fn_bits]); end % SYMBOL SEQUENCE fn_symbols = [filesep, fbody_tx, '_symbols.mat']; fp_symbols = fullfile([folder, fn_symbols]); if exist(fp_symbols, "file") == 2 fn_symbols_rel = [relfolder, fn_symbols]; else warning(['Symbols not found at: ', fn_symbols]); end % RAW RX SIGNAL fn_rxraw = [filesep, fbody_rx, '_raw_signal.mat']; fp_rxraw = fullfile([folder, fn_rxraw]); missing_raw_flag = 1; % Initialize as missing if exist(fp_rxraw, "file") == 2 fn_rxraw_rel = [relfolder, fn_rxraw]; missing_raw_flag = 0; end % SYNCHRONIZED RX SIGNAL fn_rxtsynch = [filesep, fbody_rx, '_rx_signal.mat']; fp_rxtsynch = fullfile([folder, fn_rxtsynch]); fn_rxtsynch_rel = []; if exist(fp_rxtsynch, "file") == 2 fn_rxtsynch_rel = [relfolder, fn_rxtsynch]; matObj = matfile([folder, fn_rxtsynch]); % If RX signal actually contains raw signal, handle as necessary if isprop(matObj, 'Scpe_sig_raw') sig_rx = load([folder, fn_rxtsynch]); if missing_raw_flag % Save as raw signal if original raw signal is missing Scpe_sig_raw = sig_rx.Scpe_sig_raw; save([folder, fn_rxraw], "Scpe_sig_raw"); delete([folder, fn_rxtsynch]); else % Check if raw and rx signal files are identical, then delete duplicate sig_raw = load([folder, fn_rxraw]); if isequal(sig_raw, sig_rx) delete([folder, fn_rxtsynch]); end end end elseif exist(fp_rxtsynch, "file") == 0 disp(['RX Signal not found at: ', fn_rxtsynch,' generate and save new one using symbols and raw signal']); Scpe_sig_raw = load(fp_rxraw); Scpe_sig_raw = Scpe_sig_raw.Scpe_sig_raw; Symbols = load(fp_symbols); Symbols = Symbols.Symbols; %%%%%% Sample to 2x fsym %%%%%% Scpe_sig_resampled = Scpe_sig_raw.resample("fs_in",Scpe_sig_raw.fs,"fs_out",2*Symbols.fs); %%%%%% Sync Rx signal with reference (S is a cell array with all occurences) %%%%%% [Scpe_sig_syncd,S,isFlipped] = Scpe_sig_resampled.tsynch("reference",Symbols,"fs_ref",Symbols.fs); %%%%% Plot and Save Routines: SAVE RECEIVED SIGNALS %%%%%%%%%%%%%%%%%%%%%%%%% save(fp_rxtsynch,"S"); if exist(fp_rxtsynch, "file") == 0 warndlg('Something still wromg... No rx file here but we just generated it from raw signal'); end fn_rxtsynch_rel = [relfolder, fn_rxtsynch]; elseif missing_raw_flag warndlg('No RAW or RX signal found! This is bad!'); % look at measurementStruct and configStruct end else disp('Record not filled?') % look at measurementStruct and configStruct end % Call the duplicate check function if ~isempty(fn_rxtsynch_rel) exists = db.checkIfRunExists('Runs', 'rx_sync_path', fn_rxtsynch_rel); end if ~exists % Table 1: Append to Runs newRun = db.tables.Runs; % Get the existing table structure (an empty table) newRun = struct(... 'run_id', NaN, ... % Auto-increment, leave empty 'date_of_run', datetime(datebody, 'InputFormat', 'yyyyMMdd_HHmmss'), ... 'tx_bits_path', fn_bits_rel, ... 'tx_symbols_path', fn_symbols_rel, ... 'rx_sync_path', fn_rxtsynch_rel, ... 'rx_raw_path', fn_rxraw_rel, ... 'filename', fbody_rx ... ); % Append the new row to the Runs table and get the generated run ID run_id = db.appendToTable('Runs', newRun); if isMPI assert(configStruct.interference_atten==measurementStruct.voa.value(4),'MPI attuation differs between voa state and desired config from simulation loop.'); interference_attenuation = configStruct.interference_atten; interference_path_length = 2; power_mpi_interference = measurementStruct.voa.power_state(4); power_mpi_signal = measurementStruct.voa.power_state(3); rop_attenuation = 0; wavelength = 1310; fiber_length = 1; else interference_attenuation = NaN; interference_path_length = NaN; power_mpi_interference = NaN; power_mpi_signal = NaN; rop_attenuation = configStruct.rop_atten; wavelength = configStruct.lambda; fiber_length = str2double(length_from_foldername_km); end % Table 2: Append to Configurations newConfig = db.tables.Configurations; % Get the existing table structure (an empty table) newConfig = struct(... 'configuration_id', NaN, ... % Auto-increment, leave empty 'run_id', run_id, ... % Foreign key from Runs 'unique_elab_id', "20241028-dea635ef776cd18270922ba0e52c65831ff7699f", ... % Set unique_elab_id as needed 'bitrate', configStruct.bitrate, ... 'symbolrate', floor(configStruct.bitrate * 1e-9 / log2(configStruct.M)) * 1e9, ... % Calculate symbolrate if available 'pam_level', configStruct.M, ... 'db_mode', configStruct.duobinary, ... % Assuming db_mode corresponds to duobinary mode 'v_bias', configStruct.v_bias_for_pam, ... 'v_awg', 2.7, ... 'precomp_amp', configStruct.precomp_amp_max, ... 'rop_attenuation', rop_attenuation, ... 'wavelength', wavelength, ... 'fiber_length', fiber_length, ... 'is_mpi', isMPI, ... % Set false for no MPI, change as needed 'interference_path_length', interference_path_length, ... % Set NaN if not applicable 'interference_attenuation', interference_attenuation ... % Set NaN if not applicable ); % Append the new row to the Configurations table db.appendToTable('Configurations', newConfig); % Table 3: Append to Measurements newMeas = db.tables.Measurements; % Get the existing table structure (an empty table) newMeas = struct(... 'measurement_id', NaN, ... % Auto-increment, leave empty 'run_id', run_id, ... % Foreign key from Runs 'power_laser', measurementStruct.exfo.cur_power, ... 'power_rop', measurementStruct.rop, ... 'power_pd_in', measurementStruct.pd_in, ... 'power_mpi_interference', power_mpi_interference, ... 'power_mpi_signal', power_mpi_signal, ... 'voa_class', measurementStruct.voa, ... 'pdfa_class', measurementStruct.pdfa, ... 'laser_class', measurementStruct.exfo ... ); % Append the new row to the Measurements table db.appendToTable('Measurements', newMeas); % % Table 4: Append to Bers % [ber, structure, settings] = getBers(configStruct,measurementStruct); % for t = 1:numel(ber) % % if iscell(ber(t)) % ber_ = ber(t); % ber_ = ber_{1}; % else % ber_ = ber(t); % end % % if ber_~=-1 % % newBer = struct(... % 'ber_id', NaN,... % 'run_id', run_id,... % 'processing_structure', structure(t),... % 'processing_settings', settings(t),... % 'ber', jsonencode(ber_)... % ); % % db.appendToTable('BERs', newBer); % % end % % end end end end end function [ber, structure, settings] = getBers(configStruct,measurementStruct) if configStruct.duobinary == 0 structure(1) = "vnle"; settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(1) = measurementStruct.ber_vnle; structure(2) = "vnle -> remove DC from error ""Noi{s}.signal = Noi{s}.signal - mean(Noi{s}.signal);"" -> burg(error) -> pf -> mlse"; settings(2) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(2) = measurementStruct.ber_vnle_mlse; elseif configStruct.duobinary == 1 structure(1) = "tx: duobinary precode; rx: db target -> mlse -> modulo"; settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(1) = measurementStruct.ber_db; elseif configStruct.duobinary == 2 structure(1) = "tx: duobinary precode -> encode; rx: db target -> mlse as decoder -> modulo"; settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(1) = measurementStruct.ber_db; end end function [precomp_amp_max,v_bias_for_pam] = getBias(db,M) if db == 1 ffe_only = 0; postfilter_approach = 0; db_channel_approach = 1; db_coding_approach = 0; db_precode = db_coding_approach || db_channel_approach; if M == 4 pulsef=1; precomp_amp_max = -50; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 6 pulsef=0; precomp_amp_max = -50; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 8 pulsef=0; precomp_amp_max = -50; v_bias_for_pam=2.6; pulsef = 0; end elseif db == 2 ffe_only = 0; postfilter_approach = 0; db_channel_approach = 0; db_coding_approach = 1; db_precode = db_coding_approach || db_channel_approach; if M == 4 pulsef=1; precomp_amp_max = -38; v_bias_for_pam = 2.8; pulsef = 1; elseif M == 6 pulsef=0; precomp_amp_max = -38; v_bias_for_pam = 2.8; pulsef = 1; elseif M == 8 pulsef=0; precomp_amp_max = -38; v_bias_for_pam = 2.8; pulsef = 1; end elseif db == 0 ffe_only = 0; postfilter_approach = 1; db_channel_approach = 0; db_coding_approach = 0; db_precode = db_coding_approach || db_channel_approach; if M == 4 pulsef=1; precomp_amp_max = -37; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 6 pulsef=0; precomp_amp_max = -34; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 8 pulsef=0; precomp_amp_max = -34; v_bias_for_pam=2.6; pulsef = 0; end end end