% Connect to SQLite database pathToDB = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor\silas_labor.db'; % Update the path as needed db = DBHandler("pathToDB",pathToDB); % main file path sioe_labor_path = 'C:\Users\Silas\Documents\MATLAB\Datensätze\sioe_labor'; % Get list of all folders (including subfolders) within sioe_labor_path folderList = dir(fullfile(sioe_labor_path, '**', '*')); % Filter to include only directories and exclude '.' and '..' folderNames = {folderList([folderList.isdir]).name}; folderPaths = {folderList([folderList.isdir]).folder}; % Get the full paths folderPaths = folderPaths(~ismember(folderNames, {'.', '..'})); folderNames = folderNames(~ismember(folderNames, {'.', '..'})); % Combine folder names with paths fullFolderPaths = flip(fullfile(folderPaths, folderNames)); only_mpi=0; if only_mpi fullFolderPaths = fullFolderPaths(contains(fullFolderPaths,"mpi")); end relativeFolderPaths = strrep(fullFolderPaths, sioe_labor_path, ''); disp(['Start to process ',num2str(numel(relativeFolderPaths)), ' folder in the directory']); for f = 1:numel(fullFolderPaths) folder = fullFolderPaths{f}; relfolder = relativeFolderPaths{f}; % Get list of all files in the specified folder and subfolders fileList = dir(folder); if isempty(fileList(~[fileList.isdir])) continue end matches = regexp(folder, '\d+km', 'match'); % Check if a match was found if ~isempty(matches) length_from_foldername_km = matches{1}; % Extract the first match length_from_foldername_km = strrep(length_from_foldername_km,'km',''); disp(['The length is: ', length_from_foldername_km]); else disp('No length information found in the folder name.'); end % Loop through each file and rename if necessary for i = 1:length(fileList) oldName = fileList(i).name; % Use regex to find and remove any prefix before the date string newName = regexprep(oldName, '^[^\d]*(\d{8}_\d{6}.*)', '$1'); % Insert an underscore before "PAM" if missing newName = regexprep(newName, '(\d{8}_\d{6})(PAM)', '$1_PAM'); % Rename the file only if a change was made if ~strcmp(oldName, newName) movefile(fullfile(fileList(i).folder, oldName), fullfile(fileList(i).folder, newName)); fprintf('Renamed: %s -> %s\n', oldName, newName); end end % Get new list of all files in the specified folder and subfolders, we % renamed files so we need to get the new filenames here to work on :-) fileList = dir(folder); % Initialize lists to store DataStorage objects based on size big_wh_list = {}; % For large DataStorage objects big_wh_filename = {}; % Corresponding filenames for large objects small_wh_list = {}; % For small DataStorage objects small_wh_filename = {}; % Corresponding filenames for small objects % Loop through each file and categorize based on the presence of 'wh' in the filename for i = 1:length(fileList) fileName = fileList(i).name; if contains(fileName, 'wh') % Load DataStorage object from file wh = load(fullfile(fileList(i).folder, fileName)); wh = wh.obj; % Classify as big or small based on dimensions if isa(wh, 'DataStorage') if prod(wh.dim) > 2 big_wh_list{end+1} = wh; big_wh_filename{end+1} = fileName; else small_wh_list{end+1} = wh; small_wh_filename{end+1} = fileName; end end end end % Aggregate unique parameters across all large DataStorage objects params_merge = struct; for c = 1:numel(big_wh_list) fnames = fieldnames(big_wh_list{c}.parameter); for f = 1:numel(fnames) % Initialize field if not already present if ~isfield(params_merge, fnames{f}) params_merge.(fnames{f}) = []; end % Merge unique parameter values into params_merge a = big_wh_list{c}.parameter.(fnames{f}).values; b = params_merge.(fnames{f}); vals_to_add = setdiff(a, b); % New values in a that aren't in b b = sort([b, vals_to_add]); % Combine and sort values params_merge.(fnames{f}) = b; end end % Process each large DataStorage object for w = 1:numel(big_wh_list) wh = big_wh_list{w}; % Extract date and time for filename generation datebody = regexp(big_wh_filename{w}, '^\d{8}_\d{6}', 'match', 'once'); % Get the total number of linear indices totalIndices = wh.getLastLinIndice; % Initialize the waitbar h = waitbar(0, 'Processing DataStorage...'); % Loop over each linear index in DataStorage for i = 1:wh.getLastLinIndice % Update the waitbar with the current progress waitbar(i / totalIndices, h, sprintf('Folder: %s...\n %d of %d', string(strrep(strrep(relfolder, '\', '/'),'_',' ')), i, totalIndices)); % Initialize record struct for each entry and flag for non-empty data measurementStruct = struct(); recordIsFilled = false; % Loop over each storage within DataStorage and gather data storage_names = fieldnames(wh.sto); for s = 1:length(storage_names) % Retrieve physical values, parameter names, and stored value [configStruct, stored_value] = wh.getPhysAndValueByLinIndex(storage_names{s}, i); measurementStruct.(storage_names{s}) = stored_value; if ~isempty(stored_value) recordIsFilled = true; % Mark as filled if value is present end end [configStruct.precomp_amp_max,configStruct.v_bias_for_pam] = getBias(configStruct.duobinary,configStruct.M); isMPI = isfield(measurementStruct,'i_power'); % Process record if it contains data if recordIsFilled if ~isMPI % Generate filenames with conditionally formatted parameters % Format the L parameter value (show decimal only if non-zero) if configStruct.lambda == floor(configStruct.lambda) L_str = sprintf('%.0f',configStruct.lambda); % No decimal part else L_str = sprintf('%.1f', configStruct.lambda); % Include one decimal place end % Synthesize filename base with placeholders for storage types fbody_tx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%d', datebody, ... configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, 0); fbody_tx = strrep(fbody_tx, '.', '_'); % Replace decimal point with underscore fbody_rx = sprintf('%s_PAM_%d_L_%s_R_%d_DB_%d_ROP_%d', datebody, ... configStruct.M, L_str, configStruct.bitrate, configStruct.duobinary, configStruct.rop_atten); fbody_rx = strrep(fbody_rx, '.', '_'); elseif isMPI fbody_tx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%d', datebody, ... configStruct.M, configStruct.bitrate, configStruct.duobinary, 0); fbody_tx = strrep(fbody_tx, '.', '_'); % Replace decimal point with underscore fbody_rx = sprintf('%s_PAM_%d_R_%d_DB_%d_I_atten_%d', datebody, ... configStruct.M, configStruct.bitrate, configStruct.duobinary, configStruct.interference_atten); fbody_rx = strrep(fbody_rx, '.', '_'); % Replace decimal point with underscore end % Check existence of different file types (bits, symbols, raw signal, rx signal) % BIT SEQUENCE fn_bits = [filesep, fbody_tx, '_bits.mat']; fp_bits = fullfile([folder, fn_bits]); if exist(fp_bits, "file") == 2 fn_bits_rel = [relfolder, fn_bits]; else warning(['Bits not found at: ', fn_bits]); end % SYMBOL SEQUENCE fn_symbols = [filesep, fbody_tx, '_symbols.mat']; fp_symbols = fullfile([folder, fn_symbols]); if exist(fp_symbols, "file") == 2 fn_symbols_rel = [relfolder, fn_symbols]; else warning(['Symbols not found at: ', fn_symbols]); end % RAW RX SIGNAL fn_rxraw = [filesep, fbody_rx, '_raw_signal.mat']; fp_rxraw = fullfile([folder, fn_rxraw]); missing_raw_flag = 1; % Initialize as missing if exist(fp_rxraw, "file") == 2 fn_rxraw_rel = [relfolder, fn_rxraw]; missing_raw_flag = 0; end % SYNCHRONIZED RX SIGNAL fn_rxtsynch = [filesep, fbody_rx, '_rx_signal.mat']; fp_rxtsynch = fullfile([folder, fn_rxtsynch]); if exist(fp_rxtsynch, "file") == 2 fn_rxtsynch_rel = [relfolder, fn_rxtsynch]; matObj = matfile([folder, fn_rxtsynch]); % If RX signal actually contains raw signal, handle as necessary if isprop(matObj, 'Scpe_sig_raw') sig_rx = load([folder, fn_rxtsynch]); if missing_raw_flag % Save as raw signal if original raw signal is missing Scpe_sig_raw = sig_rx.Scpe_sig_raw; save([folder, fn_rxraw], "Scpe_sig_raw"); delete([folder, fn_rxtsynch]); else % Check if raw and rx signal files are identical, then delete duplicate sig_raw = load([folder, fn_rxraw]); if isequal(sig_raw, sig_rx) delete([folder, fn_rxtsynch]); end end end elseif missing_raw_flag warning(['RX Signal not found at: ', fn_rxtsynch]); end end % Call the duplicate check function exists = db.checkIfRunExists('Runs', 'rx_sync_path', fn_rxtsynch_rel); if ~exists % Table 1: Append to Runs newRun = db.tables.Runs; % Get the existing table structure (an empty table) newRun = struct(... 'run_id', NaN, ... % Auto-increment, leave empty 'date_of_run', datetime(datebody, 'InputFormat', 'yyyyMMdd_HHmmss'), ... 'tx_bits_path', fn_bits_rel, ... 'tx_symbols_path', fn_symbols_rel, ... 'rx_sync_path', fn_rxtsynch_rel, ... 'rx_raw_path', fn_rxraw_rel, ... 'filename', fbody_rx ... ); % Append the new row to the Runs table and get the generated run ID run_id = db.appendToTable('Runs', newRun); if isMPI assert(configStruct.interference_atten==measurementStruct.voa.value(4),'MPI attuation differs between voa state and desired config from simulation loop.'); interference_attenuation = configStruct.interference_atten; interference_path_length = 2; power_mpi_interference = measurementStruct.voa.power_state(4); power_mpi_signal = measurementStruct.voa.power_state(3); rop_attenuation = 0; wavelength = 1310; fiber_length = 1; else interference_attenuation = NaN; interference_path_length = NaN; power_mpi_interference = NaN; power_mpi_signal = NaN; rop_attenuation = configStruct.rop_atten; wavelength = configStruct.lambda; fiber_length = str2double(length_from_foldername_km); end % Table 2: Append to Configurations newConfig = db.tables.Configurations; % Get the existing table structure (an empty table) newConfig = struct(... 'configuration_id', NaN, ... % Auto-increment, leave empty 'run_id', run_id, ... % Foreign key from Runs 'unique_elab_id', "20241028-dea635ef776cd18270922ba0e52c65831ff7699f", ... % Set unique_elab_id as needed 'bitrate', configStruct.bitrate, ... 'symbolrate', floor(configStruct.bitrate * 1e-9 / log2(configStruct.M)) * 1e9, ... % Calculate symbolrate if available 'pam_level', configStruct.M, ... 'db_mode', configStruct.duobinary, ... % Assuming db_mode corresponds to duobinary mode 'v_bias', configStruct.v_bias_for_pam, ... 'v_awg', 2.7, ... 'precomp_amp', configStruct.precomp_amp_max, ... 'rop_attenuation', rop_attenuation, ... 'wavelength', wavelength, ... 'fiber_length', fiber_length, ... 'is_mpi', isMPI, ... % Set false for no MPI, change as needed 'interference_path_length', interference_path_length, ... % Set NaN if not applicable 'interference_attenuation', interference_attenuation ... % Set NaN if not applicable ); % Append the new row to the Configurations table db.appendToTable('Configurations', newConfig); % Table 3: Append to Measurements newMeas = db.tables.Measurements; % Get the existing table structure (an empty table) newMeas = struct(... 'measurement_id', NaN, ... % Auto-increment, leave empty 'run_id', run_id, ... % Foreign key from Runs 'power_laser', measurementStruct.exfo.cur_power, ... 'power_rop', measurementStruct.rop, ... 'power_pd_in', measurementStruct.pd_in, ... 'power_mpi_interference', power_mpi_interference, ... 'power_mpi_signal', power_mpi_signal, ... 'voa_class', measurementStruct.voa, ... 'pdfa_class', measurementStruct.pdfa, ... 'laser_class', measurementStruct.exfo ... ); % Append the new row to the Measurements table db.appendToTable('Measurements', newMeas); % Table 4: Append to Bers [ber, structure, settings] = getBers(configStruct,measurementStruct); for t = 1:numel(ber) if iscell(ber(t)) ber_ = ber(t); ber_ = ber_{1}; else ber_ = ber(t); end if ber_~=-1 newBer = struct(... 'ber_id', NaN,... 'run_id', run_id,... 'processing_structure', structure(t),... 'processing_settings', settings(t),... 'ber', jsonencode(ber_)... ); db.appendToTable('BERs', newBer); end end end end end end function [ber, structure, settings] = getBers(configStruct,measurementStruct) if configStruct.duobinary == 0 structure(1) = "vnle"; settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(1) = measurementStruct.ber_vnle; structure(2) = "vnle -> remove DC from error ""Noi{s}.signal = Noi{s}.signal - mean(Noi{s}.signal);"" -> burg(error) -> pf -> mlse"; settings(2) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(2) = measurementStruct.ber_vnle_mlse; elseif configStruct.duobinary == 1 structure(1) = "tx: duobinary precode; rx: db target -> mlse -> modulo"; settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(1) = measurementStruct.ber_db; elseif configStruct.duobinary == 2 structure(1) = "tx: duobinary precode -> encode; rx: db target -> mlse as decoder -> modulo"; settings(1) = EQ("Ne",[50,7,7],"Nb",[0,0,0],"training_length",4096*2,"training_loops",5,"dd_loops",5,"K",2,"DCmu",0.0,"DDmu",[0.0004 0.0004 0.0004 0.0004 ],"DFEmu",0.005,"FFEmu",0,"plotfinal",0,"ideal_dfe",1); ber(1) = measurementStruct.ber_db; end end function [precomp_amp_max,v_bias_for_pam] = getBias(db,M) if db == 1 ffe_only = 0; postfilter_approach = 0; db_channel_approach = 1; db_coding_approach = 0; db_precode = db_coding_approach || db_channel_approach; if M == 4 pulsef=1; precomp_amp_max = -50; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 6 pulsef=0; precomp_amp_max = -50; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 8 pulsef=0; precomp_amp_max = -50; v_bias_for_pam=2.6; pulsef = 0; end elseif db == 2 ffe_only = 0; postfilter_approach = 0; db_channel_approach = 0; db_coding_approach = 1; db_precode = db_coding_approach || db_channel_approach; if M == 4 pulsef=1; precomp_amp_max = -38; v_bias_for_pam = 2.8; pulsef = 1; elseif M == 6 pulsef=0; precomp_amp_max = -38; v_bias_for_pam = 2.8; pulsef = 1; elseif M == 8 pulsef=0; precomp_amp_max = -38; v_bias_for_pam = 2.8; pulsef = 1; end elseif db == 0 ffe_only = 0; postfilter_approach = 1; db_channel_approach = 0; db_coding_approach = 0; db_precode = db_coding_approach || db_channel_approach; if M == 4 pulsef=1; precomp_amp_max = -37; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 6 pulsef=0; precomp_amp_max = -34; v_bias_for_pam = 2.3; pulsef = 1; elseif M == 8 pulsef=0; precomp_amp_max = -34; v_bias_for_pam=2.6; pulsef = 0; end end end