work on MPI
This commit is contained in:
Silas Oettinghaus
@@ -0,0 +1,73 @@
|
||||
% === DSP settings ===
|
||||
dsp_options = struct();
|
||||
dsp_options.mode = "run_id";
|
||||
dsp_options.recipe = @mpi_recipe_dev;
|
||||
dsp_options.append_to_db = false;
|
||||
dsp_options.start_occurence = 1;
|
||||
dsp_options.max_occurences = 15;
|
||||
dsp_options.debug_plots = false;
|
||||
|
||||
dsp_options.database_type = "mysql";
|
||||
|
||||
dsp_options.dataBase = "labor";
|
||||
dsp_options.storage_path = "W:\labdata\ECOC Silas\ecoc_2025";
|
||||
|
||||
dsp_options.server = "192.168.178.192";
|
||||
dsp_options.port = 3306;
|
||||
dsp_options.user = "silas";
|
||||
dsp_options.password = "silas";
|
||||
|
||||
db = DBHandler("dataBase", [dsp_options.dataBase],...
|
||||
"type", dsp_options.database_type,...
|
||||
"server", dsp_options.server,...
|
||||
"user", dsp_options.user, "password", dsp_options.password);
|
||||
|
||||
%%
|
||||
fp = QueryFilter();
|
||||
fp.where('Runs','run_id','GREATER_EQUAL',3153);
|
||||
fp.where('Runs', 'symbolrate', 'EQUALS', 72e9); % 72 96 112
|
||||
fp.where('Runs', 'fiber_length', 'EQUALS', 0);
|
||||
% fp.where('Runs', 'interference_path_length', 'EQUALS', 1000);
|
||||
% fp.where('Runs', 'sir', 'LESS_EQUAL', 50);
|
||||
fp.where('Runs', 'db_mode', 'EQUALS', '"no_db"');
|
||||
% fp.where('Runs', 'is_mpi', 'EQUALS', 1);
|
||||
fp.where('Runs', 'pam_level', 'EQUALS', 8);
|
||||
% fp.where('Runs', 'v_bias', 'EQUALS', 2.65);
|
||||
|
||||
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
|
||||
|
||||
% [~, uniqueSirRows] = unique(dataTable.sir, "stable");
|
||||
% dataTable = dataTable(uniqueSirRows, :);
|
||||
|
||||
[~, sortIdx] = sort(dataTable.sir, 'descend');
|
||||
dataTable = dataTable(sortIdx, :);
|
||||
% dataTable = dataTable(1, :);
|
||||
run_ids = dataTable.run_id;
|
||||
|
||||
if isempty(run_ids)
|
||||
error("run_minimal_recipe:MissingRunIds", ...
|
||||
"Set run_ids to one or more known run IDs before running this example.");
|
||||
end
|
||||
|
||||
%% === Warehouse setup ===
|
||||
dsp_options.userParameters = struct();
|
||||
dsp_options.userParameters.block_update = [1,2,4,8,16,32,64,112,224,448,512,1024,2048,2048*2,2048*4];%%logspace(-3.8,-1,22);%[linspace(2,4096,22)];
|
||||
wh = DataStorage(dsp_options.userParameters);
|
||||
|
||||
|
||||
%%
|
||||
n_realizations = (dsp_options.max_occurences - dsp_options.start_occurence + 1);
|
||||
n_userparams = prod(wh.dim);
|
||||
n_run_ids = numel(run_ids);
|
||||
parallel_jobs = n_userparams * n_run_ids;
|
||||
queried_jobs = n_realizations * n_userparams * n_run_ids;
|
||||
|
||||
fprintf("-> [ %d run_id(s) × %d userParam combination(s) = %d parallel job(s) ] × %d realizations = %d total jobs \n", ...
|
||||
n_run_ids, n_userparams, parallel_jobs, n_realizations, queried_jobs);
|
||||
|
||||
%% === Run ===
|
||||
% submitJobs returns the raw per-job results and the filled Warehouse. For a
|
||||
% single run_id and remove_dc = [0, 1], results is a 2-by-1 cell array.
|
||||
[results, wh] = submitJobs(run_ids, dsp_options, processingMode.parallel, ...
|
||||
"wh", wh, ...
|
||||
"waitbar", true);
|
||||
Binary file not shown.
@@ -0,0 +1,360 @@
|
||||
%% Analyze
|
||||
% === DSP settings ===
|
||||
dsp_options = struct();
|
||||
dsp_options.mode = "run_id";
|
||||
dsp_options.recipe = @mpi_recipe_dev;
|
||||
dsp_options.append_to_db = false;
|
||||
dsp_options.start_occurence = 1;
|
||||
dsp_options.max_occurences = 15;
|
||||
dsp_options.debug_plots = false;
|
||||
|
||||
dsp_options.database_type = "mysql";
|
||||
|
||||
dsp_options.dataBase = "labor";
|
||||
dsp_options.storage_path = "W:\labdata\ECOC Silas\ecoc_2025";
|
||||
|
||||
dsp_options.server = "192.168.178.192";
|
||||
dsp_options.port = 3306;
|
||||
dsp_options.user = "silas";
|
||||
dsp_options.password = "silas";
|
||||
|
||||
db = DBHandler("dataBase", [dsp_options.dataBase],...
|
||||
"type", dsp_options.database_type,...
|
||||
"server", dsp_options.server,...
|
||||
"user", dsp_options.user, "password", dsp_options.password);
|
||||
|
||||
%%
|
||||
|
||||
fp = QueryFilter();
|
||||
fp.where('Runs','run_id','GREATER_EQUAL',3153);
|
||||
fp.where('Runs', 'symbolrate', 'EQUALS', 72e9); % 72 96 112
|
||||
fp.where('Runs', 'fiber_length', 'EQUALS', 0);
|
||||
fp.where('Runs', 'interference_path_length', 'EQUALS', 1000);
|
||||
% fp.where('Runs', 'sir', 'LESS_EQUAL', 50);
|
||||
fp.where('Runs', 'db_mode', 'EQUALS', '"no_db"');
|
||||
% fp.where('Runs', 'is_mpi', 'EQUALS', 1);
|
||||
fp.where('Runs', 'pam_level', 'EQUALS', 8);
|
||||
% fp.where('Runs', 'v_bias', 'EQUALS', 2.65);
|
||||
|
||||
[dataTable,~] = db.queryDB(fp, db.getTableFieldNames('Runs'));
|
||||
|
||||
% [~, uniqueSirRows] = unique(dataTable.sir, "stable");
|
||||
% dataTable = dataTable(uniqueSirRows, :);
|
||||
|
||||
[~, sortIdx] = sort(dataTable.sir, 'descend');
|
||||
dataTable = dataTable(sortIdx, :);
|
||||
% dataTable = dataTable(1, :);
|
||||
run_ids = dataTable.run_id;
|
||||
%%
|
||||
|
||||
wh_ = load("C:\Users\Silas\Documents\MATLAB\imdd_simulation\projects\Diss\MPI_revisit\parallelization_analysis\wh_block_update_pam8_combined.mat");
|
||||
wh = wh_.wh;
|
||||
|
||||
% wh_ = load("C:\Users\Silas\Documents\MATLAB\imdd_simulation\projects\Diss\MPI_revisit\parallelization_analysis\wh_block_update_pam4_short_blocks.mat");
|
||||
% wh2 = wh_.wh;
|
||||
%
|
||||
% wh = mergeDataStorages(wh1,wh2);
|
||||
|
||||
%%
|
||||
|
||||
|
||||
storageNames = fieldnames(wh.sto);
|
||||
ber_by_storage = struct();
|
||||
ber_mat_by_storage = struct();
|
||||
|
||||
sir_values = dataTable.sir(:).';
|
||||
block_updates = wh.parameter.block_update.values;
|
||||
storage_parameter_names = cellstr(wh.fn);
|
||||
block_axis = find(strcmp(storage_parameter_names,"block_update"),1);
|
||||
|
||||
if exist('linspecer','file')
|
||||
curve_colors = linspecer(max(numel(storageNames),1));
|
||||
else
|
||||
curve_colors = lines(max(numel(storageNames),1));
|
||||
end
|
||||
|
||||
fec_ber_threshold = 2e-2;%3.8e-3;
|
||||
fit_coeff_by_storage = struct();
|
||||
required_sir_fec_by_storage = struct();
|
||||
for storage_idx = 1:numel(storageNames)
|
||||
storageName = storageNames{storage_idx};
|
||||
fit_coeff_by_storage.(storageName) = cell(1,numel(block_updates));
|
||||
required_sir_fec_by_storage.(storageName) = nan(1,numel(block_updates));
|
||||
end
|
||||
|
||||
for block_idx = 1:numel(block_updates)
|
||||
block_update = block_updates(block_idx);
|
||||
|
||||
figure(3000 + block_idx); clf; hold on
|
||||
for storage_idx = 1:numel(storageNames)
|
||||
storageName = storageNames{storage_idx};
|
||||
curve_color = curve_colors(storage_idx,:);
|
||||
result = wh.sto.(storageName);
|
||||
|
||||
if ~isempty(block_axis) && size(result,block_axis) == numel(block_updates)
|
||||
result_subs = repmat({':'},1,ndims(result));
|
||||
result_subs{block_axis} = block_idx;
|
||||
result_for_block = result(result_subs{:});
|
||||
result_for_block = result_for_block(:).';
|
||||
else
|
||||
result_for_block = result(:).';
|
||||
end
|
||||
|
||||
% Each stored entry is a package cell containing one or more occurrences.
|
||||
ber_all = cell(1,numel(result_for_block));
|
||||
for result_idx = 1:numel(result_for_block)
|
||||
packageCell = result_for_block{result_idx};
|
||||
if isempty(packageCell)
|
||||
ber_all{result_idx} = NaN;
|
||||
continue
|
||||
end
|
||||
|
||||
ber_values = nan(1,numel(packageCell));
|
||||
for package_idx = 1:numel(packageCell)
|
||||
if isstruct(packageCell{package_idx}) && isfield(packageCell{package_idx},"metrics")
|
||||
ber_values(package_idx) = packageCell{package_idx}.metrics.BER;
|
||||
end
|
||||
end
|
||||
ber_all{result_idx} = ber_values;
|
||||
end
|
||||
|
||||
maxPackages = max(cellfun(@numel,ber_all));
|
||||
ber_mat = nan(maxPackages,numel(ber_all));
|
||||
for k = 1:numel(ber_all)
|
||||
ber_mat(1:numel(ber_all{k}),k) = ber_all{k};
|
||||
end
|
||||
|
||||
% Replace outliers in ber_mat with NaN (operate column-wise).
|
||||
for col = 1:size(ber_mat,2)
|
||||
colData = ber_mat(:,col);
|
||||
if all(isnan(colData)); continue; end
|
||||
mask = isoutlier(colData);
|
||||
ber_mat(mask,col) = NaN;
|
||||
end
|
||||
|
||||
ber = mean(ber_mat,1,"omitnan");
|
||||
ber_min = nan(1,numel(ber));
|
||||
ber_max = nan(1,numel(ber));
|
||||
for k = 1:numel(ber)
|
||||
ber_here = ber_mat(:,k);
|
||||
ber_here = ber_here(isfinite(ber_here));
|
||||
if isempty(ber_here)
|
||||
continue
|
||||
end
|
||||
ber_min(k) = min(ber_here);
|
||||
ber_max(k) = max(ber_here);
|
||||
end
|
||||
|
||||
x = sir_values;
|
||||
if numel(x) ~= numel(ber)
|
||||
x = 1:numel(ber);
|
||||
end
|
||||
|
||||
ber_by_storage.(storageName){block_idx} = ber;
|
||||
ber_mat_by_storage.(storageName){block_idx} = ber_mat;
|
||||
|
||||
y_lower = max(ber - ber_min,0);
|
||||
y_upper = max(ber_max - ber,0);
|
||||
y_bounds = [y_lower(:), y_upper(:)];
|
||||
|
||||
[hl, hp] = boundedline(x(:),ber(:),y_bounds, ...
|
||||
'alpha', 'transparency', 0.1, ...
|
||||
'cmap', curve_color, ...
|
||||
'nan', 'fill', ...
|
||||
'orientation', 'vert');
|
||||
set(hl, ...
|
||||
'LineWidth',1.4, ...
|
||||
'LineStyle','none', ...
|
||||
'Marker','o', ...
|
||||
'Markersize',5,...
|
||||
'Color',curve_color, ...
|
||||
'DisplayName',storageName);
|
||||
set(hp, ...
|
||||
'HandleVisibility','off', ...
|
||||
'LineStyle','none');
|
||||
|
||||
for k = 1:numel(x)
|
||||
scatter(repmat(x(k),maxPackages,1),ber_mat(:,k), ...
|
||||
16, ...
|
||||
'Marker','.', ...
|
||||
'MarkerEdgeColor',curve_color, ...
|
||||
'MarkerFaceColor',curve_color, ...
|
||||
'HandleVisibility','off');
|
||||
end
|
||||
|
||||
fit_mask = isfinite(x) & isfinite(ber) & ber > 0;
|
||||
if nnz(fit_mask) >= 2
|
||||
fit_order = min(3,nnz(fit_mask)-1);
|
||||
fit_coeff = polyfit(x(fit_mask),log10(ber(fit_mask)),fit_order);
|
||||
x_fit = linspace(min(x(fit_mask)),max(x(fit_mask)),300);
|
||||
y_fit = 10.^polyval(fit_coeff,x_fit);
|
||||
fit_coeff_by_storage.(storageName){block_idx} = fit_coeff;
|
||||
|
||||
sir_req = NaN;
|
||||
threshold_mask = isfinite(y_fit) & y_fit <= fec_ber_threshold;
|
||||
if any(threshold_mask)
|
||||
first_threshold_idx = find(threshold_mask,1,"first");
|
||||
if first_threshold_idx == 1
|
||||
sir_req = x_fit(first_threshold_idx);
|
||||
else
|
||||
x_pair = x_fit(first_threshold_idx-1:first_threshold_idx);
|
||||
y_pair = log10(y_fit(first_threshold_idx-1:first_threshold_idx));
|
||||
if all(isfinite(y_pair)) && diff(y_pair) ~= 0
|
||||
sir_req = interp1(y_pair,x_pair,log10(fec_ber_threshold), ...
|
||||
"linear","extrap");
|
||||
else
|
||||
sir_req = x_fit(first_threshold_idx);
|
||||
end
|
||||
end
|
||||
end
|
||||
required_sir_fec_by_storage.(storageName)(block_idx) = sir_req;
|
||||
|
||||
plot(x_fit,y_fit, ...
|
||||
'LineWidth',1.2, ...
|
||||
'LineStyle','--', ...
|
||||
'Marker','none', ...
|
||||
'Color',curve_color, ...
|
||||
'HandleVisibility','off');
|
||||
end
|
||||
end
|
||||
|
||||
title(sprintf("BER over SIR, block update = %g",block_update));
|
||||
xlabel("SIR (dB)");
|
||||
ylabel("BER");
|
||||
xlim([15 35]);
|
||||
|
||||
beautifyBERplot("logscale",true,"setcolors",false,"setmarkers",false);
|
||||
legend("Location","best","Interpreter","none");
|
||||
end
|
||||
|
||||
%% Required SIR to reach FEC over parallelization
|
||||
|
||||
figure(5000); clf; hold on
|
||||
for storage_idx = 1:numel(storageNames)
|
||||
storageName = storageNames{storage_idx};
|
||||
curve_color = curve_colors(storage_idx,:);
|
||||
required_sir = required_sir_fec_by_storage.(storageName);
|
||||
valid = isfinite(block_updates) & isfinite(required_sir);
|
||||
|
||||
if ~any(valid)
|
||||
continue
|
||||
end
|
||||
|
||||
plot(block_updates(valid),required_sir(valid), ...
|
||||
'LineWidth',1.4, ...
|
||||
'LineStyle','-', ...
|
||||
'Marker','o', ...
|
||||
'MarkerSize',5, ...
|
||||
'Color',curve_color, ...
|
||||
'DisplayName',storageName);
|
||||
end
|
||||
|
||||
set(gca,'XScale','log');
|
||||
xticks(block_updates);
|
||||
xticklabels(string(block_updates));
|
||||
grid on
|
||||
box on
|
||||
xlabel("Parallelization / block update");
|
||||
ylabel(sprintf("Required SIR at BER = %.1e (dB)",fec_ber_threshold));
|
||||
title("Required SIR to reach FEC over parallelization");
|
||||
legend("Location","best","Interpreter","none");
|
||||
|
||||
|
||||
%%
|
||||
|
||||
|
||||
function whMerged = mergeDataStorages(varargin)
|
||||
% mergeDataStorages merges compatible DataStorage objects by physical indices.
|
||||
% Existing duplicate entries are concatenated when both values are package cells.
|
||||
|
||||
whList = varargin;
|
||||
templateWh = whList{1};
|
||||
paramNames = cellstr(templateWh.fn);
|
||||
|
||||
mergedParams = struct();
|
||||
for param_idx = 1:numel(paramNames)
|
||||
paramName = paramNames{param_idx};
|
||||
values = templateWh.parameter.(paramName).values(:).';
|
||||
|
||||
for wh_idx = 2:numel(whList)
|
||||
curWh = whList{wh_idx};
|
||||
if ~isequal(sort(cellstr(curWh.fn)),sort(paramNames))
|
||||
error("mergeDataStorages:ParameterMismatch", ...
|
||||
"All warehouses must use the same parameter names.");
|
||||
end
|
||||
if ~isfield(curWh.parameter,paramName)
|
||||
error("mergeDataStorages:ParameterMismatch", ...
|
||||
"Warehouse %d does not contain parameter '%s'.",wh_idx,paramName);
|
||||
end
|
||||
|
||||
newValues = curWh.parameter.(paramName).values(:).';
|
||||
for value_idx = 1:numel(newValues)
|
||||
if ~ismember(newValues(value_idx),values)
|
||||
values(end+1) = newValues(value_idx); %#ok<AGROW>
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if strcmp(paramName,"block_update") && isnumeric(values)
|
||||
values = sort(values);
|
||||
end
|
||||
|
||||
mergedParams.(paramName) = values;
|
||||
end
|
||||
|
||||
whMerged = DataStorage(mergedParams);
|
||||
|
||||
for wh_idx = 1:numel(whList)
|
||||
curWh = whList{wh_idx};
|
||||
curStorageNames = fieldnames(curWh.sto);
|
||||
|
||||
for storage_idx = 1:numel(curStorageNames)
|
||||
storageName = curStorageNames{storage_idx};
|
||||
if ~isfield(whMerged.sto,storageName)
|
||||
whMerged.addStorage(storageName);
|
||||
end
|
||||
|
||||
for lin_idx = 1:numel(curWh.sto.(storageName))
|
||||
storedValue = curWh.sto.(storageName){lin_idx};
|
||||
if isempty(storedValue)
|
||||
continue
|
||||
end
|
||||
|
||||
[physValues,physNames] = curWh.getPhysIndicesByLinIndex(lin_idx);
|
||||
physNames = cellfun(@char,physNames,'UniformOutput',false);
|
||||
targetSubscripts = cell(1,numel(whMerged.fn));
|
||||
|
||||
for param_idx = 1:numel(whMerged.fn)
|
||||
paramName = char(whMerged.fn(param_idx));
|
||||
sourceParamIdx = find(strcmp(physNames,paramName),1);
|
||||
if isempty(sourceParamIdx)
|
||||
error("mergeDataStorages:ParameterMismatch", ...
|
||||
"Storage entry is missing parameter '%s'.",paramName);
|
||||
end
|
||||
targetSubscripts{param_idx} = whMerged.getIndexByPhys(paramName,physValues{sourceParamIdx});
|
||||
end
|
||||
|
||||
if isscalar(targetSubscripts)
|
||||
targetLinIdx = targetSubscripts{1};
|
||||
else
|
||||
targetLinIdx = sub2ind(whMerged.getStorageSize(),targetSubscripts{:});
|
||||
end
|
||||
|
||||
existingValue = whMerged.sto.(storageName){targetLinIdx};
|
||||
whMerged.sto.(storageName){targetLinIdx} = mergeStoredValue(existingValue,storedValue);
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function out = mergeStoredValue(existingValue,newValue)
|
||||
if isempty(existingValue)
|
||||
out = newValue;
|
||||
elseif iscell(existingValue) && iscell(newValue)
|
||||
out = [existingValue(:); newValue(:)].';
|
||||
else
|
||||
out = newValue;
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
Reference in New Issue
Block a user