silas/imdd_silas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

restructure and organize

Browse Source
This commit is contained in:
Silas Oettinghaus
2026-06-22 22:58:58 +02:00
parent c4f75b7ec4
commit 33ec5b3116
36 changed files with 1914 additions and 674 deletions
Download Patch File Download Diff File Expand all files Collapse all files

80
Functions/Job_Processing/runBatch.m
View File

@@ -1,6 +1,6 @@
function results = runBatch(workerFcn, jobs, options)
function batchResults = runBatch(workerFcn, jobs, options)
%RUNBATCH Execute a list of jobs in serial or parallel.
% results = runBatch(workerFcn, jobs) executes each jobs(i).args cell
% batchResults = runBatch(workerFcn, jobs) executes each jobs(i).args cell
% with workerFcn and returns a cell array aligned with the input jobs.
%
% Supported job fields:
@@ -24,7 +24,7 @@ function results = runBatch(workerFcn, jobs, options)
mode = normalizeProcessingMode(options.mode);
jobs = normalizeJobs(jobs);
nJobs = numel(jobs);
results = cell(1, nJobs);
batchResults = cell(1, nJobs);
h = [];
if nJobs == 0
@@ -33,7 +33,7 @@ function results = runBatch(workerFcn, jobs, options)
if options.waitbar
h = waitbar(0, char(options.waitbarMessage));
cleanupWaitbar = onCleanup(@() closeWaitbar(h)); %#ok<NASGU>
cleanupWaitbar = onCleanup(@() closeWaitbar(h));
end
switch mode
@@ -41,36 +41,36 @@ function results = runBatch(workerFcn, jobs, options)
pool = setupParallelPool(options.numWorkers, options.idleTimeout, options.cancelExistingQueue);
futures = parallel.FevalFuture.empty(nJobs, 0);
for idx = 1:nJobs
fprintf('[%s] Submitted.\n', jobs(idx).label);
futures(idx) = parfeval(pool, workerFcn, 1, jobs(idx).args{:});
for jobIdx = 1:nJobs
fprintf('[%s] Submitted.\n', jobs(jobIdx).label);
futures(jobIdx) = parfeval(pool, workerFcn, 1, jobs(jobIdx).args{:});
end
consumedIdx = false(nJobs, 1);
for completedCount = 1:nJobs
try
[idx, value] = fetchNext(futures);
consumedIdx(idx) = true;
results{idx} = value;
fprintf('[%s] Completed (%d/%d).\n', jobs(idx).label, completedCount, nJobs);
[jobIdx, jobResult] = fetchNext(futures);
consumedIdx(jobIdx) = true;
batchResults{jobIdx} = jobResult;
fprintf('[%s] Completed (%d/%d).\n', jobs(jobIdx).label, completedCount, nJobs);
if ~isempty(options.resultHandler)
options.resultHandler(value, jobs(idx), idx);
options.resultHandler(jobResult, jobs(jobIdx), jobIdx);
end
catch fetchErr
idxErr = findErroredFuture(futures, consumedIdx);
if isempty(idxErr)
errorJobIdx = findErroredFuture(futures, consumedIdx);
if isempty(errorJobIdx)
rethrow(fetchErr);
end
consumedIdx(idxErr) = true;
errInfo = extractFutureError(futures(idxErr));
results{idxErr} = errInfo;
consumedIdx(errorJobIdx) = true;
errInfo = extractFutureError(futures(errorJobIdx));
batchResults{errorJobIdx} = errInfo;
if ~isempty(options.errorHandler)
options.errorHandler(errInfo, jobs(idxErr), idxErr);
options.errorHandler(errInfo, jobs(errorJobIdx), errorJobIdx);
else
defaultErrorHandler(errInfo, jobs(idxErr), idxErr);
defaultErrorHandler(errInfo, jobs(errorJobIdx), errorJobIdx);
end
end
@@ -78,27 +78,27 @@ function results = runBatch(workerFcn, jobs, options)
end
case processingMode.serial
for idx = 1:nJobs
for jobIdx = 1:nJobs
try
fprintf('[%s] Running.\n', jobs(idx).label);
value = feval(workerFcn, jobs(idx).args{:});
results{idx} = value;
fprintf('[%s] Completed (%d/%d).\n', jobs(idx).label, idx, nJobs);
fprintf('[%s] Running.\n', jobs(jobIdx).label);
jobResult = workerFcn(jobs(jobIdx).args{:});
batchResults{jobIdx} = jobResult;
fprintf('[%s] Completed (%d/%d).\n', jobs(jobIdx).label, jobIdx, nJobs);
if ~isempty(options.resultHandler)
options.resultHandler(value, jobs(idx), idx);
options.resultHandler(jobResult, jobs(jobIdx), jobIdx);
end
catch ME
results{idx} = ME;
batchResults{jobIdx} = ME;
if ~isempty(options.errorHandler)
options.errorHandler(ME, jobs(idx), idx);
options.errorHandler(ME, jobs(jobIdx), jobIdx);
else
defaultErrorHandler(ME, jobs(idx), idx);
defaultErrorHandler(ME, jobs(jobIdx), jobIdx);
end
end
updateWaitbar(options.waitbar, h, idx, nJobs);
updateWaitbar(options.waitbar, h, jobIdx, nJobs);
end
otherwise
@@ -128,23 +128,23 @@ function jobs = normalizeJobs(jobs)
return
end
for idx = 1:numel(jobs)
if ~isfield(jobs, 'args') || isempty(jobs(idx).args)
jobs(idx).args = {};
for jobIdx = 1:numel(jobs)
if ~isfield(jobs, 'args') || isempty(jobs(jobIdx).args)
jobs(jobIdx).args = {};
end
if ~iscell(jobs(idx).args)
error('runBatch:InvalidArgs', 'jobs(%d).args must be a cell array.', idx);
if ~iscell(jobs(jobIdx).args)
error('runBatch:InvalidArgs', 'jobs(%d).args must be a cell array.', jobIdx);
end
if ~isfield(jobs, 'label') || isempty(jobs(idx).label)
jobs(idx).label = sprintf('Job %d', idx);
if ~isfield(jobs, 'label') || isempty(jobs(jobIdx).label)
jobs(jobIdx).label = sprintf('Job %d', jobIdx);
else
jobs(idx).label = string(jobs(idx).label);
jobs(jobIdx).label = string(jobs(jobIdx).label);
end
if ~isfield(jobs, 'meta')
jobs(idx).meta = struct();
jobs(jobIdx).meta = struct();
end
end
end
@@ -189,9 +189,9 @@ function pool = setupParallelPool(numWorkers, idleTimeout, cancelExistingQueue)
end
end
function idxErr = findErroredFuture(futures, consumedIdx)
function errorJobIdx = findErroredFuture(futures, consumedIdx)
readMask = arrayfun(@(future) future.Read, futures).';
idxErr = find(readMask & ~consumedIdx, 1);
errorJobIdx = find(readMask & ~consumedIdx, 1);
end
function errInfo = extractFutureError(future)