restructure and organize

Functions/Job_Processing/submitJobs.m

@@ -9,9 +9,11 @@ function [results, wh] = submitJobs(run_ids, dsp_options, submit_mode, submit_op
 %   submit_options : struct with fields
 %       .waitbar (logical)
 %       .wh      (DataStorage object)
+%       .storePackages (string array, optional package whitelist)
 %
 %   results : cell(nJobsPerRunId, nRunIds)
-%   wh      : updated DataStorage
+%   wh      : updated DataStorage. For multiple run_ids, run_id is added as
+%             the first storage axis.
 
 arguments
     run_ids        int32               = 0
@@ -19,15 +21,19 @@ arguments
     submit_mode    processingMode      = processingMode.serial
     submit_options.waitbar (1,1) logical = true
     submit_options.wh = DataStorage(struct())
+    submit_options.storePackages string = string.empty()
 end
 
 % Normalize
 run_ids = run_ids(:)';
 nRunIds       = numel(run_ids);
-nJobsPerRunId = submit_options.wh.getLastLinIndice();
+sweepWh       = submit_options.wh;
+validateNoRunIdSweepParameter(sweepWh);
+
+nJobsPerRunId = sweepWh.getLastLinIndice();
 totalJobs     = nRunIds * nJobsPerRunId;
 
-wh = submit_options.wh;
+wh = buildStorageWarehouse(sweepWh, run_ids);
 results = cell(nJobsPerRunId, nRunIds);
 jobs = repmat(struct('args', {{}}, 'label', "", 'meta', struct()), 1, totalJobs);
 
@@ -35,14 +41,16 @@ jobCounter = 0;
 for r = 1:nRunIds
     for k = 1:nJobsPerRunId
         jobCounter = jobCounter + 1;
-        optionalVars = buildOptionalVars(k, submit_options.wh);
+        userParameters = buildUserParameters(k, sweepWh);
         jobOptions = dsp_options;
-        jobOptions.parameters = optionalVars;
+        jobOptions.userParameters = userParameters;
 
         jobs(jobCounter).args = [{run_ids(r)}, structToNameValue(jobOptions)];
         jobs(jobCounter).label = sprintf('RunID %d, Job %d', run_ids(r), k);
         jobs(jobCounter).meta.runIndex = r;
         jobs(jobCounter).meta.jobIndex = k;
+        jobs(jobCounter).meta.sweepJobIndex = k;
+        jobs(jobCounter).meta.storageJobIndex = buildStorageJobIndex(run_ids(r), userParameters, wh);
         jobs(jobCounter).meta.run_id = run_ids(r);
     end
 end
@@ -74,16 +82,69 @@ end
         fprintf('Full report:\n%s\n', getReport(ME,'extended'));
     end
 
-    function optionalVars = buildOptionalVars(jobIndex, wh)
-        optionalVars = struct();
+    function userParameters = buildUserParameters(jobIndex, wh)
+        userParameters = struct();
         if ~isempty(wh.getDimension())
             [vals, names] = wh.getPhysIndicesByLinIndex(jobIndex);
             for pi = 1:numel(names)
-                optionalVars.(names{pi}) = vals{pi};
+                userParameters.(names{pi}) = vals{pi};
             end
         end
     end
 
+    function validateNoRunIdSweepParameter(wh)
+        if isfield(wh.inputParams, "run_id")
+            error("submitJobs:RunIdParameterConflict", ...
+                "Do not define userParameters.run_id. submitJobs manages run_id as a storage axis.");
+        end
+    end
+
+    function storageWh = buildStorageWarehouse(sweepWh, runIds)
+        if isscalar(runIds)
+            storageWh = sweepWh;
+            return
+        end
+
+        storageParameters = struct();
+        storageParameters.run_id = runIds;
+
+        sweepParameterNames = fieldnames(sweepWh.inputParams);
+        for parameterIdx = 1:numel(sweepParameterNames)
+            parameterName = sweepParameterNames{parameterIdx};
+            storageParameters.(parameterName) = sweepWh.inputParams.(parameterName);
+        end
+
+        storageWh = DataStorage(storageParameters);
+    end
+
+    function storageJobIndex = buildStorageJobIndex(runId, userParameters, wh)
+        storageParameterNames = wh.fn;
+
+        if isempty(storageParameterNames)
+            storageJobIndex = 1;
+            return
+        end
+
+        storageSubscripts = cell(1, numel(storageParameterNames));
+        for parameterIdx = 1:numel(storageParameterNames)
+            parameterName = char(storageParameterNames(parameterIdx));
+
+            if strcmp(parameterName, "run_id")
+                parameterValue = runId;
+            else
+                parameterValue = userParameters.(parameterName);
+            end
+
+            storageSubscripts{parameterIdx} = wh.getIndexByPhys(parameterName, parameterValue);
+        end
+
+        if isscalar(storageSubscripts)
+            storageJobIndex = storageSubscripts{1};
+        else
+            storageJobIndex = sub2ind(wh.getStorageSize(), storageSubscripts{:});
+        end
+    end
+
     function nameValue = structToNameValue(options)
         names = fieldnames(options);
         nameValue = cell(1, 2*numel(names));
@@ -95,14 +156,41 @@ end
     end
 
     function storeResult(val, job, ~)
-        if ~isempty(wh)
-            jobIndex = job.meta.jobIndex;
-            wh.addValueToStorageByLinIdx(val.ffe_package,  'ffe_package',  jobIndex);
-            wh.addValueToStorageByLinIdx(val.mlse_package, 'mlse_package', jobIndex);
-            wh.addValueToStorageByLinIdx(val.vnle_package, 'vnle_package', jobIndex);
-            wh.addValueToStorageByLinIdx(val.dbtgt_package,'dbtgt_package',jobIndex);
-            wh.addValueToStorageByLinIdx(val.dbenc_package,'dbenc_package',jobIndex);
-            wh.addValueToStorageByLinIdx(val.mlmlse_package,'mlmlse_package',jobIndex);
+        if isempty(wh) || ~isstruct(val)
+            return
+        end
+
+        storageJobIndex = job.meta.storageJobIndex;
+        resultFields = fieldnames(val);
+
+        for resultIdx = 1:numel(resultFields)
+            storageName = resultFields{resultIdx};
+
+            if ~shouldStore(storageName)
+                continue
+            end
+
+            if isempty(val.(storageName))
+                continue
+            end
+
+            ensureStorage(storageName);
+            wh.addValueToStorageByLinIdx(val.(storageName), storageName, storageJobIndex);
+        end
+    end
+
+    function tf = shouldStore(storageName)
+        if isempty(submit_options.storePackages)
+            tf = true;
+            return
+        end
+
+        tf = any(string(storageName) == submit_options.storePackages);
+    end
+
+    function ensureStorage(storageName)
+        if ~isfield(wh.sto, storageName)
+            wh.addStorage(storageName);
         end
     end