Skip to content

GitLab

Commit cf05aa01 authored by Brad Kennedy's avatar Brad Kennedy
Browse files

Added ef_qsub_grid driver and added more modular driver setup ability. Drag... 

Added ef_qsub_grid driver and added more modular driver setup ability. Drag and drop drivers are now available.
parent 77fa3e45
Hide whitespace changes
Inline Side-by-side
batch/config_func/batchconfig2propgrid.m
View file @ cf05aa01
......@@ -52,7 +52,7 @@ for i=1:length(batchconfig) % 1 based indexing is awful
properties(((i-1)*num_properties+1):(i*num_properties)) = [ ...
... % basic properties...
PropertyGridField(['exec_func[',num2str(i),']'], batchconfig(i).exec_func, ...
'Type', PropertyType('char', 'row', {'ef_current_base', 'ef_sbatch', 'ef_sqsub'}), ...
'Type', PropertyType('char', 'row', get_valid_exec_func()), ...
'Category', [' ', batchconfig(i).file_name], ...
'DisplayName', ['exec_func[',num2str(i),']'], ...
'Description', 'Name of the function that will execute the htb files.') ...
......
batch/exec_func/@ef_qsub_grid_driver/ef_qsub_grid_driver.m 0 → 100644
View file @ cf05aa01
% ef_qsub_grid_driver() - subclass of ef_base_driver that implements a driver
% sharcnet scheduler based on qsub
%
% Usage:
% >> driver = ef_qsub_grid_driver()
%
% Outputs:
% driver - for use with a scheduler
%
% See also:
% @ef_qsub_driver/format_scheduler
% @ef_qsub_driver/read_jobs
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02
function obj = ef_qsub_grid_driver()
obj = class(struct(), 'ef_qsub_grid_driver', ef_base_driver());
end
batch/exec_func/@ef_qsub_grid_driver/format_scheduler.m 0 → 100644
View file @ cf05aa01
% format_scheduler() - returns a string that represents a line for a
% single input data file on the given scheduler
%
% Usage:
% >> out = format_scheduler(driver, job_spec)
%
% Inputs:
% driver - the driver we are using, should be subtype of ef_base_driver
% job_spec - scheduler independent format specified in
% @ef_base_driver/submit_line
%
% Outputs:
% out - scheduler dependent format for the concrete scheduler
% implementation
%
% See also:
% @ef_base_driver/submit_line
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function out = format_scheduler(~, job_spec)
out = '';
if ~isempty(job_spec.job_init)
out = [out job_spec.job_init];
end
out = sprintf('%sqsub -cwd', out);
if ~isempty(job_spec.name)
out = sprintf('%s -N "%s"', out, job_spec.name);
end
if ~isempty(job_spec.output)
out = sprintf('%s -o "%s"', out, job_spec.output);
out = sprintf('%s -j Y', out);
end
if ~isempty(job_spec.waitlist)
waitstring = cellfun(@(x) [x ','], job_spec.waitlist, ...
'UniformOutput', false);
waitstring = [waitstring{:}];
waitstring = waitstring(1:end-1);
out = sprintf('%s -hold_jid %s', out, waitstring);
end
if ~isempty(job_spec.memory_alloc) && ~isempty(job_spec.memory_type)
switch job_spec.memory_type
case 'g'
job_spec.memory_alloc = job_spec.memory_alloc * 1000;
job_spec.memory_type = 'm';
case 'm'
otherwise
error(['The ef_qsub_grid_driver only supports memory in ' ...
'''m'' or ''g''']);
end
out = sprintf('%s -l h_vmem=%d%s', out, ...
fix(job_spec.memory_alloc), job_spec.memory_type);
end
if ~isempty(job_spec.timeseconds)
time_span = fix(job_spec.timeseconds);
hours = fix(time_span / 3600);
time_span = mod(time_span, 3600);
minutes = fix(time_span / 60);
time_span = mod(time_span, 60);
seconds = fix(time_span);
time_str = sprintf('%02d:%02d:%02d', hours, minutes, seconds);
out = sprintf('%s -l h_rt=%s', out, time_str);
end
if job_spec.mpi
error(['qsub driver may not use the mpi flag, please specify ' ...
'mpi using submit_options field']);
end
if ~isempty(job_spec.num_proc) && job_spec.num_proc ~= 1
error(['qsub driver may not use the num_task flag, please specify ' ...
'num_task using submit_options field']);
end
if ~isempty(job_spec.num_thread_per_proc) ...
&& job_spec.num_thread_per_proc ~= 1
error(['qsub driver may not use the threads_per_task flag, please specify ' ...
'threads_per_task using submit_options field']);
end
% Add the generic options here
generic_opts_str = cellfun(@(x) [x ' '], job_spec.generic_opts, ...
'UniformOutput', false);
generic_opts_str = [generic_opts_str{:}];
generic_opts_str = generic_opts_str(1:end-1);
if ~isempty(generic_opts_str)
out = sprintf('%s %s', out, generic_opts_str);
end
wrapper = '';
if job_spec.is_octave && ~isempty(job_spec.wrappers)
wrapper = [' ' job_spec.wrappers];
end
poptions_string = cellfun(@(x) [x ' '], job_spec.program_options, ...
'UniformOutput', false);
poptions_string = [poptions_string{:}];
if job_spec.is_octave
out = sprintf('%s %s %s%s %s', ...
out, job_spec.program, poptions_string, wrapper, job_spec.mfile_name);
else
out = sprintf('%s %s%s', ...
out, job_spec.exec_str);
end
end
batch/exec_func/@ef_qsub_grid_driver/read_jobs.m 0 → 100644
View file @ cf05aa01
% read_jobs() - returns a cellstr of jobids from the given htb file
%
% Usage:
% >> out = read_jobs(driver, result_str)
%
% Inputs:
% driver - the driver we are using, should be subtype of ef_base_driver
% result_str - results of the output of running the submission script,
% must be of type char
%
% Outputs:
% out - scheduler independent format for the concrete scheduler
% implementation
%
% See also:
% pop_runhtb
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function out = read_jobs(~, result_str)
expression = 'jobid (?<id>\d+)';
id = regexp(result_str, expression, 'names');
out = {id.id};
end
\ No newline at end of file
batch/exec_func/get_valid_exec_func.m 0 → 100644
View file @ cf05aa01
function cellstrexec = get_valid_exec_func()
cellstrexec = {'ef_current_base'};
[thisdir, ~, ~] = fileparts(which(mfilename()));
dirlist = dir(thisdir);
listf = regexp({dirlist.name}, '@(?<names>.*)_driver', 'names');
for i = 1:numel(listf)
% Don't include ef_base_driver
if ~isempty(listf{i}) && ~strcmp(listf{i}.names, 'ef_base')
cellstrexec = [cellstrexec ; listf{i}.names];
end
end
cellstrexec = cellstrexec';
end
batch/pop_func/pop_runhtb.m
View file @ cf05aa01
......@@ -264,10 +264,14 @@ for hi=1:length(htb_fname)
%the first index is the order the rest are waits
%current base does not care about order
ordercode = batch_config(hi).order;
% Get all valid exec functions
vnames = get_valid_exec_func();
% Remove current base
vnames = vnames(~strcmp('ef_current_base', vnames));
if strcmp('ef_current_base', batch_config(hi).exec_func)
disp('Executing local scripts in a linear order');
elseif strcmp('ef_sqsub', batch_config(hi).exec_func) ...
|| strcmp('ef_sbatch', batch_config(hi).exec_func)
elseif any(strcmp(vnames, batch_config(hi).exec_func))
% If no order is given then make the batches consectutive
if (isempty(ordercode))
disp(['No order was specified! - Executing ' ...
......@@ -300,17 +304,9 @@ for hi=1:length(htb_fname)
if strcmp(batch_config(hi).exec_func,'ef_current_base')
job_struct = ef_current_base(job_struct);
else
switch batch_config(hi).exec_func
case 'ef_sqsub'
driver = ef_sqsub_driver();
case 'ef_sbatch'
driver = ef_sbatch_driver();
case 'ef_qsub'
error(['qsub is no longer supported, please submit a ticket' ...
' if you still require it and support will be readded']);
otherwise
error('invalid execution function... doing nothing.');
end
% Get a handle for our exec function
handle = eval(['@', batch_config(hi).exec_func '_driver']);
driver = handle();
job_struct = make_job_dir(driver, job_struct);
%% EXECUTE/SUBMIT JOBS...
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment