Commit 6865f9ae authored by Brad Kennedy's avatar Brad Kennedy
Browse files

Updated license preamble, removed qsub support for now, abstracted scheduler dependent functions

parent 100465d6
......@@ -26,28 +26,25 @@
%
% See also: pop_runhtb(), pop_batch_edit()
%Copyright (C) 2013 BUCANL
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
%Code originally written by James A. Desjardins with contributions from
%Allan Campopiano and Andrew Lofts, supported by NSERC funding to
%Sidney J. Segalowitz at the Jack and Nora Walker Canadian Centre for
%Lifespan Development Research (Brock University), and a Dedicated Programming
%award from SHARCNet, Compute Ontario.
% Code written by James A. Desjardins, Allan Campopiano, Andrew Lofts,
% Michael Cichonski, and Brad Kennedy
%
%This program is free software; you can redistribute it and/or modify
%it under the terms of the GNU General Public License as published by
%the Free Software Foundation; either version 2 of the License, or
%(at your option) any later version.
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
%This program is distributed in the hope that it will be useful,
%but WITHOUT ANY WARRANTY; without even the implied warranty of
%MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
%GNU General Public License for more details.
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
%You should have received a copy of the GNU General Public License
%along with this program (LICENSE.txt file in the root directory); if not,
%write to the Free Software Foundation, Inc., 59 Temple Place,
%Suite 330, Boston, MA 02111-1307 USA
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function properties = batchconfig2propgrid(batchconfig)
num_properties = 16;
......
% config_parse() - parses the input text by line based on the number of
% indentantion specifies if it is on the root level
% or the second level into a tree structure
%
% Usage:
% >> out = config_parse(in)
%
% Inputs:
% in - text, delimited by newlines with indentation specifying the
% depth it resides in the tree
%
% Outputs:
% out - A tree created by tree_new that has a root a index 1 with
% top-level children (or settings) below that and
% their values below them
%
% See:
% tree_new
% config_parse_multilevel
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function out = config_parse(in)
% in is a string or maybe a file descriptor
scan = regexp(in, '[\n]+', 'split');
......
% config_parse_multilevel() -
% parses the input text by line based on the number of
% indentantion specifies if it is on the root level
% or any level below that
%
% Usage:
% >> out = config_parse(in)
%
% Inputs:
% in - text, delimited by newlines with indentation specifying the
% depth it resides in the tree
%
% Outputs:
% out - A tree created by tree_new that has a root a index 1 with
% top-level children and all levels below that
%
% See:
% tree_new
% config_parse
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function out = config_parse_multilevel(in)
% in is a string or maybe a file descriptor
scan = regexp(in, '[\n]+', 'split');
......
......@@ -9,28 +9,25 @@
%
% See also: pop_runhtb(), pop_batch_edit(), propgrid2batchconfig()
%Copyright (C) 2013 BUCANL
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
%Code originally written by James A. Desjardins with contributions from
%Allan Campopiano and Andrew Lofts, supported by NSERC funding to
%Sidney J. Segalowitz at the Jack and Nora Walker Canadian Centre for
%Lifespan Development Research (Brock University), and a Dedicated Programming
%award from SHARCNet, Compute Ontario.
% Code written by James A. Desjardins, Allan Campopiano, Andrew Lofts,
% and Brad Kennedy
%
%This program is free software; you can redistribute it and/or modify
%it under the terms of the GNU General Public License as published by
%the Free Software Foundation; either version 2 of the License, or
%(at your option) any later version.
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
%This program is distributed in the hope that it will be useful,
%but WITHOUT ANY WARRANTY; without even the implied warranty of
%MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
%GNU General Public License for more details.
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
%You should have received a copy of the GNU General Public License
%along with this program (LICENSE.txt file in the root directory); if not,
%write to the Free Software Foundation, Inc., 59 Temple Place,
%Suite 330, Boston, MA 02111-1307 USA
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function batch_config=init_batch_config()
......
......@@ -27,28 +27,24 @@
%
% See also: pop_runhtb(), pop_batch_edit(), init_batch_config()
%Copyright (C) 2013 BUCANL
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
%Code originally written by James A. Desjardins with contributions from
%Allan Campopiano and Andrew Lofts, supported by NSERC funding to
%Sidney J. Segalowitz at the Jack and Nora Walker Canadian Centre for
%Lifespan Development Research (Brock University), and a Dedicated Programming
%award from SHARCNet, Compute Ontario.
% Code written by James A. Desjardins, Allan Campopiano, and Andrew Lofts
%
%This program is free software; you can redistribute it and/or modify
%it under the terms of the GNU General Public License as published by
%the Free Software Foundation; either version 2 of the License, or
%(at your option) any later version.
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
%This program is distributed in the hope that it will be useful,
%but WITHOUT ANY WARRANTY; without even the implied warranty of
%MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
%GNU General Public License for more details.
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
%You should have received a copy of the GNU General Public License
%along with this program (LICENSE.txt file in the root directory); if not,
%write to the Free Software Foundation, Inc., 59 Temple Place,
%Suite 330, Boston, MA 02111-1307 USA
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function batchconfig = propgrid2batchconfig(propgrid,batchconfig)
......
......@@ -18,28 +18,24 @@
%
% See also: pop_loadbatchconfig, pop_batch_edit(), pop_runhtb()
%Copyright (C) 2013 BUCANL
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
%Code originally written by Allan Campopiano with contributions from
%James Desjardins and Andrew Lofts, supported by NSERC funding to
%Sidney J. Segalowitz at the Jack and Nora Walker Canadian Centre for
%Lifespan Development Research (Brock University), and a Dedicated Programming
%award from SHARCNet, Compute Ontario.
% Code written by Brad Kennedy
%
%This program is free software; you can redistribute it and/or modify
%it under the terms of the GNU General Public License as published by
%the Free Software Foundation; either version 2 of the License, or
%(at your option) any later version.
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
%This program is distributed in the hope that it will be useful,
%but WITHOUT ANY WARRANTY; without even the implied warranty of
%MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
%GNU General Public License for more details.
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
%You should have received a copy of the GNU General Public License
%along with this program (LICENSE.txt file in the root directory); if not,
%write to the Free Software Foundation, Inc., 59 Temple Place,
%Suite 330, Boston, MA 02111-1307 USA
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function batch_config=text2struct_bc(fname)
......
% ef_base_driver() - base class for drivers for specific schedulers
% this function lets us abstract the differences between
% schedulers to smaller files with less repetition
%
% Usage:
% >> driver = ef_base_driver()
%
% Outputs:
% driver - for use with a scheduler
%
% See also:
% @ef_base_driver/format_scheduler
% @ef_base_driver/make_job_dir
% @ef_base_driver/read_jobs
% @ef_base_driver/submit_line
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function obj = ef_base_driver()
obj = class(struct(), 'ef_base_driver');
% format_scheduler() - returns a string that represents a line for a
% single input data file on the given scheduler
%
% Usage:
% >> out = format_scheduler(driver, job_spec)
%
% Inputs:
% driver - the driver we are using, should be subtype of ef_base_driver
% job_spec - scheduler independent format specified in
% @ef_base_driver/submit_line
%
% Outputs:
% out - scheduler dependent format for the concrete scheduler
% implementation
%
% See also:
% @ef_base_driver/submit_line
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function out = format_scheduler(~, ~)
error('The format_scheduler function MUST be overridden by drivers');
end
function job = make_job_dir(drive, job)
if nargin < 2
help @ef_base_driver/make_job_dir
return;
end
% We use this a lot so lets make a shorter name
% this job
% Note: We have to restore this at the end due to copy semantics
tjob = job(end);
%% Generate m file for Octave/Matlab or exec function for programs
switch tjob.batch_config.software
case 'octave'
tjob = ef_gen_m(tjob);
case 'matlab'
tjob = ef_gen_m(tjob);
otherwise
tjob = ef_exec_str(tjob);
end
%% Collect the remote project work
i = strfind(tjob.context_config.remote_project_work, ':');
job(end).remote_work = tjob.context_config.remote_project_work(i(1) + 1:end);
tjob.remote_work = job(end).remote_work;
cmdstr = sprintf('#!/bin/bash\nset -e\n');
%% Read the sesinit
fid_sesinit = fopen(tjob.batch_config.session_init, 'r');
if fid_sesinit
tmpstr = fread(fid_sesinit,'*char');
fclose(fid_sesinit);
else
tmpstr = tjob.batch_config.session_init;
end
cmdstr = sprintf('%s%s', cmdstr, tmpstr);
%% Loop through participants
for bfni = 1:length(tjob.batch_dfn)
% Cycle through all the subject files
waitlist = {};
for l = 1:length(job)
if any(job(l).ordernum == tjob.waitnum)
% Add elements to the wait list cell array
waitlist{end+1} = job(l).jobids{bfni};
end
end
% Append it to the file string
tmpstr = submit_line(drive, tjob, bfni, waitlist);
cmdstr = sprintf('%s\n%s', cmdstr, tmpstr);
end
%% WRITE [SUBSTRINIT,QSUBSTR] TO A *.SUB TEXT FILE IN THE LOG PATH...
fid = fopen(fullfile(tjob.context_config.log, tjob.m_path, 'submit.sh'), 'w');
fwrite(fid, cmdstr);
fclose(fid);
%% Write it back
job(end) = tjob;
% read_jobs() - returns a cellstr of jobids from the given htb file
%
% Usage:
% >> out = read_jobs(driver, result_str)
%
% Inputs:
% driver - the driver we are using, should be subtype of ef_base_driver
% job_spec - results of the output of running the submission script
%
% Outputs:
% out - scheduler independent format for the concrete scheduler
% implementation
%
% See also:
% pop_runhtb
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function out = read_jobs(driver, str)
error('The read_jobs function MUST be overridden by drivers');
end
% submit_line() - produces one line of scheduler dependent submit.sh output
% this part abstracts the common parts of each scheduler
% and passes it to the specific drivers to get the
% scheduler dependent line
%
% Usage:
% >> outstr = submit_line(drive, tjob, data_filename_index, waitlist)
%
% Inputs:
% driver - the driver we are using, should be subtype of
% ef_base_driver
% tjob - the current job, corresponds to a htb file
% data_filename_index - the current job index, this corresponds to
% participants files
% waitlist
%
% Outputs:
% out - scheduler independent format for the concrete scheduler
% implementation
%
% See also:
% pop_runhtb
% Copyright (C) 2017 Brock University Cognitive and Affective Neuroscience Lab
%
% Code written by Brad Kennedy
%
% This program is free software; you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation; either version 2 of the License, or
% (at your option) any later version.
%
% This program is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with this program (LICENSE.txt file in the root directory); if not,
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function outstr = submit_line(drive, tjob, data_filename_index, waitlist)
histfname = tjob.batch_hfn;
datafname = tjob.batch_dfn{data_filename_index};
batch_dfp = tjob.batch_dfp;
batch_config = tjob.batch_config;
%% Generate job_name
job_name = batch_config.job_name;
% batch_dfn
job_name = key_strswap(job_name, 'batch_dfn', datafname);
% batch_hfn
job_name = key_strswap(job_name, 'batch_hfn', histfname);
%% mfile_name and backup as job_name for compat
mfile_name = batch_config.mfile_name;
if isempty(mfile_name)
mfile_name = job_name;
end
exec_path = strrep(fullfile(tjob.context_config.log, tjob.m_path),'\','/');
%% Read jobinit
fid_jobinit = fopen(batch_config.job_init,'r');
if fid_jobinit ~= -1
job_init = fread(fid_jobinit,'*char')';
fclose(fid_jobinit);
else
job_init = batch_config.job_init;
end
%% Verify we are dealing with .set files otherwise skip this section
[~, ~, ext] = fileparts(datafname);
if strcmp(ext, '.set')
dimensions = ef_get_eegdims(batch_dfp, datafname);
% These variables are eval'd so they will show up as a warning
c = dimensions.channels; %#ok<NASGU> % channels
s = dimensions.samples; %#ok<NASGU> % samples
end
%% Memory calculations
memory_type = [];
memory_alloc = [];
if ~isempty(batch_config.memory)
memory_type = lower(batch_config.memory(end));
memory_alloc = eval(batch_config.memory(1:end-1));
end
%% Time limit calculations
time_span = [];
if ~isempty(batch_config.time_limit)
time_var = lower(batch_config.time_limit(end));
time_span = eval(batch_config.time_limit(1:end-1));
% Make everything into seconds
switch time_var
case 's'
case 'm'
time_span = time_span * 60;
case 'h'
time_span = time_span * 360;
otherwise
error('End of time_limit field needs to be one of {s, m, h}');
end
end
%% Other options
mpi = false;
if ~isempty(batch_config.mpi)
mpi = strcmp(batch_config.mpi, 'true');
end
num_proc = 1;
if ~isempty(batch_config.num_processors)
num_proc = str2double(batch_config.num_processors);
end
num_thread_per_proc = 1;
if isfield(batch_config, 'num_thread_per_proc') ...
&& ~isempty(batch_config.num_thread_per_proc)
num_thread_per_proc = str2double(batch_config.num_thread_per_proc);
end
program = batch_config.software;
program_options = batch_config.program_options;
exec_str = tjob.exec_str{data_filename_index};
generic_opts = batch_config.qsub_options;
is_octave = strcmp(program, 'octave') || strcmp(program, 'matlab');
%% Wrapper file for octave
% This gets the rel dir to the octave_exit_wrapper
[dir, ~, ~] = fileparts(which(mfilename()));
dir = cd(cd(fullfile(dir, '../')));
reldir = dir(length(cd)+2:end);
% Relative octave wrapper dir
wrappername = [reldir filesep 'octave_exit_wrapper.m'];
%%
% char
job_spec.name = job_name;
job_spec.output = [exec_path, '/', mfile_name , '.log'];
% number
job_spec.timeseconds = time_span;
job_spec.memory_alloc = memory_alloc;
% char
job_spec.memory_type = memory_type;
job_spec.job_init = job_init;
% bool
job_spec.mpi = mpi;
% number
job_spec.num_proc = num_proc;
job_spec.num_thread_per_proc = num_thread_per_proc;
% char