Commit 6247ac86 authored by Brad Kennedy's avatar Brad Kennedy
Browse files

Cleaned up a bunch of the submission scripts and made them more robust

parent 526cf921
......@@ -51,7 +51,8 @@ d = dir(fullfile(job_struct(end).context_config.log, ...
job_struct(end).m_path,'/*.m'));
for i=1:length(d)
disp(['Evaluating... ',d(i).name,' from ',job_struct(end).m_path,' directory...']);
disp(['Evaluating... ',d(i).name,' from ', ...
job_struct(end).m_path,' directory...']);
try
[~,evalfname] = fileparts(d(i).name);
diary(fullfile(cd, job_struct(end).context_config.log, ...
......
......@@ -35,94 +35,102 @@
% write to the Free Software Foundation, Inc., 59 Temple Place,
% Suite 330, Boston, MA 02111-1307 USA
function job_struct=ef_gen_m(job_struct)
function job_struct = ef_gen_m(job_struct)
%% BUILD THE DIRECTORY FOR THE TIME M FILES IN THE
% LOG PATH OF THE CURRENT ANALYSIS_ROOT FOLDER...
%make directory named by history fname and date-time stamp.
root_hfn=job_struct.batch_hfn(1:strfind(job_struct.batch_hfn,'.')-1);
job_struct.m_path=sprintf('%s-%s', ...
root_hfn = job_struct.batch_hfn(1:strfind(job_struct.batch_hfn,'.')-1);
job_struct.m_path = sprintf('%s-%s', ...
root_hfn, ...
job_struct.starttime);
mkdir(fullfile(job_struct.context_config.log,job_struct.m_path));
mkdir(fullfile(job_struct.context_config.log, job_struct.m_path));
%% MATLAB OR OCTAVE EXECUTION (BUILD M FILE)...
if ~strcmp(job_struct.batch_config.software,'none') || ~strcmp(job_struct.batch_config.exec_func,'ef_sqsub')
if ~strcmp(job_struct.batch_config.software, 'none') ...
|| ~strcmp(job_struct.batch_config.exec_func, 'ef_sqsub')
%% INITIATE BATCHINITSTR...
batchInitStr='';
%% M_INIT
if exist(job_struct.batch_config.m_init,'file');
fid_minit=fopen(job_struct.batch_config.m_init,'r');
tmp_batchInitStr=fread(fid_minit,'char');
batchInitStr=[batchInitStr,char(tmp_batchInitStr')];
if exist(job_struct.batch_config.m_init, 'file');
fid_minit = fopen(job_struct.batch_config.m_init, 'r');
tmp_batchInitStr = fread(fid_minit, 'char');
batchInitStr=[batchInitStr, char(tmp_batchInitStr')];
else
batchInitStr=[batchInitStr,job_struct.batch_config.m_init];
batchInitStr = [batchInitStr, job_struct.batch_config.m_init];
end
%% GET THE STRING FROM THE HTB FILES AND START BUILDING BATCHHISTSTR...
%Create batchHistStr from the current HistFName file...
[cPath,root_hfn,cExt]=fileparts(job_struct.batch_hfn);
[~, root_hfn, ~] = fileparts(job_struct.batch_hfn);
eval(['fidRHT=fopen(''' job_struct.batch_hfp job_struct.batch_hfn ''',''r'');']);
batchHistStr=char(fread(fidRHT,'char')');
batchHistStr = char(fread(fidRHT,'char')');
qsubstr='';
disp(['Generating .m files in ',fullfile(job_struct.context_config.log,job_struct.m_path),'...']);
disp(['Generating .m files in ', fullfile( ...
job_struct.context_config.log, ...
job_struct.m_path), '...']);
%% START LOOP THROUGH DATA FILES...
for bfni=1:length(job_struct.batch_dfn);
% DO FOR EACH DATA FILE ...
%% INITIATE TMPHISTSTR...
%% SWAP THE HISTORY STRING KEY STRINGS...
batchStr=batch_strswap([batchInitStr,batchHistStr],job_struct.batch_config, ...
'datafname',job_struct.batch_dfn{bfni}, ...
'datafpath',job_struct.batch_dfp, ...
'log',job_struct.context_config.log, ...
'local_project',job_struct.context_config.local_project, ...
'local_dependency',job_struct.context_config.local_dependency, ...
'remote_project_archive',job_struct.context_config.remote_project_archive, ...
'remote_project_work',job_struct.context_config.remote_project_work, ...
'remote_dependency',job_struct.context_config.remote_dependency, ...
'mount_archive',job_struct.context_config.mount_archive, ...
'mount_work',job_struct.context_config.mount_work);
batchStr = batch_strswap([batchInitStr, batchHistStr], ...
job_struct.batch_config, ...
'datafname', job_struct.batch_dfn{bfni}, ...
'datafpath', job_struct.batch_dfp, ...
'log', job_struct.context_config.log, ...
'local_project', job_struct.context_config.local_project, ...
'local_dependency', job_struct.context_config.local_dependency, ...
'remote_project_archive', job_struct.context_config.remote_project_archive, ...
'remote_project_work', job_struct.context_config.remote_project_work, ...
'remote_dependency', job_struct.context_config.remote_dependency, ...
'mount_archive', job_struct.context_config.mount_archive, ...
'mount_work', job_struct.context_config.mount_work);
if strcmp(job_struct.batch_config.software,'none');
batchStr=['system(''',strtrim(batchStr),''')'];
if strcmp(job_struct.batch_config.software, 'none');
batchStr = ['system(''', strtrim(batchStr), ''')'];
end
%% SAVE THE STRSWAPPED HISTORY STRING TO M FILE IN THE TIME STAMPED LOG PATH...
% save cBatchFName m file...
[cPath,root_dfn,cExt]=fileparts(job_struct.batch_dfn{bfni});
c_mfn=[root_hfn,'_',root_dfn,'.m'];
% New Naming Scheme to use batch config jobname swapstring
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
try
c_mfn = key_strswap(job_struct.batch_config.job_name,'batch_dfn',char(job_struct.batch_dfn{bfni}));
c_mfn = key_strswap(c_mfn,'batch_hfn',char(job_struct.batch_hfn));
c_mfn = [c_mfn,'.m'];
c_mfn = strtrim(c_mfn);
catch
%If there sting is not acceptable default to:
[cPath,root_dfn,cExt]=fileparts(job_struct.batch_dfn{bfni});
c_mfn=[root_hfn,'_',root_dfn,'.m'];
end
if strcmp(c_mfn,'.m')
%If they did not use a batch config containing jobname default to:
[cPath,root_dfn,cExt]=fileparts(job_struct.batch_dfn{bfni});
c_mfn=[root_hfn,'_',root_dfn,'.m'];
end
if length(c_mfn) >= 64
error('Script name too long! Please change your job name config.')
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
if isempty(job_struct.batch_config.mfile_name)
job_struct.batch_config.mfile_name = ...
job_struct.batch_config.job_name;
end
fidM=fopen(fullfile(job_struct.context_config.log,job_struct.m_path,c_mfn),'w');%WRITE M FILE TO LOG PATH...
fwrite(fidM,batchStr,'char');
try
c_mfn = key_strswap(job_struct.batch_config.mfile_name, ...
'batch_dfn', char(job_struct.batch_dfn{bfni}));
c_mfn = key_strswap(c_mfn, 'batch_hfn', char(job_struct.batch_hfn));
c_mfn = [c_mfn, '.m'];
c_mfn = strtrim(c_mfn);
catch
%If there sting is not acceptable default to:
[~, root_dfn, ~] = fileparts(job_struct.batch_dfn{bfni});
c_mfn=[root_hfn, '_', root_dfn, '.m'];
end
if strcmp(c_mfn, '.m')
%If they did not use a batch config containing jobname default to:
[~, root_dfn, ~] = fileparts(job_struct.batch_dfn{bfni});
c_mfn = [root_hfn, '_', root_dfn, '.m'];
end
if length(c_mfn) >= 64
error('Script name too long! Please change your job name config.')
end
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%WRITE M FILE TO LOG PATH...
fidM = fopen(fullfile(job_struct.context_config.log, ...
job_struct.m_path,c_mfn),'w');
fwrite(fidM, batchStr, 'char');
fclose(fidM);
job_struct.exec_str{bfni}='';
job_struct.exec_str{bfni} = '';
end
end
......@@ -42,39 +42,62 @@
function job_struct=rsub_sys(job_struct, driver)
tjob = job_struct(end);
cc = tjob.context_config;
%% SFTP FROM MATLAB M FILES TO THE REMOTE LOG...
zip([fullfile(job_struct(end).context_config.log, ...
job_struct(end).m_path),'.zip'], ...
'*.*',fullfile(job_struct(end).context_config.log, ...
job_struct(end).m_path));
job_struct(end).remote_log = fullfile(job_struct(end).remote_work, ...
job_struct(end).context_config.log);
zip([fullfile(cc.log, tjob.m_path),'.zip'], ...
'*.*', ...
fullfile(cc.log, tjob.m_path));
tjob.remote_log = fullfile(tjob.remote_work, ...
cc.log);
% adjust path delimiters if necessary...
job_struct(end).remote_log = strrep(job_struct(end).remote_log,'\','/');
tjob.remote_log = strrep(tjob.remote_log, '\', '/');
% Copy back to our alias
job_struct(end).remote_log = tjob.remote_log;
%% SECURE COPY M FILE TO REMOTE HOST...
system(['scp ', ...
fullfile(cd,job_struct(end).context_config.log, ...'
[job_struct(end).m_path,'.zip']), ' ', ...
job_struct(end).context_config.remote_user_name,'@', ...
job_struct(end).context_config.remote_exec_host,':',job_struct(end).remote_log]);
cmdstr = sprintf('scp %s %s@%s:%s', ...
fullfile(cd, cc.log, [tjob.m_path,'.zip']), ...
cc.remote_user_name, ...
cc.remote_exec_host, ...
tjob.remote_log);
system_cmd_or_error(cmdstr, 'Failed to copy to remote');
%% GET THE QSUBSTR... UPDATE TO EXECUTION OF BASH SCRIPT...
% TODO(brad) fix these so they are readable
sys_str=sprintf('%s "%s"',['ssh ', ...
job_struct(end).context_config.remote_user_name,'@', ...
job_struct(end).context_config.remote_exec_host], ...
['cd ',job_struct(end).remote_work,'; ', ...
'unzip ',[fullfile(job_struct(end).context_config.log,job_struct(end).m_path),'.zip -d '], ...
fullfile(job_struct(end).context_config.log,job_struct(end).m_path),'; ', ...
'rm ',[fullfile(job_struct(end).context_config.log,job_struct(end).m_path),'.zip -d '], '; ', ...
' chmod 755 ',fullfile(job_struct(end).context_config.log,job_struct(end).m_path),'/submit.sh; ', ...
fullfile(job_struct(end).context_config.log,job_struct(end).m_path),'/submit.sh;']);
[status,result_str] = system(sys_str, '-echo');
long_mpath = fullfile(cc.log, tjob.m_path);
sshhost = sprintf('%s@%s', cc.remote_user_name, cc.remote_exec_host);
sys_str = ['ssh %s ''cd %s && unzip %s.zip -d %s' ...
' && rm %s.zip && chmod +x %s/submit.sh && %s/submit.sh'''];
cmdstr = sprintf(sys_str, sshhost, ...
tjob.remote_work, ...
long_mpath, long_mpath, ...
long_mpath, ...
long_mpath, ...
long_mpath);
result_str = system_cmd_or_error(cmdstr, 'Failed to submit script');
%% COLLECT THE NEW JOBIDS FROM THE SCHEDULER...
% New method uses read_jobs from driver to read in the jobids as a cellstr
job_struct(end).jobids = read_jobs(driver, result_str);
end
function result_str = system_cmd_or_error(cmd, msg)
[exit_status, result_str] = system(cmd, '-echo');
if exit_status ~= 0
logging_log('ERROR', msg);
logging_log('INFO', sprintf('\tCommand was: %s', cmd));
error('System error: %s', msg);
end
end
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment