## Note that the special string AppName will be replaced upon initialisation
## in all cases with the relavent app name (DaVinci, Gauss etc...)
import os, tempfile, pprint, sys
from GangaGaudi.Lib.Applications.Gaudi import Gaudi
from GangaGaudi.Lib.Applications.GaudiUtils import fillPackedSandbox
from GangaLHCb.Lib.Applications.AppsBaseUtils import available_apps, guess_version, available_packs
from GangaLHCb.Lib.Applications.AppsBaseUtils import backend_handlers, activeSummaryItems
from Ganga.GPIDev.Lib.File.FileBuffer import FileBuffer
from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
from Ganga.GPIDev.Schema import *
from Ganga.Utility.Shell import Shell
from GangaLHCb.Lib.Applications.PythonOptionsParser import PythonOptionsParser
from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
from Ganga.Utility.Config import getConfig
from Ganga.Utility.files import expandfilename
from Ganga.Utility.execute import execute
import Ganga.Utility.logging
import GangaLHCb.Lib.Applications.CMTscript
import pickle
import subprocess
from GangaLHCb.Lib.Applications import XMLPostProcessor
from Ganga.Core.exceptions import ApplicationConfigurationError
logger = Ganga.Utility.logging.getLogger()
class Kepler(Gaudi):
_name = 'Kepler'
_category = 'applications'
#__doc__ = GaudiDocString('AppName')
_schema = Gaudi._schema.inherit_copy()
docstr = 'The package the application belongs to (e.g. "Sim", "Phys")'
_schema.datadict['package'] = SimpleItem(defvalue=None,
typelist=['str','type(None)'],
doc=docstr)
docstr = 'The package where your top level requirements file is read ' \
'from. Can be written either as a path ' \
'\"Tutorial/Analysis/v6r0\" or in a CMT style notation ' \
'\"Analysis v6r0 Tutorial\"'
_schema.datadict['masterpackage'] = SimpleItem(defvalue=None,
typelist=['str','type(None)'],
doc=docstr)
_schema.datadict['platform'] = SimpleItem( defvalue="x86_64-slc6-gcc48-opt" )
docstr = 'Extra options to be passed onto the SetupProject command '\
'used for configuring the environment. As an example '\
'setting it to \'--dev\' will give access to the DEV area. '\
'For full documentation of the available options see '\
'https://twiki.cern.ch/twiki/bin/view/LHCb/SetupProject'
_schema.datadict['setupProjectOptions'] = SimpleItem(defvalue='',
typelist=['str','type(None)'],
doc=docstr)
_exportmethods = Gaudi._exportmethods[:]
_exportmethods += ['readInputData']
def _get_default_version(self, gaudi_app):
return guess_version(gaudi_app)
def _auto__init__(self):
self.appname='Kepler'
# super(self.appname, self)._auto__init__()
def postprocess(self):
XMLPostProcessor.postprocess(self,logger)
def readInputData(self,optsfiles,extraopts=False):
def dummyfile():
temp_fd,temp_filename=tempfile.mkstemp(text=True,suffix='.py')
os.write(temp_fd,"Dummy file to keep the Optionsparser happy")
os.close(temp_fd)
return temp_filename
if type(optsfiles)!=type([]): optsfiles=[optsfiles]
if len(optsfiles)==0: optsfiles.append(dummyfile())
if extraopts: extraopts=self.extraopts
else: extraopts=""
# parser = check_inputs(optsfiles, extraopts, self.env)
try:
parser = PythonOptionsParser(optsfiles,extraopts,self.getenv(False))
except Exception, e:
msg = 'Unable to parse the job options. Please check options ' \
'files and extraopts.'
raise ApplicationConfigurationError(None,msg)
return GPIProxyObjectFactory(parser.get_input_data())
def getpack(self, options=''):
"""Performs a getpack on the package given within the environment
of the application. The unix exit code is returned
"""
command = 'getpack ' + options + '\n'
if options == '':
command = 'getpack -i'
return CMTscript.CMTscript(self,command)
def make(self, argument=None):
"""Performs a CMT make on the application. The unix exit code is
returned. Any arguments given are passed onto CMT as in
dv.make('clean').
"""
config = Ganga.Utility.Config.getConfig('GAUDI')
command = config['make_cmd']
if argument:
command+=' '+argument
return CMTscript.CMTscript(self,command)
def cmt(self, command):
"""Execute a cmt command in the cmt user area pointed to by the
application. Will execute the command "cmt <command>" after the
proper configuration. Do not include the word "cmt" yourself. The
unix exit code is returned."""
command = '###CMT### ' + command
return CMTscript.CMTscript(self,command)
def _getshell(self):
opts = ''
if self.setupProjectOptions: opts = self.setupProjectOptions
fd = tempfile.NamedTemporaryFile()
script = '#!/bin/sh\n'
if self.user_release_area:
script += 'User_release_area=%s; export User_release_area\n' % \
expandfilename(self.user_release_area)
if self.platform:
script += '. `which LbLogin.sh` -c %s\n' % self.platform
useflag = ''
cmd = '. SetupProject.sh %s %s %s %s' % (useflag, opts, self.appname, self.version)
script += '%s \n' % cmd
fd.write(script)
fd.flush()
logger.debug(script)
self.shell = Shell(setup=fd.name)
if (not self.shell): raise ApplicationConfigurationError(None,'Shell not created.')
logger.debug(pprint.pformat(self.shell.env))
fd.close()
app_ok = False
ver_ok = False
for var in self.shell.env:
if var.find(self.appname) >= 0: app_ok = True
if self.shell.env[var].find(self.version) >= 0: ver_ok = True
if not app_ok or not ver_ok:
msg = 'Command "%s" failed to properly setup environment.' % cmd
logger.error(msg)
raise ApplicationConfigurationError(None,msg)
import copy
self.env = copy.deepcopy( self.shell.env )
return self.shell.env
def _get_parser(self):
optsfiles = [fileitem.name for fileitem in self.optsfile]
# add on XML summary
extraopts = ''
if self.extraopts:
extraopts += self.extraopts
try:
parser = PythonOptionsParser(optsfiles,extraopts,self.getenv(False))
except ApplicationConfigurationError, e:
# fix this when preparing not attached to job
msg2=''
try:
debug_dir = self.getJobObject().getDebugWorkspace().getPath()
msg2+='You can also view this from within ganga '\
'by doing job.peek(\'../debug/gaudirun.<whatever>\').'
except:
debug_dir = tempfile.mkdtemp()
messages = e.message.split('###SPLIT###')
if len(messages) is 2:
stdout = open(debug_dir + '/gaudirun.stdout','w')
stderr = open(debug_dir + '/gaudirun.stderr','w')
stdout.write(messages[0])
stderr.write(messages[1])
stdout.close()
stderr.close()
msg = 'Unable to parse job options! Please check options ' \
'files and extraopts. The output and error streams from gaudirun.py can be ' \
'found in %s and %s respectively . ' % (stdout.name, stderr.name)
else:
f = open(debug_dir + '/gaudirun.out','w')
f.write(e.message)
f.close()
msg = 'Unable to parse job options! Please check options ' \
'files and extraopts. The output from gaudirun.py can be ' \
'found in %s . ' % f.name
msg+=msg2
# logger.error(msg)
raise ApplicationConfigurationError(None,msg)
return parser
def _parse_options(self):
try:
parser = self._get_parser()
except ApplicationConfigurationError, e:
raise e
share_dir = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
'shared',
getConfig('Configuration')['user'],
self.is_prepared.name)
fillPackedSandbox([FileBuffer('options.pkl',parser.opts_pkl_str)],
os.path.join(share_dir,
'inputsandbox',
'_input_sandbox_%s.tar' % self.is_prepared.name))
inputdata = parser.get_input_data()
if len(inputdata.files) > 0:
logger.warning('Found inputdataset defined in optsfile, '\
'this will get pickled up and stored in the '\
'prepared state. Any change to the options/data will '\
'therefore require an unprepare first.')
logger.warning('NOTE: the prefered way of working '\
'is to define inputdata in the job.inputdata field. ')
logger.warning('Data defined in job.inputdata will superseed optsfile data!')
logger.warning('Inputdata can be transfered from optsfiles to the job.inputdata field '\
'using job.inputdata = job.application.readInputData(optsfiles)')
share_path = os.path.join(share_dir,'inputdata')
if not os.path.isdir(share_path): os.makedirs(share_path)
f=open(os.path.join(share_path,'options_data.pkl'),'w+b')
pickle.dump(inputdata, f)
f.close()
share_path = os.path.join(share_dir,'output')
if not os.path.isdir(share_path): os.makedirs(share_path)
f=open(os.path.join(share_path,'options_parser.pkl'),'w+b')
pickle.dump(parser, f)
f.close()
from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
for (backend, handler) in backend_handlers().iteritems():
allHandlers.add('Kepler', backend, handler)
iaro