Newer
Older
TB_Chris / Kepler / GangaPlugin / Lib / .svn / text-base / KeplerApp.py.svn-base
  1. ## Note that the special string AppName will be replaced upon initialisation
  2. ## in all cases with the relavent app name (DaVinci, Gauss etc...)
  3. import os, tempfile, pprint, sys
  4. from GangaGaudi.Lib.Applications.Gaudi import Gaudi
  5. from GangaGaudi.Lib.Applications.GaudiUtils import fillPackedSandbox
  6. from GangaLHCb.Lib.Applications.AppsBaseUtils import available_apps, guess_version, available_packs
  7. from GangaLHCb.Lib.Applications.AppsBaseUtils import backend_handlers, activeSummaryItems
  8. from Ganga.GPIDev.Lib.File.FileBuffer import FileBuffer
  9. from Ganga.GPIDev.Base.Proxy import GPIProxyObjectFactory
  10. from Ganga.GPIDev.Schema import *
  11. from Ganga.Utility.Shell import Shell
  12. from GangaLHCb.Lib.Applications.PythonOptionsParser import PythonOptionsParser
  13. from Ganga.GPIDev.Adapters.StandardJobConfig import StandardJobConfig
  14. from Ganga.Utility.Config import getConfig
  15. from Ganga.Utility.files import expandfilename
  16. from Ganga.Utility.execute import execute
  17. import Ganga.Utility.logging
  18. import GangaLHCb.Lib.Applications.CMTscript
  19. import pickle
  20. import subprocess
  21. from GangaLHCb.Lib.Applications import XMLPostProcessor
  22. from Ganga.Core.exceptions import ApplicationConfigurationError
  23.  
  24. logger = Ganga.Utility.logging.getLogger()
  25.  
  26. class Kepler(Gaudi):
  27. _name = 'Kepler'
  28. _category = 'applications'
  29. #__doc__ = GaudiDocString('AppName')
  30. _schema = Gaudi._schema.inherit_copy()
  31. docstr = 'The package the application belongs to (e.g. "Sim", "Phys")'
  32. _schema.datadict['package'] = SimpleItem(defvalue=None,
  33. typelist=['str','type(None)'],
  34. doc=docstr)
  35. docstr = 'The package where your top level requirements file is read ' \
  36. 'from. Can be written either as a path ' \
  37. '\"Tutorial/Analysis/v6r0\" or in a CMT style notation ' \
  38. '\"Analysis v6r0 Tutorial\"'
  39. _schema.datadict['masterpackage'] = SimpleItem(defvalue=None,
  40. typelist=['str','type(None)'],
  41. doc=docstr)
  42. _schema.datadict['platform'] = SimpleItem( defvalue="x86_64-slc6-gcc48-opt" )
  43. docstr = 'Extra options to be passed onto the SetupProject command '\
  44. 'used for configuring the environment. As an example '\
  45. 'setting it to \'--dev\' will give access to the DEV area. '\
  46. 'For full documentation of the available options see '\
  47. 'https://twiki.cern.ch/twiki/bin/view/LHCb/SetupProject'
  48. _schema.datadict['setupProjectOptions'] = SimpleItem(defvalue='',
  49. typelist=['str','type(None)'],
  50. doc=docstr)
  51. _exportmethods = Gaudi._exportmethods[:]
  52. _exportmethods += ['readInputData']
  53. def _get_default_version(self, gaudi_app):
  54. return guess_version(gaudi_app)
  55. def _auto__init__(self):
  56. self.appname='Kepler'
  57. # super(self.appname, self)._auto__init__()
  58.  
  59. def postprocess(self):
  60. XMLPostProcessor.postprocess(self,logger)
  61.  
  62. def readInputData(self,optsfiles,extraopts=False):
  63. def dummyfile():
  64. temp_fd,temp_filename=tempfile.mkstemp(text=True,suffix='.py')
  65. os.write(temp_fd,"Dummy file to keep the Optionsparser happy")
  66. os.close(temp_fd)
  67. return temp_filename
  68.  
  69. if type(optsfiles)!=type([]): optsfiles=[optsfiles]
  70. if len(optsfiles)==0: optsfiles.append(dummyfile())
  71. if extraopts: extraopts=self.extraopts
  72. else: extraopts=""
  73. # parser = check_inputs(optsfiles, extraopts, self.env)
  74. try:
  75. parser = PythonOptionsParser(optsfiles,extraopts,self.getenv(False))
  76. except Exception, e:
  77. msg = 'Unable to parse the job options. Please check options ' \
  78. 'files and extraopts.'
  79. raise ApplicationConfigurationError(None,msg)
  80.  
  81. return GPIProxyObjectFactory(parser.get_input_data())
  82.  
  83. def getpack(self, options=''):
  84. """Performs a getpack on the package given within the environment
  85. of the application. The unix exit code is returned
  86. """
  87. command = 'getpack ' + options + '\n'
  88. if options == '':
  89. command = 'getpack -i'
  90. return CMTscript.CMTscript(self,command)
  91. def make(self, argument=None):
  92. """Performs a CMT make on the application. The unix exit code is
  93. returned. Any arguments given are passed onto CMT as in
  94. dv.make('clean').
  95. """
  96. config = Ganga.Utility.Config.getConfig('GAUDI')
  97. command = config['make_cmd']
  98. if argument:
  99. command+=' '+argument
  100. return CMTscript.CMTscript(self,command)
  101.  
  102. def cmt(self, command):
  103. """Execute a cmt command in the cmt user area pointed to by the
  104. application. Will execute the command "cmt <command>" after the
  105. proper configuration. Do not include the word "cmt" yourself. The
  106. unix exit code is returned."""
  107. command = '###CMT### ' + command
  108. return CMTscript.CMTscript(self,command)
  109.  
  110. def _getshell(self):
  111. opts = ''
  112. if self.setupProjectOptions: opts = self.setupProjectOptions
  113.  
  114. fd = tempfile.NamedTemporaryFile()
  115. script = '#!/bin/sh\n'
  116. if self.user_release_area:
  117. script += 'User_release_area=%s; export User_release_area\n' % \
  118. expandfilename(self.user_release_area)
  119. if self.platform:
  120. script += '. `which LbLogin.sh` -c %s\n' % self.platform
  121. useflag = ''
  122. cmd = '. SetupProject.sh %s %s %s %s' % (useflag, opts, self.appname, self.version)
  123. script += '%s \n' % cmd
  124. fd.write(script)
  125. fd.flush()
  126. logger.debug(script)
  127.  
  128. self.shell = Shell(setup=fd.name)
  129. if (not self.shell): raise ApplicationConfigurationError(None,'Shell not created.')
  130. logger.debug(pprint.pformat(self.shell.env))
  131. fd.close()
  132. app_ok = False
  133. ver_ok = False
  134. for var in self.shell.env:
  135. if var.find(self.appname) >= 0: app_ok = True
  136. if self.shell.env[var].find(self.version) >= 0: ver_ok = True
  137. if not app_ok or not ver_ok:
  138. msg = 'Command "%s" failed to properly setup environment.' % cmd
  139. logger.error(msg)
  140. raise ApplicationConfigurationError(None,msg)
  141.  
  142. import copy
  143. self.env = copy.deepcopy( self.shell.env )
  144.  
  145. return self.shell.env
  146.  
  147.  
  148. def _get_parser(self):
  149. optsfiles = [fileitem.name for fileitem in self.optsfile]
  150. # add on XML summary
  151.  
  152. extraopts = ''
  153. if self.extraopts:
  154. extraopts += self.extraopts
  155. try:
  156. parser = PythonOptionsParser(optsfiles,extraopts,self.getenv(False))
  157. except ApplicationConfigurationError, e:
  158. # fix this when preparing not attached to job
  159. msg2=''
  160. try:
  161. debug_dir = self.getJobObject().getDebugWorkspace().getPath()
  162. msg2+='You can also view this from within ganga '\
  163. 'by doing job.peek(\'../debug/gaudirun.<whatever>\').'
  164. except:
  165. debug_dir = tempfile.mkdtemp()
  166. messages = e.message.split('###SPLIT###')
  167. if len(messages) is 2:
  168. stdout = open(debug_dir + '/gaudirun.stdout','w')
  169. stderr = open(debug_dir + '/gaudirun.stderr','w')
  170. stdout.write(messages[0])
  171. stderr.write(messages[1])
  172. stdout.close()
  173. stderr.close()
  174. msg = 'Unable to parse job options! Please check options ' \
  175. 'files and extraopts. The output and error streams from gaudirun.py can be ' \
  176. 'found in %s and %s respectively . ' % (stdout.name, stderr.name)
  177. else:
  178. f = open(debug_dir + '/gaudirun.out','w')
  179. f.write(e.message)
  180. f.close()
  181. msg = 'Unable to parse job options! Please check options ' \
  182. 'files and extraopts. The output from gaudirun.py can be ' \
  183. 'found in %s . ' % f.name
  184. msg+=msg2
  185. # logger.error(msg)
  186. raise ApplicationConfigurationError(None,msg)
  187. return parser
  188.  
  189.  
  190. def _parse_options(self):
  191. try:
  192. parser = self._get_parser()
  193. except ApplicationConfigurationError, e:
  194. raise e
  195.  
  196. share_dir = os.path.join(expandfilename(getConfig('Configuration')['gangadir']),
  197. 'shared',
  198. getConfig('Configuration')['user'],
  199. self.is_prepared.name)
  200. fillPackedSandbox([FileBuffer('options.pkl',parser.opts_pkl_str)],
  201. os.path.join(share_dir,
  202. 'inputsandbox',
  203. '_input_sandbox_%s.tar' % self.is_prepared.name))
  204. inputdata = parser.get_input_data()
  205. if len(inputdata.files) > 0:
  206. logger.warning('Found inputdataset defined in optsfile, '\
  207. 'this will get pickled up and stored in the '\
  208. 'prepared state. Any change to the options/data will '\
  209. 'therefore require an unprepare first.')
  210. logger.warning('NOTE: the prefered way of working '\
  211. 'is to define inputdata in the job.inputdata field. ')
  212. logger.warning('Data defined in job.inputdata will superseed optsfile data!')
  213. logger.warning('Inputdata can be transfered from optsfiles to the job.inputdata field '\
  214. 'using job.inputdata = job.application.readInputData(optsfiles)')
  215. share_path = os.path.join(share_dir,'inputdata')
  216. if not os.path.isdir(share_path): os.makedirs(share_path)
  217. f=open(os.path.join(share_path,'options_data.pkl'),'w+b')
  218. pickle.dump(inputdata, f)
  219. f.close()
  220.  
  221. share_path = os.path.join(share_dir,'output')
  222. if not os.path.isdir(share_path): os.makedirs(share_path)
  223. f=open(os.path.join(share_path,'options_parser.pkl'),'w+b')
  224. pickle.dump(parser, f)
  225. f.close()
  226.  
  227. from Ganga.GPIDev.Adapters.ApplicationRuntimeHandlers import allHandlers
  228. for (backend, handler) in backend_handlers().iteritems():
  229. allHandlers.add('Kepler', backend, handler)