""" CCP4TaskManager.py: CCP4 GUI Project Copyright (C) 2010 University of York This library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3, modified in accordance with the provisions of the license to address the requirements of UK law. You should have received a copy of the modified GNU Lesser General Public License along with this library. If not, copies may be downloaded from http://www.ccp4.ac.uk/ccp4license.php This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. """ """ Liz Potterton Sept 2010 - Class to keep track of all CCP4Tasks """ # To see the demo/test wrappers listed in menu set SHOW_WRAPPERS=True SHOW_WRAPPERS = False # To see the timings for loading modules set TIMING=True TIMING = False from CCP4Config import GRAPHICAL,DEVELOPER from CCP4ErrorHandling import * from CCP4Modules import WEBBROWSER,PROJECTSMANAGER,WORKFLOWMANAGER,CUSTOMTASKMANAGER MODULE_ORDER = ['data_entry','data_processing','data_reduction','expt_phasing', 'bioinformatics', 'molecular_replacement','model_building', 'refinement', 'ligands', 'validation','export','expt_data_utility','model_data_utility', 'developer_tools', 'preferences','wrappers','test'] MODULE_TITLES = { 'data_entry' : 'Import merged data, sequences, alignments or coordinates', 'data_processing' : 'Integrate X-ray images', 'data_reduction' : 'X-ray data reduction and analysis', 'expt_phasing' : 'Experimental phasing', 'bioinformatics' : 'Bioinformatics including model preparation for Molecular Replacement', 'molecular_replacement' : 'Molecular Replacement', 'model_building' : 'Model building and Graphics', 'refinement' : 'Refinement', 'ligands' : 'Ligands', 'validation' : 'Validation and analysis', 'export' : 'Export and Deposition', 'expt_data_utility' : 'Reflection data tools', 'model_data_utility' : 'Coordinate data tools', 'developer_tools' : 'Developer tools', 'preferences' : 'Preferences', 'wrappers' : 'Wrappers', 'demo' : 'Demo code for developers only', 'test' : 'Test code for developers only' } #'refln_data_analysis' : 'Reflection data analysis', MODULE_DEFAULTS = { 'molecular_replacement' : [ 'mrbump_basic', 'phaser_simple','phaser_pipeline', 'molrep_pipe', 'molrep_den','csymmatch', 'parrot', 'phaser_rnp_pipeline' ], 'data_reduction' : [ 'aimless_pipe', 'freerflag', 'matthews', 'molrep_selfrot' ] , 'data_entry' : [ 'import_merged', 'import_xia2' , 'ProvideSequence' , 'ProvideAlignment' ], 'data_processing' : [ 'xia2_dials', 'xia2_xds', 'imosflm' ] , 'expt_phasing' : [ 'crank2', 'shelx', 'phaser_EP_AUTO' ], 'refinement' : [ 'prosmart_refmac'], 'bioinformatics' : [ 'ccp4mg_edit_model' , 'ccp4mg_edit_nomrbump', 'chainsaw', 'sculptor','phaser_ensembler','clustalw' ], 'model_building' : [ 'buccaneer_build_refine_mr', 'coot_rebuild','coot_script_lines', 'coot_find_waters' ], 'validation' : [ 'edstats', 'privateer_validate', 'qtpisa' ], 'export' : [ 'PrepareDeposit'], 'expt_data_utility' : [ 'pointless_reindexToMatch', 'phaser_EP_LLG', 'cmapcoeff', 'chltofom', 'cphasematch', 'ctruncate', 'splitMtz' ], 'model_data_utility' : [ 'csymmatch', 'gesamt', 'coordinate_selector', 'qtpisa' ], 'developer_tools' : [ ], 'test' : ['demo_copycell'] } # The taskname ( the key) maps to different directory and def.xml filename TASKNAME2DIR = { 'refmac' : 'refmac_i2' } def TASKMANAGER(): if CTaskManager.insts is None: CTaskManager.insts = CTaskManager() return CTaskManager.insts def LISTTASKS(ifPrint = True): tm = TASKMANAGER() tm.buildLookup() text = '\n\nThe current menu\n\n' tree = tm.taskTree() for moduleName,moduleTitle,taskList in tree: text += "{0:30} {1:50}\n".format(moduleName,moduleTitle) for taskName in taskList: text+= " {0:30} {1:50}\n".format(taskName, tm.getTaskAttribute(taskName,'TASKTITLE')) text+= tm.printLookup() if ifPrint: print text return text class CTaskManager: ERROR_CODES = { 101 : { 'description' : 'No definintion (def.xml) file for task' }, 102 : { 'description' : 'Undefined error opening def file in task viewer' }, 103 : { 'description' : 'Undefined error loading task viewer in browser' }, 104 : { 'description' : 'Attempting to open task viewer in non-graphical mode' }, 105 : { 'description' : 'taskName not recognised in whatNext' }, 106 : { 'description' : 'whatNext error importing module' }, 107 : { 'description' : 'whatNext error calling module whatNext' }, 108 : { 'description' : 'Error importing Python module' }, 120 : { 'description' : 'Error loading local task customisation file' } } insts = None def __init__(self): import os import CCP4Utils self._guiSearchPath = [os.path.join(CCP4Utils.getCCP4I2Dir(),'tasks','*')] self._searchPath = [os.path.join(CCP4Utils.getCCP4I2Dir(),'wrappers','*','script'),os.path.join(CCP4Utils.getCCP4I2Dir(),'wrappers2','*','script'),os.path.join(CCP4Utils.getCCP4I2Dir(),'pipelines','*','script'),os.path.join(CCP4Utils.getCCP4I2Dir(),'pipelines','*','wrappers','*','script'),os.path.join(CCP4Utils.getCCP4I2Dir(),'deprecated')] self.taskLookup= {} self.moduleLookup= {} self.scriptLookup = {} self.reportLookup = {} self.performanceClassLookup = {} self.taskPerformanceClassLookup = {} self.i1TaskLookup = None #Local customisations self.blockLocal = [] errReport = self.buildLookup() # Do i2 task title lookup separately on demand if len(errReport)>0: print 'ERROR LOADING TASK MODULES' print errReport.report() errReport = self.loadCustomisation() if len(errReport)>0: print 'ERROR LOADING LOCAL TASK CUSTOMISATIONS' print errReport.report() def searchPath(self): return self._guiSearchPath + self._searchPath def appendSearchPath(self,path): # Append a path which could include wildcard * - must work # with glob import os path = os.path.abspath(path) if not self._searchPath.count(path): self._searchPath.append(path) def buildLookup(self): import sys graphical = GRAPHICAL() import inspect,os,re import CCP4Utils myErrorReport = CErrorReport() loadTarget = [ [self._searchPath,True] ] if graphical: import CCP4TaskWidget loadTarget.insert(0,[self._guiSearchPath,False]) import CCP4PluginScript, CCP4ReportParser , CCP4PerformanceData # Get the performance data classes module,err = CCP4Utils.importFileModule(os.path.join(CCP4Utils.getCCP4I2Dir(),'core','CCP4PerformanceData.py'),report=False) clsList = inspect.getmembers(module,inspect.isclass) for className,cls in clsList: if issubclass(cls,CCP4PerformanceData.CPerformanceIndicator) and not cls == CCP4PerformanceData.CPerformanceIndicator: self.performanceClassLookup[className] = cls #print 'TASKMANAGER.performanceClassLookup',self.performanceClassLookup if TIMING: import time t3 = time.time() moduleLookup = { 'wrappers' : [] } guiedTasks = [] """ Cope with dodgy SLOW Python load of modules hosted on NFS. This is only done with the i2 Python stuff. It is arguable that other stuff (mmdb2,mmut,BioPython,etc.,phaser also needs this treatment. However this can get quickly complicated and slow (phaser depends on cctbx, etc.) """ if sys.platform == "linux2": try: import subprocess import copy nfs = False nfsDirs = [] nfsall = subprocess.check_output(['mount','-l','-t','nfs']).split('\n') nfsall4 = subprocess.check_output(['mount','-l','-t','nfs4']).split('\n') nfsall.extend(nfsall4) for nfs in nfsall: if len(nfs)>0: splits = nfs.split() if len(splits)>2: theDir = splits[2] nfsDirs.append(theDir) thisF = copy.deepcopy(__file__) while len(thisF)>1: if thisF in nfsDirs: print "Probably NFS",thisF nfs = True break thisF = os.path.dirname(thisF) if nfs: import tempfile tempd = tempfile.mkdtemp() sys.path.insert(0,tempd) print "Copying to Local Temp Disk ..." pyFileList = [] for target,isWrapper in loadTarget: pyFileList.extend(CCP4Utils.globSearchPath(target,'*.pyc')) for pyFile in pyFileList[:]: moduleName = os.path.splitext(os.path.basename(pyFile))[0] f = open(pyFile) b = f.read() f.close() modfname = os.path.join(tempd,moduleName+'.pyc') ft = open(modfname,"wb+") ft.write(b) ft.close() pyFileList = [] for target,isWrapper in loadTarget: pyFileList.extend(CCP4Utils.globSearchPath(target,'*.py')) for pyFile in pyFileList[:]: moduleName = os.path.splitext(os.path.basename(pyFile))[0] modfnamec = os.path.join(tempd,moduleName+'.pyc') if not os.path.exists(modfnamec): f = open(pyFile) b = f.read() f.close() modfname = os.path.join(tempd,moduleName+'.py') ft = open(modfname,"wb+") ft.write(b) ft.close() ''' # probably unnecessary to copy xml files as well xmlFileList = [] for target,isWrapper in loadTarget: xmlFileList.extend(CCP4Utils.globSearchPath(target,'*.def.xml')) for xFile in xmlFileList[:]: f = open(xFile) b = f.read() f.close() modfname = os.path.join(tempd,os.path.split(xFile)[1]) ft = open(modfname,"wb+") ft.write(b) ft.close() ''' print "Done copy to Local Temp Disk ..." except: print "Possible error trying to determine if on NFS." for target,isWrapper in loadTarget: pyFileList = CCP4Utils.globSearchPath(target,'*.py') #print 'buildLookup', target,isWrapper,pyFileList for pyFile in pyFileList: if TIMING: t1 = time.time() module,err = CCP4Utils.importFileModule(pyFile,report=False) if err is not None and (graphical or err.count('CCP4TaskWidget')==0): myErrorReport.append(self.__class__,108,stack=False,details='For file: '+str(pyFile)+'\nError:\n'+err) else: clsList = inspect.getmembers(module,inspect.isclass) #print 'CTaskManager pyFile',pyFile,clsList for className,cls in clsList: #print 'className',className,cls,issubclass(cls,CCP4TaskWidget.CTaskWidget),cls==CCP4TaskWidget.CTaskWidget if graphical and issubclass(cls,CCP4TaskWidget.CTaskWidget) and not cls==CCP4TaskWidget.CTaskWidget: # The taskName used for internal reference is the name of directory # containing CTask*.py and *.def.xml files taskName = getattr(cls,'TASKNAME',None) if taskName is None: taskName = os.path.split(os.path.split(pyFile)[0])[1] taskVersion = getattr(cls,'TASKVERSION','0.0.0') guiName = getattr(cls,'GUINAME',taskName) guiVersion = getattr(cls,'GUIVERSION','0.0.0') taskModuleList = getattr(cls,'TASKMODULE','test') maintainer = getattr(cls,'MAINTAINER','Nobody') shortTitle = getattr(cls,'SHORTTASKTITLE',None) if shortTitle is None: shortTitle = getattr(cls,'TASKTITLE',None) if not isinstance(taskModuleList,list): taskModuleList = [ taskModuleList ] if not self.taskLookup.has_key('guiName'): self.taskLookup[guiName] = {} self.taskLookup[guiName][guiVersion] = { 'class' : cls, 'taskName' : taskName, 'taskVersion' : taskVersion, 'pyFile' : pyFile, 'MAINTAINER' : maintainer, 'shortTitle' : shortTitle } for taskModule in taskModuleList: if not moduleLookup.has_key(taskModule): moduleLookup[taskModule] = [] if not guiName in moduleLookup[taskModule]: moduleLookup[taskModule].append(guiName) guiedTasks.append(taskName) #print 'buildLookup',taskName,taskModule elif issubclass(cls,CCP4PluginScript.CPluginScript) and not cls==CCP4PluginScript.CPluginScript : taskModuleList = getattr(cls,'TASKMODULE',None) if taskModuleList != 'deprecated': taskName = getattr(cls,'TASKNAME',None) if taskName is None: taskName = os.path.split(os.path.split(os.path.split(pyFile)[0])[0])[1] taskVersion = getattr(cls,'TASKVERSION','0.0.0') maintainer = getattr(cls,'MAINTAINER','Nobody') interruptLabel = getattr(cls,'INTERRUPTLABEL',None) if not isinstance(taskModuleList,list): taskModuleList = [ taskModuleList ] subTasks = getattr(cls,'SUBTASKS',[]) #print 'buildLookup CPluginScript',taskName,taskModule internal = issubclass(cls,CCP4PluginScript.CInternalPlugin) if not self.scriptLookup.has_key('taskName'): self.scriptLookup[taskName] = {} self.scriptLookup[taskName][taskVersion] = { 'class' : cls, 'subTasks': subTasks, 'pyFile' : pyFile, 'internal' : internal, 'MAINTAINER' : maintainer, 'INTERRUPTLABEL' : interruptLabel } if cls.PERFORMANCECLASS is not None and self.performanceClassLookup.has_key(cls.PERFORMANCECLASS): self.taskPerformanceClassLookup[taskName] = self.performanceClassLookup[cls.PERFORMANCECLASS] for taskModule in taskModuleList: if taskModule is not None and not taskName in guiedTasks: if not moduleLookup.has_key(taskModule): moduleLookup[taskModule] = [] if not taskName in moduleLookup[taskModule]: moduleLookup[taskModule].append(taskName) #else: # if not taskName in moduleLookup['wrappers'] and not taskName in ['customtask','workflow']: # moduleLookup['wrappers'].append(taskName) #print 'buildLookup wrappers',taskName,taskModule,cls,pyFile elif issubclass(cls,CCP4ReportParser.Report) and not cls==CCP4ReportParser.Report: taskName = getattr(cls,'TASKNAME',None) taskVersion = getattr(cls,'TASKVERSION','0.0.0') maintainer = getattr(cls,'MAINTAINER','Nobody') if taskName is None: taskName = re.sub('_report','',cls.__name__) if not self.reportLookup.has_key('taskName'): self.reportLookup[taskName] = {} self.reportLookup[taskName][taskVersion] = { 'class' : cls, 'pyFile' : pyFile, 'MAINTAINER' : maintainer, 'modes' : [ ] } try: if cls.RUNNING : self.reportLookup[taskName][taskVersion]['modes'].append('Running') if cls.FAILED : self.reportLookup[taskName][taskVersion]['modes'].append('Failed') except: pass if TIMING: t2 = time.time() if(t2-t1)>.1: print "Time to load",pyFile,t2-t1 if TIMING: t4 = time.time() print "Time to load all",t4-t3 # Sort the task order according to MODULE_DEFAULTS - allow a wrapper to be # included if it is listed in the MODULE_DEFAULTS for module,taskList in moduleLookup.items(): self.moduleLookup[module] = [] if MODULE_DEFAULTS.has_key(module): for item in MODULE_DEFAULTS[module]: if item in taskList: self.moduleLookup[module].append(item) elif item in moduleLookup['wrappers']: self.moduleLookup[module].append(item) for item in taskList: if not item in self.moduleLookup[module]: self.moduleLookup[module].append(item) else: self.moduleLookup[module].extend(taskList) if TIMING: t5 = time.time() print "Time to do rest",t5-t4 #import sys #print sys.path return myErrorReport def loadCustomisation(self): import os import CCP4Utils,CCP4File err = CErrorReport() customFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'custom_code','task_customisation.xml') if not os.path.exists(customFile): return err try: xmlFile = CCP4File.CI2XmlDataFile(customFile) body = xmlFile.getBodyEtree() except: err.append(self.__class__,120,customFile) return err self.blockLocal = [] for node in body: if node.tag in ['blocklocal','blockLocal']: for item in node: if item.tag in ['taskname','taskName']: self.blockLocal.append(str(item.text)) #print 'loadCustomisation blockLocal',self.blockLocal return err def printLookup(self): import os import CCP4Utils i2dir = CCP4Utils.getCCP4I2Dir() text = '\n\nGui (CTaskWidget) list:\n\n' classNameList = self.taskLookup.keys() classNameList.sort() for task in classNameList: for version in self.taskLookup[task].keys(): path = os.path.relpath(self.taskLookup[task][version]['pyFile'],i2dir) text += "{0:30} {4:20} {5:30} {2}\n".format(task,version,path,self.taskLookup[task][version]['MAINTAINER'],self.getTaskAttribute(task,'TASKMODULE'),self.getTaskAttribute(task,'TASKTITLE')) text += '\n\nScript (CPLuginScript) list:\n\n' classNameList = self.scriptLookup.keys() classNameList.sort() for task in classNameList: for version in self.scriptLookup[task].keys(): path = os.path.relpath(self.scriptLookup[task][version]['pyFile'],i2dir) text += "{0:30} {1:10} {3:30} {2}\n".format(task,version,path,self.scriptLookup[task][version]['MAINTAINER']) text += '\n\nReport list:\n\n' classNameList = self.reportLookup.keys() classNameList.sort() for task in classNameList: for version in self.reportLookup[task].keys(): path = os.path.relpath(self.reportLookup[task][version]['pyFile'],i2dir) text += "{0:30} {1:10} {3:30} {2}\n".format(task,version,path,self.reportLookup[task][version]['MAINTAINER'],str(self.reportLookup[task][version]['modes'])) return text def sortVersions(self,inList): return inList def getTaskData(self,name='',version=None): if self.taskLookup.has_key(name): versions = self.sortVersions(self.taskLookup[name].keys()) if version is not None and version !='0.0.0': if version in versions: return self.taskLookup[name][version] else: return self.taskLookup[name][versions[0]] return {} def getScriptData(self,name='',version=None): if self.scriptLookup.has_key(name): versions = self.sortVersions(self.scriptLookup[name].keys()) if version is not None and version !='0.0.0': if version in versions: return self.scriptLookup[name][version] else: return self.scriptLookup[name][versions[0]] return {} def getReportData(self,name='',version=None): rv = {} if self.reportLookup.has_key(name): versions = self.sortVersions(self.reportLookup[name].keys()) if version is not None and version !='0.0.0': if version in versions: rv = self.reportLookup[name][version] rv['version'] = version else: rv = self.reportLookup[name][versions[0]] rv['version'] = versions[0] return rv def getTaskWidgetClass(self,name='',version=None): data = self.getTaskData(name,version) #print 'getTaskWidgetClass',name,data, data.get('class',None) if data.get('class',None) is not None: return data['class'] if name in WORKFLOWMANAGER().getList(): data = self.getTaskData('workflow') return data.get('class',None) else: return None def getReportClass(self,name='',version=None,jobStatus=None,doReload=False): import CCP4ReportParser,CCP4DbApi data = self.getReportData(name,version) if jobStatus == 'Running remotely' : jobStatus = 'Running' if jobStatus is None or jobStatus in CCP4DbApi.FINISHED_JOB_STATUS or jobStatus in data.get('modes',[]): if doReload: # Attempt to reload the module import inspect cls = data.get('class',None) if cls is None: return None module = __import__(cls.__module__) module = reload(module) clsList = inspect.getmembers(module,inspect.isclass) for className,cls1 in clsList: if issubclass(cls1,CCP4ReportParser.Report) and getattr(cls1,'TASKNAME')==name and getattr( cls1,'TASKVERSION') == data['version']: self.reportLookup[name][data['version']]['class'] = cls1 print 'Reloading task report module for',className return cls1 # Return the currently imported cls even if reload has failed return data.get('class',None) else: return None def getPluginScriptClass(self,name='',version=None): #print 'getPluginScriptClass',self.scriptLookup.keys(),self.scriptLookup.has_key(name) data = self.getScriptData(name,version) #print 'getPluginScriptClass', cls = data.get('class',None) if cls is not None: return cls # Try if is a 'derived' gui data = self.getTaskData(name,version) pluginName = data.get('taskName',None) #print 'getPluginScriptClass ',name,data,pluginName if pluginName is not None and pluginName!=name: data = self.getScriptData(pluginName, data.get('taskVersion',None)) cls = data.get('class',None) if cls is not None: return cls if name in WORKFLOWMANAGER().getList(): data = self.getScriptData('workflow',version) cls = data.get('class',None) return cls elif name in CUSTOMTASKMANAGER().getList(): data = self.getScriptData('customtask',version) cls = data.get('class',None) return cls else: return None def getClass(self,name='',version=None): data = self.getTaskData(name,version) #print 'TASKMANAGER.getClass getTaskData',name,data if len(data)==0: data = self.getScriptData(name,version) return data.get('class',None) def isInternalPlugin(self,name='',version=None): data = self.getScriptData(name,version) return data.get('internal',False) def getTitle(self,taskName='',version=None): if taskName is None: return 'UNKNOWN' taskTitle = None cls = self.getTaskData(taskName,version).get('class',None) if cls is not None: taskTitle = getattr(cls,'TASKTITLE',None) if taskTitle is None: cls = self.getScriptData(taskName,version).get('class',None) if cls is not None: taskTitle = getattr(cls,'TASKTITLE',None) if taskTitle is not None: return taskTitle else: # Try customisations - will return same input taskName if not recognised title = WORKFLOWMANAGER().getTitle(taskName) if title == taskName: title = CUSTOMTASKMANAGER().getTitle(taskName) if title is None: title = taskName return title def getShortTitle(self,taskName='',version=None,substitute=True): if taskName is None: return 'UNKNOWN' shortTitle = self.getTaskData(taskName,version).get('shortTitle',None) if shortTitle is not None: return shortTitle else: return taskName def getAllShortTitles(self): ''' Return list if taskName,shortTitle''' ret = [] for taskName,versionDict in self.taskLookup.items(): for version,infoDict in versionDict.items(): ret.append((taskName,infoDict.get('shortTitle',taskName))) break return ret def getTaskLabel(self,taskName='',version=None,substitute=True): if taskName is None: return 'UNKNOWN' label = None cls = self.getTaskData(taskName,version).get('class',None) if cls is not None: label = getattr(cls,'TASKLABEL',None) if label is not None: return label else: return taskName def getTaskAttribute(self,taskName,attribute,version=None,default=None,script=False): if attribute in ['INTERRUPTLABEL','MAINTAINER','internal','subTasks']: #print 'getTaskAttribute using getScriptData',self.getScriptData(name=taskName,version=version) return self.getScriptData(name=taskName,version=version).get(attribute,None) if attribute in ['blockLocal']: return (taskName in self.blockLocal) if script: cls = self.getPluginScriptClass(taskName,version) else: cls = self.getClass(taskName,version) #print 'CTaskManager.getTaskAttribute',taskName,attribute,cls if cls is not None: return getattr(cls,attribute,default) else: return default def getReportAttribute(self,taskName,attribute,version=None,default=None): cls = self.getReportClass(taskName,version) #print 'CTaskManager.getReportAttribute',taskName,attribute,cls if cls is not None: return getattr(cls,attribute,default) else: return default def getPerformanceClass(self,taskName): return self.taskPerformanceClassLookup.get(taskName,None) def taskTree(self,shortTitles=False): # Return list of [module_name,taskList] tree = [] for module in MODULE_ORDER: if module not in ['preferences','test','demo','wrappers','deprecated']: if self.moduleLookup.has_key(module): taskList = [] if shortTitles: for item in self.moduleLookup[module]: taskList.append([item,self.getShortTitle(item)]) else: taskList.extend(self.moduleLookup[module]) tree.append([module,MODULE_TITLES.get(module,module),taskList]) if SHOW_WRAPPERS: for module in ['wrappers','test','demo']: if self.moduleLookup.has_key(module): taskList = [] if shortTitles: for item in self.moduleLookup[module]: taskList.append([item,self.getShortTitle(item)]) else: taskList.extend(self.moduleLookup[module]) tree.append([module,MODULE_TITLES.get(module,module),taskList]) #print 'TASKMANAGER.taskTree',tree ''' workflows = WORKFLOWMANAGER().getList() if len(workflows)>0: tree.append(['workflows','Workflows',workflows]) customtasks =CUSTOMTASKMANAGER().getList() if len(customtasks)>0: tree.append(['customTasks','Custom tasks',customtasks]) ''' return tree def openDataFile(self,fileName='',editableData=True): import CCP4File h = CCP4File.CI2XmlHeader() h.loadFromXml(fileName) taskName = str(h.pluginName) #prin t'CTaskManager.openDataFile taskName',taskName widget = self.openTask(taskName,editableData=editableData) if widget is not None: widget.loadData(fileName) def openTask(self,taskName,version=None,projectId=None,editableData=True): #print 'CTaskManager.openTask',task if not GRAPHICAL(): raise CException(self.__class__,104,'task name: '+taskName) defFile = self.lookupDefFile(taskName,version=version) if defFile is None: defFile = self.searchDefFile(taskName,version=version) if defFile is None: e = CException(self.__class__,101,'task name: '+taskName) e.warningMessage(windowTitle='Task menu') cls = self.getClass(taskName) if cls is not None: taskTitle= getattr(cls,'TASKTITLE',taskName) else: taskTitle = taskName #print 'CTaskManager.openTask',taskName,cls,taskTitle import CCP4TaskViewer widget = CCP4TaskViewer.CTaskViewer(parent=WEBBROWSER(),project=projectId) if DEVELOPER(): widget.open(fileName=defFile,taskName=taskName,taskTitle=taskTitle,editableData=editableData) else: try: widget.open(fileName=defFile,taskName=taskName,taskTitle=taskTitle,editableData=editableData) except CException as e: e.warningMessage(windowTitle='Task menu') return None except: e = CException(self.__class__,102,'task name: '+taskName) e.warningMessage(windowTitle='Task menu') return None #print 'CTaskManager.openTask widget',widget if DEVELOPER(): widget.setObjectName(taskName) WEBBROWSER().newTab(widget,taskName) widget.show() else: try: widget.setObjectName(taskName) WEBBROWSER().newTab(widget,taskName) widget.show() except CException as e: e.warningMessage(windowTitle='Task menu') return None except: e = CException(self.__class__,103,'task name: '+taskName) e.warningMessage(windowTitle='Task menu') return None return widget def searchDefFile(self,name=None,version=None): import os import CCP4Utils if name is None: return None defFileList = CCP4Utils.globSearchPath(self.searchPath(),os.path.join(name+'.def.xml')) if len(defFileList)>0: return defFileList[0] pluginName = self.getTaskAttribute(taskName=name,attribute='USEPLUGIN') if pluginName is not None: defFileList = CCP4Utils.globSearchPath(self.searchPath(),os.path.join(pluginName+'.def.xml')) if len(defFileList)>0: return defFileList[0] return None def searchXrtFile(self,name='',version=None,jobStatus=None): import os,re for path in self.searchPath(): if jobStatus == 'Running': fileName = os.path.join(re.sub('\*',name,path),name+'.running.xrt') #print 'TASKMANAGER.searchXrtFile',fileName,os.path.exists(fileName) if os.path.exists(fileName): return fileName else: fileName = os.path.join(re.sub('\*',name,path),name+'.xrt') if os.path.exists(fileName): return fileName return None def searchReferenceFile(self,name=None,version=None,format='medline',drillDown=False): import os import CCP4Utils if name is None: return [] defFileList = CCP4Utils.globSearchPath(self.searchPath(),name+'.'+format+'.txt') if drillDown: subTaskList = self.getScriptData(name=name,version=version).get('subTasks',[]) #print 'CTaskManager.searchReferenceFile subTaskList',name,subTaskList for subTask in subTaskList: defFileList.extend(self.searchReferenceFile(name=subTask,format=format,drillDown=True)) return defFileList def searchHelpFile(self,name=None,version=None): import os import CCP4Utils if name is None: return None #helpFileList = CCP4Utils.globSearchPath(self.searchPath(),os.path.join(name+'.html')) #if len(helpFileList)>0: return helpFileList[0] helpFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','tasks',name,'index.html') if os.path.exists(helpFile): return helpFile helpFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','tasks',name,name+'.html') if os.path.exists(helpFile): return helpFile helpFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'pipelines',name,'docs','index.html') if os.path.exists(helpFile): return helpFile return None def searchIconFile(self,name=None): import os import CCP4Utils if name is not None: for ext in ['.svg','.png','.ico']: pixFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'qticons',name+ext) if os.path.exists(pixFile): return pixFile #print "Icon not found in qticons",name if name is not None: for ext in ['.svg','.png','.ico']: pixFileList = CCP4Utils.globSearchPath(self.searchPath(),os.path.join(name+ext)) if len(pixFileList)>0: return pixFileList[0] print "Icon not found",name return os.path.join(CCP4Utils.getCCP4I2Dir(),'qticons','ccp4.svg') def hasReportDefinition(self,name='',jobStatus=None): if jobStatus == 'Running remotely': jobStatus = 'Running' if self.getReportClass(name=name,jobStatus=jobStatus) is not None: return True elif self.searchXrtFile(name=name,jobStatus=jobStatus) is not None: return True defFile = WORKFLOWMANAGER().getDefFile(name) return (defFile is not None) def lookupDefFile(self,name='',version=None): import os,glob if version=='': version = None data = self.getTaskData(name,version) if data.get('pyFile',None) is not None: defFile = os.path.join(os.path.split(data['pyFile'])[0],name+'.def.xml') #print 'lookupDefFile getTaskData',name,version,data,defFile if os.path.exists(defFile): return defFile data = self.getScriptData(name,version) #print 'lookupDefFile getScriptData',name,version,data if data.get('pyFile',None) is not None: defFile = os.path.join(os.path.split(data['pyFile'])[0],name+'.def.xml') if os.path.exists(defFile): return defFile import CCP4Utils # No py files so try just simple directory names = task name # check it is not one of the mismatched tasname/dir name = TASKNAME2DIR.get(name,name) defFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'tasks',name,name+'.def.xml') if os.path.exists(defFile): return defFile for d in ['wrappers','pipelines','wrappers2']: defFile = os.path.join(CCP4Utils.getCCP4I2Dir(),d,name,'script',name+'.def.xml') if os.path.exists(defFile): return defFile fileList = glob.glob(os.path.join(CCP4Utils.getCCP4I2Dir(),'pipelines','*','wrappers',name,'script',name+'.def.xml')) if len(fileList)>0: return fileList[0] # Try if it is a workflow - returns None if not recognised defFile = WORKFLOWMANAGER().getDefFile(name) if defFile is not None: return defFile # Try derived name - used by crank2 to load the crank2 parent for subjobs crank2_phas, crank2_comb etc if '_' in name: der_name = name.split('_')[0] defFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'pipelines',der_name,'script',der_name+'.def.xml') if os.path.exists(defFile): return defFile defFile = os.path.join(CCP4Utils.getCCP4I2Dir(),'tasks',der_name,der_name+'.def.xml') if os.path.exists(defFile): return defFile defFile = CUSTOMTASKMANAGER().getDefFile(name) return defFile ''' def whatNext(self,taskName,jobId=None): else: return () def whatNext(self,taskName,jobId=None): import os,re import CCP4Utils fileName = os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','whatnext',taskName+'.html') if not os.path.exists(fileName): return [] text = CCP4Utils.readFile(fileName) x = re.compile(r"(.*)(.*)",re.DOTALL) s = x.match(text) #print 'TASKMANAGER.whatNext groups',s.groups() if s is None: return [] lineList = s.groups()[1].split() taskList = [] for line in lineList: nameList = line.split(',') for name in nameList: taskList.append((name,self.getTitle(name))) print 'TASKMANAGER.whatNext taskList',taskList return taskList ''' def exportJobFiles(self,taskName=None,jobId=None,mode='menu'): data = self.getScriptData(taskName) if len(data)==0: print 'CTaskManager.exportMtz no module for',taskName return [] import os try: pyFile = data.get('pyFile') moduleName = os.path.splitext(os.path.split(pyFile)[-1])[0] mod = __import__(moduleName) except: print 'CTaskManager.exportMtz no module for',taskName return [] if mode == 'menu': if not 'exportJobFileMenu' in dir(mod): return [] else: try: return mod.exportJobFileMenu(jobId=jobId) except: print 'CTaskManager.exportJobFiles menu failed for',taskName return [] else: try: return mod.exportJobFile(jobId=jobId,mode=mode) except: print 'CTaskManager.exportJobFiles failed for',taskName return [] def exportMtzColumnLabels(self,taskName=None,jobId=None,paramNameList=[],sourceInfoList=[]): data = self.getTaskData(taskName) if len(data)==0: print 'CTaskManager.exportMtzColumnLabels no module for',taskName return [] import os try: pyFile = data.get('pyFile') moduleName = os.path.splitext(os.path.split(pyFile)[-1])[0] mod = __import__(moduleName) except: print 'CTaskManager.exportMtzColumnLabels no module for',taskName return [] try: return mod.exportMtzColumnLabels(jobId=jobId,paramNameList=paramNameList,sourceInfoList=sourceInfoList) except: print 'CTaskManager.exportMtzColumnLabels failed for',taskName return [] def whatNext(self,taskName,jobId=None): #print 'CCP4TaskManager.whatNext',taskName,jobId,type(jobId),self.taskLookup.has_key(taskName) # Get the jobs parent - the parent pipeline should provide the info for what to do next childJobs = [jobId] pipelineTaskName = taskName childJobNumber = None if jobId is None: parentJobId = None else: parentJobId = PROJECTSMANAGER().db().getJobInfo(jobId=jobId,mode='parentJobId') while parentJobId is not None: childJobs.insert(0,parentJobId) parentJobId = PROJECTSMANAGER().db().getJobInfo(jobId=parentJobId,mode='parentJobId') if len(childJobs)>1: pipelineTaskName = PROJECTSMANAGER().db().getJobInfo(jobId=childJobs[0],mode='taskName') childJobNumber = PROJECTSMANAGER().db().getJobInfo(jobId=childJobs[-1],mode='jobNumber') #print 'CTaskManager.whatNext',taskName,jobId,childJobs,pipelineTaskName projectName = PROJECTSMANAGER().db().getJobInfo(jobId=jobId,mode='projectname') if parentJobId is not None: jobId = parentJobId whatList = [] data = self.getTaskData(pipelineTaskName) if len(data)==0: self.getScriptData(pipelineTaskName) if len(data)>0: import os try: pyFile = data.get('pyFile') moduleName = os.path.splitext(os.path.split(pyFile)[-1])[0] mod = __import__(moduleName) except: pass else: try: whatList = mod.whatNext(jobId=jobId,childTaskName=taskName,childJobNumber=childJobNumber,projectName=projectName) except: #print 'CDbApi.whatNext failed in running whatNext from module' pass if len(whatList)==0: cls = self.getTaskWidgetClass(pipelineTaskName) if cls is not None: whatList = getattr(cls,'WHATNEXT',[]) if len(whatList)==0: cls = self.getPluginScriptClass(pipelineTaskName) if cls is not None: whatList = getattr(cls,'WHATNEXT',[]) rv = [] for name in whatList: if isinstance(name,list): rv.append((name[0],self.getShortTitle(name[0]),name[1])) else: rv.append((name,self.getShortTitle(name),None)) return rv def getWhatNextPage(self,taskName=None,jobId=None): import CCP4Utils,os if taskName is None and jobId is not None: taskName = PROJECTSMANAGER().db().getJobInfo(jobId=jobId,mode='taskname') if taskName is None: return None #page = os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','whatnext',taskName+'.html') page = os.path.join(CCP4Utils.getCCP4I2Dir(),'tasks',taskName,taskName+'.whatnext.html') #print 'getWhatNextPage',page if os.path.exists(page): #return 'whatnext/'+taskName+'.html' return page else: return None def viewSelection(self,taskName,jobId=None): #Does task need to define what files are displayed in graphics pass def getI1TaskTitle(self,taskName): if self.i1TaskLookup is None: self.loadI1TaskTitles() return self.i1TaskLookup.get(taskName,taskName) def loadI1TaskTitles(self,fileName=None): import os import CCP4I1Projects,CCP4Utils if fileName is None: fileName = os.path.join(CCP4Utils.getCCP4Dir(),'share','ccp4i','etc','modules.def') metaData,params,err = CCP4I1Projects.readI1DefFile(fileName) if err.maxSeverity()>SEVERITY_WARNING or not params.has_key('TASK_NAME'): return CErrorReport(self.__class__,103,details=fileName,stack=False) self.i1TaskLookup = {} for nT in range(len(params['TASK_NAME'][1])): self.i1TaskLookup[params['TASK_NAME'][1][nT]] = params['TASK_TITLE'][1][nT] return CErrorReport() class CMakeDocsIndex(): ERROR_CODES = { 101: { 'description' : 'Error saving to file - do you have write permission?'} , 102: { 'description' : 'Error searching for task doc files' } , 103: { 'description' : 'Error creating html' } } def __init__(self): pass def run(self): import glob,os import CCP4Utils,CCP4ReportParser from lxml import etree # List of documented tasks try: dList = glob.glob(os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','tasks','*')) docList = [] for f in dList: docList.append(os.path.split(f)[1]) except: return CErrorReport(self.__class__,102) try: # html document as etree docTree = CCP4ReportParser.htmlDoc(htmlBase='.',title='CCP4i2 Tasks',cssFile='task.css') body = docTree.getroot().xpath('./body')[0] #print 'makeDocsIndex',docList t = etree.Element('div') t.text = 'CCP4i2 Task Documentation' t.set('class','title') body.append(t) for module,title,taskList in TASKMANAGER().taskTree(): modEle = etree.Element('div') modEle.set('class','module') imgEle = etree.Element('img') pixFile = TASKMANAGER().searchIconFile(module) if pixFile is not None: imgEle.set('src','../../qticons/'+os.path.split(pixFile)[1]) imgEle.set('alt',module) modEle.append(imgEle) e = etree.Element('p') e.text = title modEle.append(e) body.append(modEle) listEle = etree.Element('div') listEle.set('class','taskList') body.append(listEle) for taskName in taskList: rank = TASKMANAGER().getTaskAttribute(taskName,'RANK') pixFile = TASKMANAGER().searchIconFile(taskName) imgEle = etree.Element('img') if pixFile is not None: imgEle.set('src','../../qticons/'+os.path.split(pixFile)[1]) imgEle.set('alt',taskName) imgEle.set('class','taskIcon') taskEle = etree.Element('p') taskEle.text = TASKMANAGER().getTitle(taskName) if taskName in docList: hrefEle = etree.Element('a') hrefEle.set('href','./'+taskName+'/index.html') hrefEle.append(taskEle) else: hrefEle = None descEle = etree.Element('p') descEle.set('class','taskDesc') descEle.text = TASKMANAGER().getTaskAttribute(taskName,'DESCRIPTION') div1 = etree.Element('div') div1.set('class','taskTitle') div1.append(imgEle) if hrefEle is not None: div1.append(hrefEle) else: div1.append(taskEle) listEle.append(div1) listEle.append(descEle) except Exception as e: return CErrorReport(self.__class__,101,str(e)) try: text = etree.tostring(docTree, pretty_print=True) CCP4Utils.saveFile(os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','tasks','index.html'),text,overwrite=True) except: return CErrorReport(self.__class__,101,details=os.path.join(CCP4Utils.getCCP4I2Dir(),'docs','tasks','index.html')) return CErrorReport()