""" refmac.py: CCP4 GUI Project Copyright (C) 2010 University of York This library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3, modified in accordance with the provisions of the license to address the requirements of UK law. You should have received a copy of the modified GNU Lesser General Public License along with this library. If not, copies may be downloaded from http://www.ccp4.ac.uk/ccp4license.php This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. """ from CCP4PluginScript import CPluginScript import CCP4ErrorHandling import CCP4Utils import os,sys,shutil from lxml import etree from PyQt4 import QtCore class prosmart_refmac(CPluginScript): TASKMODULE = 'refinement' TASKTITLE = 'Refine with Refmac & optional Prosmart restraints' TASKNAME = 'prosmart_refmac' TASKVERSION= 0.0 WHATNEXT = ['prosmart_refmac','buccaneer_build_refine_mr'] ASYNCHRONOUS = True TIMEOUT_PERIOD = 240 MAXNJOBS = 4 PERFORMANCECLASS = 'CRefinementPerformance' SUBTASKS=['prosmart','refmac'] PURGESEARCHLIST = [[ 'refmac%*/hklout.mtz', 0, None ] ] ERROR_CODES = { 101 : { 'description' : 'Error copying data file from final job to pipeline directory' } } def process(self): self.pipelinexmlfile = self.makeFileName(format='PROGRAMXML') self.refmacMonitors = {} self.xmlroot = etree.Element("RefmacOptimiseWeight") # Check all input files exist nonExFiles = self.checkInputData() if len(nonExFiles)>0: self.reportStatus(CPluginScript.FAILED) return CPluginScript.FAILED # Provide default output file names if necessary self.checkOutputData() if self.container.inputData.REFERENCE_MODEL.isSet(): self.executeProsmart() self.container.inputData.REFERENCE_MODEL.annotation = 'Molecule to which model is restrained' else: if self.container.controlParameters.WEIGHT_OPT.__str__()=='MANUAL' and self.container.controlParameters.WEIGHT.isSet(): withWeight = float(self.container.controlParameters.WEIGHT) else: withWeight = -1. self.executeFirstRefmac(withWeight) #return CPluginScript.SUCCEEDED def executeProsmart(self): self.prosmart = self.makePluginObject('prosmart') self.prosmart.container.inputData = self.container.inputData self.connectSignal(self.prosmart,'finished',self.prosmartFinished) self.prosmart.waitForFinished = -1 self.prosmart.process() def prosmartFinished(self, statusDict): status = statusDict['finishStatus'] if status == CPluginScript.FAILED: self.reportStatus(status) return try: prosmartXMLPath = self.prosmart.makeFileName('PROGRAMXML') prosmartXML = CCP4Utils.openFileToEtree(prosmartXMLPath) prosmartElements = prosmartXML.xpath("//PROSMART") self.xmlroot.append(prosmartElements[0]) if self.container.controlParameters.WEIGHT_OPT.__str__()=='MANUAL' and self.container.controlParameters.WEIGHT.isSet(): withWeight = float(self.container.controlParameters.WEIGHT) else: withWeight = -1. self.executeFirstRefmac() except: self.reportStatus(CPluginScript.FAILED) def executeFirstRefmac(self, withWeight=-1): #create wrapper self.firstRefmac = self.refmacJobWithWeight(withWeight) #Run asynchronously ...this is needed so that commands downsrteam ofprocess launch (i.e. logwather) will be called #before process completion self.firstRefmac.async = True self.firstRefmac.connectSignal(self.firstRefmac,'finished',self.firstRefmacFinished) #Install xml node for an in progress refmac self.xmlLength = 0 #Start process firstRefmacXMLFilename = self.firstRefmac.makeFileName(format='PROGRAMXML') self.watchFile(firstRefmacXMLFilename, handler=self.handleXmlChanged, minDeltaSize=34, unwatchWhileHandling=True) rv = self.firstRefmac.process() def handleXmlChanged(self, xmlFilename): self.xmlroot.clear() refmacEtree = CCP4Utils.openFileToEtree(xmlFilename) refmacXML = refmacEtree.xpath('//REFMAC') if len(refmacXML) == 1: refmacXML[0].tag="RefmacInProgress" self.xmlroot.append(refmacXML[0]) self.saveXml() def saveXml(self): # Save the xml if it has grown newXml = etree.tostring(self.xmlroot,pretty_print=True) if len(newXml) > self.xmlLength: tmpFileName = self.pipelinexmlfile+'_tmp' with open(tmpFileName,'w') as aFile: aFile.write( newXml ) import shutil shutil.move(tmpFileName, self.pipelinexmlfile) self.xmlLength = len(newXml) def refmacJobWithWeight(self, withWeight=-1): result = self.makePluginObject('refmac') #input data for this refmac instance is the same as the input data for the program result.container.inputData = self.container.inputData if self.container.inputData.REFERENCE_MODEL.isSet(): result.container.inputData.EXTERNALRESTRAINTS=self.prosmart.container.outputData.RESTRAINTS #Copy over most of the control parameters for attr in self.container.controlParameters.dataOrder(): if attr != "OPTIMISE_WEIGHT": setattr(result.container.controlParameters, attr, getattr(self.container.controlParameters, attr)) #specify weight if a meaningful one hsa been offered if withWeight>=0.: result.container.controlParameters.WEIGHT_OPT='MANUAL' result.container.controlParameters.WEIGHT = withWeight return result def firstRefmacFinished(self, statusDict): from lxml import etree if statusDict['finishStatus'] == CPluginScript.UNSATISFACTORY: import os if os.path.isfile(self.firstRefmac.container.outputData.LIBOUT.__str__()): import acedrg try: rdkitMol = acedrg.molFromDict(self.firstRefmac.container.outputData.LIBOUT.__str__()) from rdkit import Chem molRemovedHs = Chem.RemoveHs(rdkitMol) svgXml = acedrg.svgFromMol(molRemovedHs) self.xmlroot.append(svgXml) except: print 'Unable to generate svg from DICT' import shutil shutil.copyfile(self.firstRefmac.container.outputData.LIBOUT.__str__(), self.container.outputData.LIBOUT.__str__()) self.container.outputData.LIBOUT.annotation = 'Refmac-generated library...use with caution' if os.path.isfile(self.firstRefmac.container.outputData.PSOUT.__str__()): shutil.copyfile(self.firstRefmac.container.outputData.PSOUT.__str__(), self.container.outputData.PSOUT.__str__()) self.container.outputData.PSOUT.annotation.set('Pictures of ligand prepared by refmac') with open(self.makeFileName('PROGRAMXML'),'w') as programXML: programXML.write(etree.tostring(self.xmlroot,pretty_print=True)) self.reportStatus(CPluginScript.UNSATISFACTORY) elif self.firstRefmac.errorReport.maxSeverity() > CCP4ErrorHandling.SEVERITY_WARNING: #This gets done in thefirstRefmac.reportStatus() - Liz #self.extendErrorReport(self.firstRefmac.errorReport) import CCP4Utils try: refmacEtree = CCP4Utils.openFileToEtree(self.firstRefmac.makeFileName('PROGRAMXML')) refmacXML = refmacEtree.xpath('//REFMAC') if len(refmacXML) == 1: self.xmlroot.append(refmacXML[0]) except: print 'Failed attempt to read XML file from first Refmac' try: newXml = etree.tostring(self.xmlroot,pretty_print=True) aFile = open(self.pipelinexmlfile,'w') aFile.write(newXml) aFile.close() except: print 'Failed attempt to write pipeline XML file' self.reportStatus(CPluginScript.FAILED) return self.handleXmlChanged(self.firstRefmac.makeFileName(format='PROGRAMXML')) import os self.fileSystemWatcher = None if statusDict['finishStatus'] == CPluginScript.FAILED: #This gets done in the firstRefmac.reportStatus() - Liz self.reportStatus(CPluginScript.FAILED) return else: self.addCycleXML(self.firstRefmac) aFile=open( self.pipelinexmlfile,'w') aFile.write( etree.tostring(self.xmlroot,pretty_print=True) ) aFile.close() if self.container.controlParameters.OPTIMISE_WEIGHT: weightUsed = float(self.xmlroot.xpath('//weight')[-1].text) self.tryVariousRefmacWeightsAround(weightUsed) else: self.finishUp(self.firstRefmac) print 'done prosmart_refmac.firstRefmacFinished' return def finishUp(self, refmacJob): import shutil print 'into prosmart_refmac.finishUp' for attr in self.container.outputData.dataOrder(): print 'prosmart_refmac.finishUp attr',attr wrappersAttr = getattr(refmacJob.container.outputData, attr) pipelinesAttr = getattr(self.container.outputData, attr) if attr in [ "PERFORMANCEINDICATOR"]: setattr(self.container.outputData, attr, wrappersAttr) else: if os.path.exists(str(wrappersAttr.fullPath)): try: shutil.copyfile( str(wrappersAttr.fullPath), str(pipelinesAttr.fullPath) ) except: self.appendErrorReport(101,str(wrappersAttr.fullPath)+' to '+str(pipelinesAttr.fullPath)) #self.container.outputData.copyData(refmacJob.container.outputData,[attr]) if attr == "XMLOUT": pass print 'prosmart_refmac.finishUp 1' import CCP4XtalData # Apply database annotations self.container.outputData.XYZOUT.annotation.set('Model refined by Prosmart/Refmac') self.container.outputData.FPHIOUT.annotation = 'Weighted map from refinement' self.container.outputData.FPHIOUT.subType = 1 self.container.outputData.DIFFPHIOUT.annotation = 'Weighted difference map from refinement' self.container.outputData.DIFFPHIOUT.subType = 2 self.container.outputData.ABCDOUT.annotation = 'Calculated phases from refinement' self.container.outputData.ABCDOUT.contentFlag = CCP4XtalData.CPhsDataFile.CONTENT_FLAG_HL self.container.outputData.TLSOUT.annotation = 'TLS parameters from refinement' self.container.outputData.COOTSCRIPTOUT.annotation = 'Coot script written from refinement' self.container.outputData.ANOMFPHIOUT.annotation = 'Anomalous map coefficients' self.container.outputData.DIFANOMFPHIOUT.annotation = 'Difference anomalous map coefficients (LLG map)' print 'prosmart_refmac.finishUp 2' if self.container.outputData.LIBOUT.exists(): annotation = 'Refmac generated geometry' try: print 'LIBOUT monomerList',self.container.outputData.LIBOUT.fileContent.monomerList if len(self.container.outputData.LIBOUT.fileContent.monomerList)>0: annotation = 'Refmac generated geometry for:' for item in self.container.outputData.LIBOUT.fileContent.monomerList: annotation = annotation + ' ' + str(item.three_letter_code) ligxml = etree.SubElement(self.xmlroot,"LIGANDS") for item in self.container.outputData.LIBOUT.fileContent.monomerList: ligNode = etree.SubElement(ligxml,"ligand") ligNode.text = str(item.three_letter_code) self.saveXml() except: print 'Error creating LIBOUT annotation' self.container.outputData.LIBOUT.annotation = annotation try: self.mergeDictToProjectLib(fileName=self.container.outputData.LIBOUT.__str__()) except: print 'Error merging library to Project Dictionary' print 'prosmart_refmac.finishUp 3' # self.reportStatus(CPluginScript.SUCCEEDED) print 'done prosmart_refmac.finishUp' def tryVariousRefmacWeightsAround(self, weight): import math print 'Generating jobs with weights around ', weight #make an array to hold the child-jobs refmacJobs = [] for factorExponent in range(-3,4): if factorExponent != 0: factor = math.pow(2., factorExponent) refmacJobs.append(self.refmacJobWithWeight(float(factor)*float(weight))) # Set to run asynchronously and set a callback self.submitBunchOfJobs(refmacJobs) def submitBunchOfJobs(self, jobs): self.jobsToSubmit = [] self.jobsInTrain = {} self.jobsCompleted = [] for job in jobs: job.async = True job.connectSignal(job,'finished',self.handleDone) self.jobsToSubmit.append(job) print 'ready to submit from list of length ',len(self.jobsToSubmit) for job in self.jobsToSubmit: if len(self.jobsInTrain) < prosmart_refmac.MAXNJOBS: self.submitJob(job) def submitJob(self,job): rv = job.process() #The mtzdump instance must be saved to keep it in scope and everything else can be got from that. self.jobsInTrain[str(job.processId)]=job self.jobsToSubmit.remove(job) print 'submitted job 0' def handleDone(self, ret): pid = ret.get('pid',None) status,exitStatus,exitCode = self.postProcessCheck(pid) if status == CPluginScript.FAILED: self.reportStatus(status) return import sys # callback is passed the jobId (=Non # if not in ccp4i2-db context) and processId that # can serve at identifier for subProcess import sys import time import CCP4Utils from copy import deepcopy #print 'demo_multi_mtzdump.handleDone',ret rtask = self.jobsInTrain[str(pid)] self.addCycleXML(rtask) # Save the xml on every cycle aFile=open( self.pipelinexmlfile,'w') aFile.write( etree.tostring(self.xmlroot,pretty_print=True) ) aFile.close() #decrement count of running jobs self.jobsCompleted.append(rtask) del self.jobsInTrain[str(pid)] if len(self.jobsToSubmit) != 0: job = self.jobsToSubmit[0] self.submitJob(job) elif len(self.jobsInTrain)==0: best_r_free = 9999. self.best_rtask = 0 for rtask in self.jobsCompleted: if float(rtask.container.outputData.PERFORMANCEINDICATOR.RFree) < best_r_free: best_r_free = float(rtask.container.outputData.PERFORMANCEINDICATOR.RFree) self.best_rtask = rtask self.finishUp(self.best_rtask) return def addCycleXML(self, rtask): xmlcyc = etree.SubElement(self.xmlroot,"RefmacWeight") cycWeightNode = etree.SubElement(xmlcyc,"weight") rxml = CCP4Utils.openFileToEtree(rtask.makeFileName('PROGRAMXML')) try: cycWeightNode.text = rxml.xpath("//Cycle/WeightUsed")[-1].text except: pass rstats = rxml.xpath("//REFMAC") xmlcyc.append (rstats[0]) def handleTimeout(self): import sys;sys.stdout.flush() for rtask in self.jobsInTrain: print 'TERMINATING', rtask.processId,sys.stdout.flush() try: rtask.terminate() except: pass self.appendErrorReport(40,str(self.TIMEOUT_PERIOD)) self.reportStatus(CPluginScript.FAILED) # Function called from gui to support exporting MTZ files def exportJobFile(jobId=None,mode=None,fileInfo={}): import os import CCP4Modules #print 'refmac.exportJobFile',mode childJobs = CCP4Modules.PROJECTSMANAGER().db().getChildJobs(jobId=jobId,details=True) #print 'exportJobFile childJobs',childJobs if childJobs[-1][2] == 'refmac': jobDir = CCP4Modules.PROJECTSMANAGER().jobDirectory(jobId=childJobs[-1][1],create=False) if os.path.exists(os.path.join(jobDir,'hklout.mtz')): return os.path.join(jobDir,'hklout.mtz') return None # Function to return list of names of exportable MTZ(s) def exportJobFileMenu(jobId=None): # Return a list of items to appear on the 'Export' menu - each has three subitems: # [ unique identifier - will be mode argument to exportJobFile() , menu item , mime type (see CCP4CustomMimeTypes module) ] return [ [ 'complete_mtz' ,'MTZ file' , 'application/CCP4-mtz' ] ] #============================================================================================ import unittest class testRefmac(unittest.TestCase): def test1(self): # Test creation of log file using ../test_data/test1.params.xml input from CCP4Utils import getCCP4I2Dir import CCP4Utils, os workDirectory = CCP4Utils.getTestTmpDir() logFile = os.path.join(workDirectory,'prosmart_refmac_test1.log') # Delete any existing log file if os.path.exists(logFile): os.remove(logFile) self.wrapper = prosmart_refmac(name='prosmart_refmac_test1',workDirectory=workDirectory) self.wrapper.container.loadDataFromXml(os.path.join(getCCP4I2Dir(),'wrappers','prosmart_refmac','test_data','test1.params.xml')) self.wrapper.setWaitForFinished(1000000) pid = self.wrapper.process() self.wrapper.setWaitForFinished(-1) if len(self.wrapper.errorReport)>0: print self.wrapper.errorReport.report() #self.assertTrue(os.path.exists(logFile),'No log file found') def TESTSUITE(): suite = unittest.TestLoader().loadTestsFromTestCase(testRefmac) return suite def testModule(): suite = TESTSUITE() unittest.TextTestRunner(verbosity=2).run(suite)