# This version of scarf.py uses urllib2 instead of requests and httplib2
# Therefore it also doesn't require pac_api or any libraries not included in
# ccp4-python. Reference: https://docs.python.org/2.7/howto/urllib2.html

#from pac import pac_api
import logging
import os
import shutil
import urllib2
#from requests.exceptions import SSLError
import sys
import time
import io
from os.path import expanduser
import re
#import requests
import json
import getpass
import urllib
#import httplib2
from threading import Event
from xml.etree import ElementTree
from xml.dom import minidom
from xml.parsers.expat import ExpatError

ALL_PROGRAMS = ['freerflag', 'dimple']

PACCONTEXT='platform'
PACPASSFILE='.pacpass'
ACCEPT_TYPE='text/plain,application/xml,text/xml'
ERROR_STR='errMsg'
NOTE_STR='note'
ERROR_TAG='<' + ERROR_STR + '>'
ACTION_STR='actionMsg'
ACTION_TAG='<' + ACTION_STR + '>'
CMD_STR='cmdMsg'
CMD_TAG='<' + CMD_STR + '>'
APP_NAME='APPLICATION_NAME'
DIAGNOSTIC = True

def getCerts():
  if os.environ.has_key('CA_BUNDLE'):
    return os.environ['CA_BUNDLE']
  else:
    path = os.path.join(os.path.dirname(__file__),'cacerts.txt')
    return path

def getHOME():
  try:
    if sys.platform == 'win32':
      homedir = os.path.normpath(os.environ.get('USERPROFILE'))
    else:
      homedir = os.environ.get('HOME')
  except:
    homedir = ''
  if homedir and os.path.exists(homedir):
    return homedir
  else:
    return ''

def getpacpass():
  return os.path.join(getHOME(),'.pacpass')

def check(application='CCP4'):
    """Tests if the access token is valid and the service is available. 
    Returns a tuple (status, msg), where status is True/False."""
    if not os.path.exists(getCerts()):
      return (False, "Certs file not found:"+getCerts())
    url, token = getToken()
    if not url or not token:
        return (False,"No URL or security token found.")
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': 'application/json'}
    req = urllib2.Request(url + 'webservice/pacclient/appStatus', headers=headers)
    try:
        response = urllib2.urlopen(req,cafile=getCerts())
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            status = e.code
            return (False, status + ": The URL is not accessible.")
    mess = json.load(response)
    # check if there's an error (usually whether the token is valid)
    possible_err_msg = mess["AppInfo"][0]
    if possible_err_msg.has_key("errMsg"):
        return (False, possible_err_msg["errMsg"])
    # check that CCP4 is available
    if DIAGNOSTIC: print 'checking for',application
    for item in mess["AppInfo"]:
        if DIAGNOSTIC: print 'check',item
        if item["appName"] == application and item["status"] == "Published":
            return(True, "OK")
    return (False, application+" is not available.")

def list(jobId):
    """Lists all files in the job directory"""
    url0, token = getToken()
    if not url0 or not token:
        return (False,"No URL or security token found.")
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': 'text/plain'}
    url = url0 + 'webservice/pacclient/jobfiles/'+str(jobId)
    req = urllib2.Request(url, headers=headers)
    try:
        response = urllib2.urlopen(req, cafile=getCerts())
	files = response.read().split(';')
        filepaths = []
	for ff in files:
	    if len(ff) <= 0:
	        break
	    # hostname*filepath*fileType*fileSize*controlFlag;
	    fileArray = ff.split("*")
	    if ((len(fileArray) == 5) and (fileArray[2] == 'DIR')):
                url = url0 + 'webservice/pacclient/userCmd'
                body = '<UserCmd><cmd>restrictedls %s</cmd></UserCmd>' % fileArray[1]
                headers = {'Content-Type': 'application/xml', 'Cookie': token, 'Accept': 'text/xml','Content-Length': str(len(body))}
                req = urllib2.Request(url, data=body, headers=headers)
                response = urllib2.urlopen(req, cafile=getCerts())
                print(body)
                print(response.read())
            elif (len(fileArray) == 5):
                filepaths.append(fileArray[1])
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            status = e.code
            return (False, status + ": The URL is not accessible.")
    return(True, filepaths)

def listdir(jobId,path):
    """Lists all files in a subdirectory of the job directory."""
    url0, token = getToken()
    if not url0 or not token:
        return (False,"No URL or security token found.")
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': 'text/plain'}
    url = url0 + 'webservice/pacclient/jobfiles/'+str(jobId)
    req = urllib2.Request(url, headers=headers)
    try:
        # get the job directory
        response = urllib2.urlopen(req, cafile=getCerts())
	files = response.read().split(';')
        path0 = os.path.dirname(files[0].split('*')[1])
        url = url0 + 'webservice/pacclient/userCmd'
        body = '<UserCmd><cmd>restrictedls %s</cmd></UserCmd>' % os.path.join(path0,path)
        headers = {'Content-Type': 'application/xml', 'Cookie': token, 'Accept': 'text/xml','Content-Length': str(len(body))}
        req = urllib2.Request(url, data=body, headers=headers)
        response = urllib2.urlopen(req, cafile=getCerts())
        content = response.read()
        root = ElementTree.fromstring(content)
        if root:
            files = root.find('cmdMsg').text.split()
        else:
            files = []
        return (True,files)
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            status = e.code
            return (False, status + ": The URL is not accessible.")

def listdir2(jobId,path):
    """This one is like listdir, but it emulates the ls -l command, and returns the output of that command as a string."""
    url0, token = getToken()
    if not url0 or not token:
        return (False,"No URL or security token found.")
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': 'text/plain'}
    url = url0 + 'webservice/pacclient/jobfiles/'+str(jobId)
    req = urllib2.Request(url, headers=headers)
    try:
        # get the job directory
        response = urllib2.urlopen(req, cafile=getCerts())
	files = response.read().split(';')
        path0 = os.path.dirname(files[0].split('*')[1])
        url = url0 + 'webservice/pacclient/userCmd'
        body = '<UserCmd><cmd>restrictedls -lhtr %s</cmd></UserCmd>' % os.path.join(path0,path)
        headers = {'Content-Type': 'application/xml', 'Cookie': token, 'Accept': 'text/xml','Content-Length': str(len(body))}
        req = urllib2.Request(url, data=body, headers=headers)
        response = urllib2.urlopen(req, cafile=getCerts())
        root = ElementTree.fromstring(response.read())
        files = root.find('cmdMsg').text
        return (True,files)
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            status = e.code
            return (False, status + ": The URL is not accessible.")

def download(jobId, path, destdir,dstName=None ):
    """Downloads a specified file from the job directory. The path should be relative to the job directory."""
    url0, token = getToken()
    if not url0 or not token:
        return (False,"No URL or security token found.")
    body = path
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': 'text/plain','Content-Length': str(len(body))}
    url = url0 + 'webservice/pacclient/filePost/'+str(jobId)
    print 'scarf2.download dstName',dstName
    if dstName is None:
      dstName = os.path.join(destdir, path)
    destdir2 = os.path.dirname(dstName)
    req = urllib2.Request(url, data=body, headers=headers)
    try:
        response = urllib2.urlopen(req, cafile=getCerts())
	if int(response.headers['content-length']) > 0:
            if not os.path.exists(destdir2):
                os.makedirs(destdir2)
            with open(dstName,'wb') as fd:
                '''
                for chunk in response.read(1024):
                    fd.write(chunk)
                '''
                fd.write(response.read())
                fd.close()
        else:
            nList = path.split('/')
            srcName= nList.pop()
            print 'scarf2.download failed'
            return (False,"Download failed. You have no permission or the specified file does not exist: %s" % srcName)
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            #return (False, e.reason)
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            status = e.code
            print 'scarf2.download failed'
            return (False, status + ": The URL is not accessible.")
    except IOError:
  	  return (False,"   --Permission denied to write the file: %s" % dstName)
    return (True, "")

def logon(username=None, password=None,application='CCP4'):
    """Log on to SCARF web portal, if necessary."""
    status, msg = check(application)
    if status: # logon not needed
        return True
    if username is None or password is None:
      import pwDialog
      username, password = pwDialog.getCredentials()
    url = 'https://portal.scarf.rl.ac.uk/cgi-bin/token.py'
    params = urllib.urlencode({'username': username, 'password': password })
    try:
        response = urllib2.urlopen(url, params, cafile=getCerts()).read()
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            return False
    # save the token to ~/.pacpass
    if "https://portal.scarf.rl.ac.uk" in response:
        f = open(getpacpass(),'w')
        f.write(response)
        f.close() 
        return True
    else:
        return False

def submitJob(jobDict):
    url, token = getToken()
    if (len(token) <= 0):
        return 'error', "You must log on to PAC. To log on, run pacclient logon."
    url_submit = url + 'webservice/pacclient/submitapp'
    if len(jobDict) == 0:
        return 'error', 'The file does not contain any job parameters. The job cannot be submitted.'
    boundary='bqJky99mlBWa-ZuqjC53mG6EzbmlxB'
    if 'PARAMS'  in jobDict.keys():
        job_params=jobDict['PARAMS']
    else:
        job_params={}
    if 'INPUT_FILES' in jobDict.keys():
        job_files=jobDict['INPUT_FILES']
    else:
        job_files={}
    print 'to encode_body',job_params,job_files
    body = encode_body(boundary, job_params, job_files, appName = jobDict.get('APPLICATION_NAME','CCP4'))
    headers = {'Content-Type': 'multipart/mixed; boundary='+boundary,
            'Accept': 'text/xml,application/xml;', 'Cookie': token,
            'Content-Length': str(len(body))}
    req = urllib2.Request(url_submit, data=body, headers=headers)
    try:
        response = urllib2.urlopen(req, cafile=getCerts())
        content = response.read()
        xdoc=minidom.parseString(content)
        if ERROR_TAG not in content:
            jobIdTag=xdoc.getElementsByTagName("id")
            return (True, jobIdTag[0].childNodes[0].nodeValue)
        else:
            err=xdoc.getElementsByTagName(ERROR_STR)
            return (False, err[0].childNodes[0].nodeValue)
    except urllib2.URLError as e:
        if hasattr(e, 'reason'): # failed to reach the server
            raise urllib2.URLError, e.reason
        elif hasattr(e, 'code'):
            status = e.code
            return (False, status + ": The URL is not accessible.")
    except Exception as e:
        print 'SCARF submitJob fail'
        print e
        return (False, str(e))
    
def downloadJobFiles(jobId, dstPath, dFile):
    url, token = getToken();
    if os.access(dstPath, os.W_OK) == 0:
        print("You do not have write permission on the directory: %s" % dstPath)
        return
    url_jobfiles= url + 'webservice/pacclient/jobfiles/' + jobId
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': ACCEPT_TYPE}
    req = urllib2.Request(url_jobfiles, headers=headers)
    try:
        response = urllib2.urlopen(req, cafile=getCerts())
        content = response.read()
        if ('\\' not in content) and ('/' not in content):
            print(content)
            return
        files = content.split(';')
        if len(dFile) > 0:
            # Download specified file
            downloadSpecifiedFile(dstPath, files, jobId, dFile)
        else:
            # Download all files in job directory
            for ff in files:
                if len(ff) <= 0:
                    break
                downloadUtil(dstPath, ff, jobId)
    except AttributeError:
        print("Cannot connect to the web service: %s" % url)

def downloadUtil(dstPath, dFile, jobId):
    """
    download a file from server into local host path :dstPath.
    """
    # hostname*filepath*fileType*fileSize*controlFlag;
    fileArray = dFile.split("*")
    # Don't download job directory
    if ((len(fileArray) == 5) and (fileArray[2] == 'DIR')):
        return
    # If file size is 0, remember it and check it later
    fileSize = ''
    controlFlag = 'true'
    if ((len(fileArray) == 5) and (fileArray[2] != 'DIR')):
        fileSize = fileArray[3]
        controlFlag = fileArray[4]
    if ((len(fileArray) == 5) | (len(fileArray) == 2)):
        dFile = fileArray[1]
    # If has no control permission on job file, show error message
    if controlFlag == 'false':
        print("Download failed. you have no permission: %s" % dFile)
        return
    if dFile[0] != '/':
        # For Windows path
        nList = dFile.split('\\')
    else:
        # For Linux path
        nList = dFile.split('/')
        fName= nList.pop()
    if fName[-1] == ' ':
	fName = fName[0:-1]
    if len(dstPath) > 0:
        if dstPath[-1] != getFileSeparator():
            dstName= dstPath + getFileSeparator() + fName
        else:
            dstName= dstPath + fName
    else:
        dstName= fName
    # For 0 kb file, just create it
    if fileSize == '0':
        f=open(dstName,'wb')
        f.close()
        return
    # Cannot download files larger than or equal to 2 GB
    if long(fileSize) >= 2147483648L:
        print("Unable to download file. Files that are 2 GB or larger cannot be downloaded: %s" % fName)
        return
    # For other files, download it from server
    downloadFile(dFile, dstName, jobId)

def downloadSpecifiedFile(dstPath, files, jobId, specifiedFile):
    """
    download a specified file from server into local host path: dstPath.
    """
    fileSize = ''
    controlFlag = 'true'
    specifiedFileList = ''
    downloadFlag = False
    for dFile in files:
        # hostname*filepath*fileType*fileSize*controlFlag;
        fileArray = dFile.split("*")
	# Don't download job directory
	if ((len(fileArray) == 5) and (fileArray[2] == 'DIR')):
            continue
	# If file size is 0, remember it and check it later
        if ((len(fileArray) == 5) and (fileArray[2] != 'DIR')):
            fileSize = fileArray[3]
            controlFlag = fileArray[4]
	if ((len(fileArray) == 5) | (len(fileArray) == 2)):
            dFile = fileArray[1]
	if dFile.endswith(specifiedFile):
	    downloadFlag = True
	    break
	if downloadFlag == False:
            print("Download failed. The specified file does not exist: %s" % specifiedFile)
            return
	# If has no control permission on job file, show error message
        if controlFlag == 'false':
            print("Download failed. you have no permission: %s" % dFile)
            return
	if dFile[0] != '/':
            # For Windows path
            nList = dFile.split('\\')
	else:
            # For Linux path
            nList = dFile.split('/')
	fName= nList.pop()
        if fName[-1] == ' ':
            fName = fName[0:-1]
        if len(dstPath) > 0:
            if dstPath[-1] != getFileSeparator():
                dstName= dstPath + getFileSeparator() + fName
            else:
                dstName= dstPath + fName
        else:
            dstName= fName
	# For 0 kb file, just create it
	if fileSize == '0':
            f=open(dstName,'wb')
            f.close()
            return
        # Cannot download files larger than or equal to 2 GB
        if long(fileSize) >= 2147483648L:
            print("Unable to download file. Files that are 2 GB or larger cannot be downloaded using Web Services: %s" % fName)
            return
	# For other files, download it from server
	downloadFile(dFile, dstName, jobId)

def downloadFile(srcName, dstName, jobId):
    url,token = getToken()
    url_file = url + 'webservice/pacclient/filePost/' + jobId
    body=srcName
    headers = {'Content-Type': 'text/plain', 'Cookie': token, 'Accept': ACCEPT_TYPE}
    req = urllib2.Request(url_file, data=body, headers=headers)
    try:
        response = urllib2.urlopen(req, cafile=getCerts())
        content = response.read()
        if len(content) > 0:
            f=open(dstName,'wb')
            f.write(content)
            f.close()
        else:
            nList = srcName.split('/')
            srcName= nList.pop()
            print("Download failed. you have no permission or the specified file does not exist: %s" % srcName)
    except AttributeError:
        print("Cannot connect to the web service: %s" % url)
    except IOError:
        print 'debug2'
        print("   --Permission denied to write the file: %s" % dstName)

def encode_body(boundary, params, inputFiles, appName='CCP4'):
    slash = getFileSeparator()
    boundary2='_Part_1_701508.1145579811786'
    def encode_appname():
        return ('--' + boundary,
                'Content-Disposition: form-data; name="AppName"',
                'Content-ID: <AppName>',
                '', appName)

    def encode_paramshead():
        return('--' + boundary,
                'Content-Disposition: form-data; name="data"',
		'Content-Type: multipart/mixed; boundary='+ boundary2,
		'Content-ID: <data>', '')

    def encode_param(param_name):
        return('--' + boundary2,
                'Content-Disposition: form-data; name="%s"' % param_name,
		'Content-Type: application/xml; charset=US-ASCII',
		'Content-Transfer-Encoding: 8bit',
		'', '<AppParam><id>%s</id><value>%s</value><type></type></AppParam>' %(param_name, params[param_name]))

    def encode_fileparam(param_name, param_value):
        return('--' + boundary2,
		'Content-Disposition: form-data; name="%s"' % param_name,
		'Content-Type: application/xml; charset=US-ASCII',
		'Content-Transfer-Encoding: 8bit',
		'', '<AppParam><id>%s</id><value>%s</value><type>file</type></AppParam>' %(param_name, param_value))

    def encode_file(filepath, filename):
        return('--' + boundary,
                'Content-Disposition: form-data; name="%s"; filename="%s"' %(filename, filename),
                'Content-Type: application/octet-stream',
                'Content-Transfer-Encoding: UTF-8',
                'Content-ID: <%s>' % filename,
                '', open(filepath, 'rb').read ())

    lines = []
    upType = ''
    upFlag = False
    lines.extend(encode_appname())
    lines.extend(encode_paramshead())
    for name in params:
        lines.extend (encode_param(name))
    for name in inputFiles:
        value=inputFiles[name]
        if ',' in value:
            try:
                upType = value.split(',')[1]
                if (upType == 'link') | (upType == 'copy'):
                    lines.extend (encode_fileparam(name, value))
                    if upType == 'copy':
                        print("Copying server file: %s " % value.split(',')[0])
                else:
                    upFlag = True
                    filename = value.replace('\\', '/').split('/').pop()	
                    lines.extend (encode_fileparam(name, filename))
            except IndexError:
                raise IndexError
        else:
            return
    lines.extend (('--%s--' % boundary2, ''))
    if upFlag == True:
        for name in inputFiles:
            value=inputFiles[name]
            if ',' in value:
                upType = value.split(',')[1]
                filepath = value.split(',')[0]
                if upType == 'upload':
                    filename = filepath.replace('\\', '/').split('/').pop()
                    try:
                        lines.extend(encode_file(filepath, filename))
                    except IOError:
                        #return "Submit job failed, No such file or directory: %s" % filepath
                        raise IOError, "Submit job failed, No such file or directory: %s" % filepath
    lines.extend (('--%s--' % boundary, ''))
    return '\r\n'.join (lines)

def getToken():
    token=''
    url=''
    '''
    if os.name == 'nt':
        fpath=os.environ['HOMEPATH']
        if len(fpath) > 0:
            fpath = os.environ['HOMEDRIVE'] + fpath + '\\' +PACPASSFILE
        else:
            fpath += '\\' + PACPASSFILE
    else:
        fpath = os.environ['HOME'] + '/' + PACPASSFILE
    '''
    try:
        ff= open(getpacpass(), "rb")		
    except IOError:
        return url,token
    else:
        url_token=ff.read().split('\n')
        ff.close()
        url=url_token[0]
        token=url_token[1].replace('"', '#quote#')
        if len(token) <= 0:
            return url, token
        else:
            return url, 'platform_token='+token

def getFileSeparator():
    if os.name == 'nt':
        return '\\'
    else:
        return '/'

class SCARFProgram:
    """A somewhat generic class for running a job remotely in SCARF through the web API."""
    def __init__(self,sleepperiod=None,downloaddir=None):
        self.JobDict = {}
        self.JobDict['APPLICATION_NAME'] = 'CCP4'
        self.JobDict['INPUT_FILES'] = {}
        self.JobDict['PARAMS'] = {}
        self.jobid = None
        if downloaddir is None:
          self.downloaddir = os.path.join(os.getcwd(), 'downloads')
        else:
          self.downloaddir = downloaddir
        self.status = None
        self.sleepperiod = sleepperiod
        if sleepperiod is not None:
          self.stop_event = Event()
        if not os.path.exists(self.downloaddir):
            os.mkdir(self.downloaddir)

    def submit(self):
        print 'SCARFProgram.submit',self.JobDict
        status, message = submitJob(self.JobDict)
        if status:
            self.jobid = int(message)
            self.status = 'Pending'
        else:
            self.status = 'Failed to start'
        return status, message

    def poll(self):
        """Poll for the status of a job."""
        url, token = getToken()
        if not url or not token:
            return (False,"No URL or security token found.")
        headers = {'Content-Type': 'application/json', 'Cookie': token, 'Accept': 'application/json'}
        params = urllib.urlencode({'id': self.jobid})
        req = urllib2.Request(url+'webservice/pacclient/jobs'+'?'+params, headers=headers)
        try:
            response = urllib2.urlopen(req, cafile=getCerts())
        except urllib2.URLError as e:
            if hasattr(e, 'reason'): # failed to reach the server
                raise urllib2.URLError, e.reason
            elif hasattr(e, 'code'):
                status = e.code
                return (False, status + ": The URL is not accessible.")
        rjson = json.load(response)
        status = 'Unknown'
        if rjson.has_key('Job'):
            jobs = rjson['Job']
            if isinstance(jobs, dict):
                job = jobs
                status = job['status']
        # possible status values: Pending Suspended Running Exit Done
        return (True, status)

    def after(self):
        """Any tasks to be done after the job has finished."""
        if os.path.exists(self.downloaddir):
            shutil.rmtree(self.downloaddir)

    def download(self):
        """Download the files to the download directory."""
        file = ''
        dir = self.downloaddir
        downloadJobFiles(str(self.jobid), dir, file)

    def getStatus(self):
        """Get the job status"""
        url, token = pac_api.getToken()
        headers = {'Content-Type': 'application/json', 'Cookie': token, 'Accept': 'application/json'}
        params = urllib.urlencode({'id': self.jobid})
        req = urllib2.Request(url+'webservice/pacclient/jobs'+'?'+params, headers=headers)
        try:
            response = urllib2.urlopen(req, cafile=getCerts())
        except urllib2.URLError as e:
            if hasattr(e, 'reason'): # failed to reach the server
                raise urllib2.URLError, e.reason
            elif hasattr(e, 'code'):
                status = e.code
                return (False, status + ": The URL is not accessible.")
        rjson = json.load(response)
        if rjson.has_key('Job'):
            job = rjson['Job']
            if isinstance(job, dict):
                self.status = job['status']

    def run(self):
        status, msg = check(application=self.JobDict['APPLICATION_NAME'])
        if status is False:
            logon(application=self.JobDict['APPLICATION_NAME'])
        self.submit()
        while not self.stop_event.isSet():
            self.stop_event.wait(self.sleepperiod)
            b, status = self.poll()
            if status != 'Pending':
                self.download()
            if status != 'Running' and status != 'Pending':
                self.stop_event.set()
        self.after()

class Freerflag(SCARFProgram):
    """Freerflag"""
    def __init__(self,inputFiles,params,sleepperiod):
        """inputFiles should be a dictionary including the paths of hklin and
        input feed. params should be a dictionary including the path to
        hklout."""
        SCARFProgram.__init__(self,sleepperiod)
        self.JobDict['INPUT_FILES'] = { 'HKLIN': inputFiles['hklin'] + ',upload', 'FEED_INPUT': inputFiles['feed'] + ',upload' }
        self.outputdir = os.path.dirname(params['hklout'])
        self.outputfile = os.path.basename(params['hklout'])
        self.JobDict['PARAMS'] = { 'JOB_NAME': 'freerflag', 'CCP4_PROGRAM_NAME': 'freerflag', 'HKLOUT': self.outputfile }

    def after(self):
        ouf = os.path.join(self.downloaddir,self.outputfile)
        log = os.path.join(self.downloaddir,"summary.log")
        if os.path.exists(ouf):
            logging.debug("Moving the output file %s from the downloads directory." % ouf)
            shutil.move(ouf, os.path.join(self.outputdir, self.outputfile))
        else:
            logging.warning("The output file %s not found." % ouf)
        if os.path.exists(log):
            # print the freerflag output to stdout to emulate the local freerflag    
            with open(log,'r') as f:
                ipr = False
                pattern = re.compile("^The output \(if any\) follows:")
                for line in f:
                    if ipr:
                        sys.stdout.write(line)
                    elif pattern.match(line):
                        ipr = True
        SCARFProgram.after(self)

class Dimple(SCARFProgram):
    """Dimple"""
    def __init__(self,inputFiles,outputdir,sleepperiod):
        """inputFiles should include hklin and pdbin"""
        SCARFProgram.__init__(self,sleepperiod)
        self.outputdir = outputdir
        self.JobDict['INPUT_FILES'] = { 'HKLIN': inputFiles['hklin'] + ',upload', 'PDBIN': inputFiles['pdbin'] + ',upload' }
        self.JobDict['PARAMS'] = { 'JOB_NAME': 'dimple', 'CCP4_PROGRAM_NAME': 'dimple'}
        self.outputfiles = ['dimple.log','screen.log','final.mtz','final.pdb']

    def download(self):
        """Download the files to the download directory, including selected files from the output directory."""
        SCARFProgram.download(self)
        if not os.path.exists(self.outputdir):
            os.makedirs(self.outputdir)
        status, self.outputfiles = listdir(self.jobid, 'output')
        if status:
            for f in self.outputfiles:
                srcpath = os.path.join('output', f)
                downloadpath = os.path.join(self.downloaddir, 'output', f)
                destpath = os.path.join(self.outputdir, f)
                download(self.jobid, srcpath, self.downloaddir)
                if os.path.exists(downloadpath):
                    shutil.copy(downloadpath,destpath)

    def after(self):
        SCARFProgram.after(self)