import os,sys
try:
  from CCP4ReportParser import *
except:
  execfile(os.path.join(os.environ['CCP4I2_TOP'],'bin/ccp4i2.pythonrc'))
  from CCP4ReportParser import *

from aimless_pipe_utils import *

# - - - - - - - - - - - - - - - - -
class aimless_report(Report):
  TASKNAME='aimless'

  def __init__(self,xmlnode=None,jobInfo={},jobStatus=None,**kw):
    Report.__init__(self,xmlnode=xmlnode,jobInfo=jobInfo,jobStatus=None,**kw)

    try:
      self.fileroot = self.jobInfo['fileroot']
    except:
      self.fileroot = None

    self.datasetsProcessed = False
    
    # 'nooutput' mode would be used by another report class that wanted
    # to use some method(s) from this class for its own report
    if jobStatus is not None and jobStatus.lower() == 'nooutput':
      return
  
    fail = self.Errors(self)

    if fail == False:
      self.justAimless(self)

  # - - - - - - - - - - - - - - - - -
  def justAimless(self, parent=None):
    ''' report for single Aimless step '''
    
    parent.addText(text='AIMLESS', style='font-size: 150%;')

    displayFile(self.fileroot, parent,
                ['job_2/log.txt', './log.txt'], 'Show log file')
    summaryDiv = parent.addDiv(\
      style="width:100%;border-width: 1px; border-color: black; clear:both; margin:0px; padding:0px;")
    self.keyText(summaryDiv)
    
    mainDiv = parent.addDiv(\
      style="width:100%;border-width: 1px; border-color: black; clear:both; margin:0px; padding:10px 0px 10px 0px;")
    

    if self.numberofdatasets == 1:
      leftDiv = mainDiv.addDiv(style="width:47%;float:left;text-align:center;margin:0px; padding:0px; line-height:100%; font-size:100%;border:1px solid black;")
      rightDiv = mainDiv.addDiv(style="width:48%;float:left;text-align:center;margin:0px; padding:10px;border:1px solid black;")
    
      self.addAimlessSummary(rightDiv)
    
      leftDiv.append('Resolution estimates')      
      self.ResolutionEstimates(leftDiv, self.xmlnode.xpath("Result/Dataset"))
      self.otherStatistics(leftDiv)

    else:  #  > 1 datasets
      tableDiv = mainDiv.addDiv(style="width:100%;float:left;text-align:center;margin:0px; padding:0px; line-height:100%; font-size:100%;border:1px solid black;")
      self.addAimlessSummary(tableDiv)
      leftDiv = mainDiv.addDiv(style="width:47%;float:left;text-align:center;margin:0px; padding:0px; line-height:100%; font-size:100%;border:1px solid black;")
      rightDiv = mainDiv.addDiv(style="width:48%;float:left;text-align:center;margin:0px; padding:10px;border:1px solid black;")
      leftDiv.append('Resolution estimates')      
      self.ResolutionEstimates(leftDiv, self.xmlnode.xpath("Result/Dataset"))
      self.otherStatistics(rightDiv)

      self.addInterDatasetGraphs(mainDiv)

    nextDiv = parent.addDiv(\
      style="width:100%;border-width: 1px; border-color: black; clear:both; margin:0px; padding:10px 0px 10px 0px;")
    self.importantGraphs(nextDiv)

    gDiv = parent.addDiv(\
      style="width:100%;border-width: 1px; border-color: black; clear:both; margin:0px; padding:10px 0px 10px 0px;")

    self.moreGraphs(gDiv)
    rDiv = parent.addDiv(\
      style="width:100%;border-width: 1px; border-color: black; clear:both; margin:0px; padding:10px 0px 10px 0px;")

    self.ReflectionDataDetails(rDiv, infold=False)

  # - - - - - - - - - - - - - - - - -
  def setFileRoot(self, fileroot):
    ''' set file root from superior script '''
    self.fileroot = fileroot
    
  # - - - - - - - - - - - - - - - - -
  def keyText(self,parent=None):

    self.processDatasets()
    if self.numberofdatasets == 1:
      parent.append("Key statistics for "+self.datasetheader)

    cells = self.xmlnode.xpath("Result/Dataset/cell")

    s = ""
    jdts = 0
    innerCompleteness = []
    overallCompleteness = []
    
    for datasetresultnode in self.datasetresultnodes:  # loop datasets
      if self.numberofdatasets > 1:
        s += "Key statistics for "+datasetresultnode.get("name")
      
      s += '<div style="margin-left: 40px;">'

      if len(cells) >= jdts:
        celltext = self.formatCell(cells[jdts], astext=True)
        s += "Unit cell: "+celltext+"<br/>"

      datareso = datasetresultnode.select("ResolutionHigh/Overall")
      s += "Resolution of input data: "+ datareso + "&#197;"
      reslimitnodes = \
       datasetresultnode.xpath("ResolutionLimitEstimate")
      if reslimitnodes[0].select("@type") == "CChalf":
        #print "reslimitnodes[0]", reslimitnodes[0].select("Message")
        reslimitmessage = reslimitnodes[0].select("Message")
        reslimit = reslimitnodes[0].select("MaximumResolution")
        if reslimitmessage == " == maximum resolution":
          s += ", resolution estimate: beyond "+reslimit+"&#197;"
        elif reslimitmessage == "insufficient data":
          s += ", resolution estimate undetermined, insufficient data"
        else:
          s += ", resolution estimate "+reslimit+"&#197;"
      
      s += "<br/>"
      s += " Rmeas: overall "+datasetresultnode.select("Rmeas/Overall")
      s += ", inner bin "+datasetresultnode.select("Rmeas/Inner")
      s += "<br/>"
      s += "In outer bin: Mean(I/sdI)"+\
           datasetresultnode.select("MeanIoverSD/Outer")
      s += "    CC(1/2) "+\
           datasetresultnode.select("CChalf/Outer")
      s += "<br/>"

      # Completeness in inner shell and overall
      innerCompleteness.append(datasetresultnode.select("Completeness/Inner"))
      overallCompleteness.append(datasetresultnode.select("Completeness/Overall"))
      
      if self.hasAnomalous:
        s += "Anomalous CC(1/2) in inner bin  "+\
           datasetresultnode.select("AnomalousCChalf/Inner")
        if datasetresultnode.haspath('AnomalousLimitEstimate'):
          anomresolution = datasetresultnode.select("AnomalousLimitEstimate/MaximumResolution")
          ##print "anomresolution", anomresolution
          anomthreshold = datasetresultnode.select("AnomalousLimitEstimate/Threshold")
          if float(anomresolution) == 0.0:
            s += "<br/>No significant anomalous signal detected<br/>"
          else:
            s += "<br/>Significant anomalous signal extends to a resolution of"+\
                 anomresolution+"A (above CCanom threshold "+anomthreshold+")<br/>"
      else:
        s += "No anomalous data present<br/>"
        
      s += "</div>"
      jdts += 1

    parent.append(s)
      
    # Check overloads
    noverloads = 0
    acceptedoverloads = 0
    if self.xmlnode.haspath("ObservationFlags/ProfileFittedOverloads/NumberFlagged"):
      noverloads = self.xmlnode.select("ObservationFlags/ProfileFittedOverloads/NumberFlagged")
      acceptedoverloads = self.xmlnode.select("ObservationFlags/ProfileFittedOverloads/NumberAccepted")
    #print "Overloads", noverloads,acceptedoverloads,type(noverloads)
    minInnerCompleteness = float(min(innerCompleteness))
    minOverallCompleteness = float(min(overallCompleteness))
    #print " Min completeness", minInnerCompleteness, type(minInnerCompleteness)
    COMPLETENESS_WARNING_THRESHOLD = 0.95; # fraction of overall completeness, for warning
    completeness_warning = (minInnerCompleteness < minOverallCompleteness*COMPLETENESS_WARNING_THRESHOLD)

    if (int(noverloads) > 0) or completeness_warning:
      s1 = ""
      if self.numberofdatasets > 1:
        s1 = " for "+str(self.numberofdatasets)+" datasets"
      s2 = ""
      if int(noverloads) > 0:
        s2 = noverloads+" observations were overloaded, "

      s = "Note: "+s2+"inner shell completeness: "+\
           ",".join(innerCompleteness)+"%"+s1

      if int(acceptedoverloads) > 0:
        s += "<br/>"+acceptedoverloads+" profile-fitted overloads were accepted"
        if completeness_warning:
          s += "<br/>Warning: low inner shell completeness persists after accepting overloaded strong observations"
      else:
        if completeness_warning:
          s += "<br/>Warning: low inner shell completeness may indicate that overloaded strong observations have been lost"

      overloaddiv = parent.addDiv(style='color: red;')
      overloaddiv.append(s)

  # - - - - - - - - - - - - - - - - -
  def keyTextMerged(self,parent=None):
    # cut down version for merged files, one dataset
    self.processDatasets()
    self.numberofdatasets = 1
    parent.append("Key statistics for "+self.datasetheader)

    # usally no CC(1/2) information for merged files
    s = ""
    for datasetresultnode in self.datasetresultnodes:  # loop datasets
      s += '<div style="margin-left: 40px;">'
      datareso = datasetresultnode.select("ResolutionHigh/Overall")
      s += "Resolution of input data: "+ datareso + "&#197;"
      reslimitnodes = \
       datasetresultnode.xpath("ResolutionLimitEstimate")
      s += ", "
      for reslimitnode in reslimitnodes:
        if reslimitnode.select("@type") == "I/sd":
          reslimit = reslimitnode.select("MaximumResolution")
          direction = reslimitnode.select("Direction")
          if direction != 'Overall':
            s += " Along "
          s += direction+", resolution estimate "+reslimit+"&#197;"
          s += "<br/>"
          
      s += "In outer bin: Mean(I/sdI)"+\
           datasetresultnode.select("MeanIoverSD/Outer")
      s += "<br/>"

      if self.hasAnomalous:
        anomslope = datasetresultnode.select("AnomalousNPslope")
        s += "Anomalous Q-Q plot slope = " + anomslope
        if (float(anomslope) < 1.1):
          s += "<br/>No significant anomalous signal detected<br/>"
        else:
          s += "<br/>Possibly significant anomalous signal<br/>"
      else:
        s += "No anomalous data present<br/>"
        
      s += "</div>"
            
    parent.append(s)

  # - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  def addAimlessSummary(self, parent=None):
      aHeaderDiv = parent.addDiv(
        style="clear:both;font-weight:bold; font-size:130%;margin:0px;padding:0px;")
      aHeaderDiv.append('Data internal consistency statistics')
      self.ResultTable(parent)


  # - - - - - - - - - - - - - - - - -
  def importantGraphs(self,parent=None):
    """ Main graphs by resolution and batch """

    summaryGraphsDiv = parent.addDiv(
      style="width:100%; border-width: 1px; border-color: black; clear:both; margin:0px; padding:0px;")
    summaryGraphsDiv.append('<br/>')
    if self.numberofdatasets > 1:
      titleDiv = summaryGraphsDiv.addDiv(
        style="width:100%; border-width: 1px; border-color: black; clear:both; text-align:center; margin:0px; padding:0px;")
      titleDiv.addText(text='Note that there are separate graphs for each dataset',
                              style='font-size:150%; color: orange;')

    # Put the key graphs that report on data quality and image-to-image quality at the top
##      byResolutionDiv = summaryGraphsDiv.addDiv(style="width:370px;float:left;font-weight:bold; font-size:130%; text-align:center;margin:0px; padding:0px; ")
    byResolutionDiv = summaryGraphsDiv.addDiv(style="width:48%;float:left;text-align:center;margin:6px; padding:0px; ")
    byResolutionDiv.addText(text='Analysis as a function of resolution',style='font-size:130%;font-weight:bold;')
    byResolutionDiv.append('<br/>')
    resmessage = "Plot of CC(1/2) vs. resolution may indicate a suitable resolution cutoff, and indicate presence of an anomalous signal"  
    byResolutionDiv.addText(text=resmessage,style='font-size:100%;font-style:italic;')
    byResolutionDiv.append('<br/>')
    resmessage = "(but check anisotropy)"  
    byResolutionDiv.addText(text=resmessage,style='font-size:100%;font-style:italic;')
    byResolutionDiv.append('<br/>')
    
    self.ByResolutionGraphs(byResolutionDiv)
    
    byBatchDiv = summaryGraphsDiv.addDiv(style="width:48%;float:left;text-align:center;margin:6px; padding:0px; ")
    byBatchDiv.addText(text='Analysis as a function of batch',style='font-size:130%;font-weight:bold;')
    byBatchDiv.append('<br/>')
    batchmessage = "Analyses against Batch may show radiation damage, and which parts of the data should be removed"  
    byBatchDiv.addText(text=batchmessage,style='font-size:100%;font-style:italic;')
    byBatchDiv.append('<br/>')
    batchmessage = " (but consider completeness)"  
    byBatchDiv.addText(text=batchmessage,style='font-size:100%;font-style:italic;')
    byBatchDiv.append('<br/>')
    self.ByBatchGraphs(byBatchDiv)

    # - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  def moreGraphs(self, parent=None):
    ''' all graphs and scatterplots etc '''
    othergraphDiv = parent.addDiv(style="width:48%;float:left;text-align:center;margin:0px; padding:0px; ")
    othergraphDiv.addText(text='Other merging statistics graphs',style='font-weight:bold; font-size:130%;')
    self.Graphs(othergraphDiv, select="All")
    #self.Graphs(othergraphDiv, select="Batch")
    #self.Graphs(othergraphDiv, select="notBatch")
    
    scatterplotDiv = parent.addDiv(style="width:48%;float:left;font-weight:bold; font-size:130%;\
    text-align:center;margin:0px; padding:0px;")
    scatterplotDiv.append('Scatter plots etc')
    self.aimlessScatterPlots(scatterplotDiv)

    displayFile(self.fileroot, scatterplotDiv,
                ['job_2/ROGUES.log', './ROGUES.log', './ROGUES'],
                'Show list of outliers')

  # - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  def addInterDatasetGraphs(self,parent=None):
    tableDiv = parent.addDiv(style="border-width: 1px; border-color: black; clear:both; margin:0px; padding:0px;")
    leftDiv = tableDiv.addDiv(style="width:50%;float:left;font-weight:bold;text-align:center;margin:0px; padding:0px; ")
    self.interdatasetAnomalousTable(leftDiv)

    if self.numberofdatasets > 2:
      rightDiv = tableDiv.addDiv(style="width:50%;float:left;font-weight:bold; text-align:center;margin:0px; padding:0px; ")
      self.interdatasetDispersionTable(rightDiv)
      #  dispersive difference on right if present
      graphDiv = parent.addDiv(\
        style="border-width: 1px; border-color: black; clear:both; margin:0px; padding:0px;")
      leftDiv = graphDiv.addDiv(style="width:50%;float:left;font-weight:bold;text-align:center;margin:0px; padding:0px; ")
      self.interdatasetAnomalousGraph(leftDiv)
      rightDiv = graphDiv.addDiv(style="width:50%;float:left;font-weight:bold; text-align:center;margin:0px; padding:0px; ")
      self.interdatasetDispersionGraph(rightDiv)

    else:
      #  if no dispersive differences, put anomalous graph on right
      rightDiv = tableDiv.addDiv(style="width:50%;float:left;font-weight:bold; text-align:center;margin:0px; padding:0px; ")
      self.interdatasetAnomalousGraph(rightDiv)

  # - - - - - - - - - - - - - - - - - - - - - - - - - - - -
  def aimlessScatterPlots(self, parent=None):
    #    parent.append('Normal probability plot')
    self.NormalProbabilityPlot(parent)
    #    parent.append('Normal probability of anomalous differences')
    self.AnomalousNormalProbabilityPlot(parent)
    #    parent.append('DelAnom/RMS scatter plot')
    self.AnomalousScatterPlot(parent)
    #    parent.append('Outliers on detector (horizontal rotation axis)')
    self.RoguePlot(parent)
    
  # - - - - - - - - - - - - - - - - -
  def Details(self,parent=None):
    """ Folded details """
    fold = parent.addFold(label="Details")
    self.ReflectionDataDetails(fold)

  # - - - - - - - - - - - - - - - - -
  def isFatalError(self):
    # True if there is a FatalErrorMessage
    if self.xmlnode.haspath('FatalErrorMessage'):
      return True
    return False

  # - - - - - - - - - - - - - - - - -
  def Errors(self,parent=None):
    fail = False
    if self.xmlnode.haspath('FatalErrorMessage'):
      message = self.xmlnode.select('FatalErrorMessage')
      message = html_linebreak(message)
      parent.append(message)
      fail = True
    if self.xmlnode.haspath('FatalErrorMessage2'):
      message = self.xmlnode.select('FatalErrorMessage2')
      message = html_linebreak(message)
      parent.append(message)
      fail = True
    return fail

  # - - - - - - - - - - - - - - - - -
  def Get3numbers(self, parent=None, tag=""):
    """ get a set of 3 numbers for Overall, Inner, Outer from tag """
    list = []
    if parent.haspath(tag+"/Overall"):
      list = [parent.select(tag+"/Overall")]
      list.append(parent.select(tag+"/Inner"))
      list.append(parent.select(tag+"/Outer"))
    elif parent.haspath(tag):  # just one entry
      list = [parent.select(tag), "",""]
    return list

  # - - - - - - - - - - - - - - - - -
  def AddToTable(self, parent=None, \
                 datasetresultnodes="",\
                 table="", label="", tag="", tip=""):
    data = []
    ndts = len(datasetresultnodes)
    count = 0
    for datasetresultnode in datasetresultnodes:  # loop datasets
      num3 = self.Get3numbers(datasetresultnode, tag)
      data += num3
      count += 1
      if (count < ndts): data += " "

    table.addData(title=label, data=data, tip=tip)

  # - - - - - - - - - - - - - - - - -
  def numberOfDatasets(self):
    return int(self.xmlnode.select("ReflectionData/NumberDatasets"))

  # - - - - - - - - - - - - - - - - -
  def processDatasets(self):

    if self.datasetsProcessed: return
    self.datasetsProcessed = True
    
    numberofdatasets = \
        int(self.xmlnode.select("ReflectionData/NumberDatasets"))
    # List of Results for each dataset
    datasetresultnodes = self.xmlnode.xpath("Result/Dataset")
    self.hasAnomalous = False

    self.numberofdatasets = numberofdatasets
    self.datasetresultnodes = datasetresultnodes
    if len(datasetresultnodes) == 0:
      self.numberofdatasets = 0
      return
    
    if numberofdatasets == 1:
      datasetname = datasetresultnodes[0].get("name")
      self.datasetheader = "Dataset: "+datasetname
      if datasetresultnodes[0].haspath("AnomalousCompleteness"):
        anomcompleteness = datasetresultnodes[0].xpath0("AnomalousCompleteness/Overall")
        if float(anomcompleteness.text) > 0.0:
          self.hasAnomalous = True
    else:
      self.datasetheader = "Datasets: "
      self.datasetlabellist = []
      notfirst = False
      for datasetresultnode in datasetresultnodes:
        if datasetresultnode.haspath("AnomalousCompleteness"):
          anomcompleteness = datasetresultnode.xpath0("AnomalousCompleteness/Overall")
          if float(anomcompleteness.text) > 0.0:
            self.hasAnomalous = True
        datasetname = datasetresultnode.get("name")
        if notfirst:
          self.datasetheader += ", "
        notfirst = True
        self.datasetheader += datasetname
        self.datasetlabellist.append(datasetname)

  # - - - - - - - - - - - - - - - - -
  def ResultTable(self,parent=None):

    self.processDatasets()
    datasettext = ""
    datasetnumberlist = []
    if self.numberofdatasets == 0:
      return
    
    if self.numberofdatasets == 1:
      datasetname = self.datasetresultnodes[0].get("name")
      parent.append("Summary of merging statistics for dataset "+"<br/>"+datasetname)
    else:
      parent.append("Summary of merging statistics for multiple datasets")
      #print "self.datasetlabellist", self.datasetlabellist
      datasettext = "Datasets: "
      jset = int(0)
      notfirst = False
      for label in self.datasetlabellist:
        jset += 1
        numbers = []
        if notfirst:
          datasettext += ", "
          numbers = [""]
        notfirst = True
        datasettext += "("+str(jset)+") "+label
        numbers.extend([str(jset), str(jset), str(jset)])
        datasetnumberlist.extend(numbers)
      
      #print "datasettext", datasettext
      parent.append(datasettext)

    table = parent.addTable(select="Result", transpose=True,
                            style="line-height:100%; font-size:100%;",downloadable=True)

    if self.numberofdatasets > 1:
      table.addData(title="Dataset number", data=datasetnumberlist)

    headers = []
    for i in range(self.numberofdatasets):
      headers += ["Overall", "Inner", "Outer"]
      if (i+1 < self.numberofdatasets): headers += " "

    table.addData(title="", data=headers)

    # List of header, tag, tool-tip
    taglist = \
        [["Low resolution limit", "ResolutionLow",""],
         ["High resolution limit", "ResolutionHigh",""],
         ["Rmerge(within I+/I-)", "Rmerge",
          u'\u2211 \u2211 | Ihl - <Ih> |/ \u2211  <Ih>'],
         ["Rmerge(all I+ and I-)", "RmergeOverall",
          u'\u2211 \u2211 | Ihl - <Ih> |/ \u2211  <Ih>'],
         ["Rmeas (within I+/I-)", "Rmeas",
          u'\u2211 \u2211 \u221A(n/n-1) | Ihl - <Ih> |/ \u2211  <Ih>'],
         ["Rmeas (all I+ & I-)", "RmeasOverall",
          u'\u2211 \u2211 \u221A(n/n-1) | Ihl - <Ih> |/ \u2211  <Ih>'],
         ["Rpim (within I+/I-)", "Rpim",
          u'\u2211 \u2211 \u221A(1/n-1) | Ihl - <Ih> |/ \u2211  <Ih>'],
         ["Rpim (all I+ & I-)", "RpimOverall",
          u'\u2211 \u2211 \u221A(1/n-1) | Ihl - <Ih> |/ \u2211  <Ih>'],
         ["Rmerge in top intensity bin", "RmergeTopI",''],
         ["Number of observations", "NumberObservations",''],
         ["Number unique", "NumberReflections",''],
         ["Mean((I)/sd(I))", "MeanIoverSD",''],
         ["Half-set correlation CC(1/2)", "CChalf",''],
         ["Completeness %", "Completeness",''],
         ["Multiplicity", "Multiplicity",'']]
    if self.hasAnomalous:
      taglist += \
         [["Anomalous completeness %", "AnomalousCompleteness",''],
         ["Anomalous multiplicity", "AnomalousMultiplicity",''],
         ["DelAnom CC(1/2)", "AnomalousCChalf",''],
         ["Mid-Slope of Anom Probability", "AnomalousNPslope",'']]

    for label, tag, tip in taglist:
      self.AddToTable(parent, datasetresultnodes=self.datasetresultnodes, \
                      table=table, label=label, tag=tag, tip=tip)

    if self.xmlnode.haspath("AnomalousStatus"):
      parent.append(self.xmlnode.select("AnomalousStatus"))

  # - - - - - - - - - - - - - - - - -
  def ResultTableMerged(self,parent=None):

    self.processDatasets()
    datasettext = ""
    datasetnumberlist = []
    
    datasetname = self.datasetresultnodes[0].get("name")
    parent.append("Summary of statistics for merged dataset "+"<br/>"+datasetname)

    table = parent.addTable(select="Result", transpose=True,
                style="line-height:100%; font-size:100%;",downloadable=True)

    headers = ["Overall", "Inner", "Outer"]
    table.addData(title="", data=headers)

    # List of header, tag, tool-tip
    taglist = \
        [["Low resolution limit", "ResolutionLow",""],
         ["High resolution limit", "ResolutionHigh",""],
         ["Number unique reflections", "NumberReflections",''],
         ["Mean((I)/sd(I))", "MeanIoverSD",''],
         ["Half-set correlation CC(1/2)", "CChalf",''],
         ["Completeness %", "Completeness",'']]
    if self.hasAnomalous:
      taglist += \
         [["Anomalous completeness %", "AnomalousCompleteness",''],
         ["Mid-Slope of Anom Probability", "AnomalousNPslope",'']]

    for label, tag, tip in taglist:
      self.AddToTable(parent, datasetresultnodes=self.datasetresultnodes, \
                      table=table, label=label, tag=tag, tip=tip)

    parent.append('Note that CC(1/2) is derived from correlating I+ and I-')


  # - - - - - - - - - - - - - - - - -
  def ResolutionEstimates(self,parent=None,\
                          datasetresultnodes=""):
    numberofdatasets = len(datasetresultnodes)
    if numberofdatasets == 0:
      return
    
    # 2 entry types for each direction, CC1/2 and I/sd
    numberoftypes = 2

    #  datasets may have different numbers of directions if some are missing!
    nreslimits = len(datasetresultnodes[0].xpath("ResolutionLimitEstimate"))

    # get thresholds
    thresholdCC = 0.0
    thresholdIovsd = 0.0

    for datasetresultnode in datasetresultnodes:  # loop datasets
      reslimitnodes = datasetresultnode.xpath("ResolutionLimitEstimate")
      if reslimitnodes[0].select("@type") == "CChalf":
        thresholdCC = reslimitnodes[0].select("Threshold")
      if reslimitnodes[1].select("@type") == "I/sd":
        thresholdIovsd = reslimitnodes[1].select("Threshold")

    message = "Estimates of limits from CC(1/2) (threshold"+\
              thresholdCC+") <br/>and Mn(I/sd) (threshold"+thresholdIovsd+")"
    parent.append(message)

    table = parent.addTable(transpose=True)
    pad = ["",""]   # 2 columns of padding

    # Overall limits, for each dataset
    data = ["CC(1/2)","Mn(I/sd)"]
    table.addData(title="", data=data)

    nodata = 0
    maxres = 0

    for datasetresultnode in datasetresultnodes:  # loop datasets
      reslimitnodes = datasetresultnode.xpath("ResolutionLimitEstimate")
      
      datasetname = datasetresultnode.select("@name")
      if numberofdatasets > 1:
        table.addData(title=">>>> Dataset: "+datasetname, data=pad)

      reslimitnodes = datasetresultnode.xpath("ResolutionLimitEstimate")
      ndirections = len(reslimitnodes)/numberoftypes
    
      for j in range(ndirections):  # loop directions and types
        # for this direction, for each type and dataset
        reslimitnodes12 = [reslimitnodes[j*2], \
                           reslimitnodes[j*2+1]]    #  CC, I/sd
        direction = reslimitnodes12[0].select("Direction")
        direction2 = reslimitnodes12[1].select("Direction")

        if direction == "Overall":  # skip Overall as already done
          title = "Overall"
        else:
          title = " --- Along "+direction

        data = []
        for i in range(2):
          reslim = reslimitnodes12[i].select("MaximumResolution")
          if float(reslim) == 0.0:
            reslim = "-"

          if reslimitnodes12[i].haspath("Message"):
            message = reslimitnodes12[i].select("Message")
            if message == "insufficient data":
              nodata += 1
            elif message == " == maximum resolution":
              reslim += '*'
              maxres += 1
                
          data.append(reslim)

        table.addData(title=title, data=data)

    s = ""
    if nodata > 0:
      s += "entries with insufficient data for an estimate are marked '-'"
    if maxres > 0:
      if s != "":
        s += "<br/>"
      s += "estimates extending to the maximum resolution marked '*'"

    if s != "":
      parent.append(s)
      
  # - - - - - - - - - - - - - - - - -
  def getDataList(self, nodes, tag, npad):
    # return a list of parameters for one or more data sets
    data = []
    for node in nodes:
      data.append(node.xpath0(tag).text)

    # Pad to npad items
    if len(data) < npad:
      for i in range(npad-len(data)):
        data.append("")

    return data

  # - - - - - - - - - - - - - - - - -
  def formatCell(self, cellxml, astext=False):
    # astext = False, return list[[a,b,c,][alpha,beta,gamma]
    # astext = True,  return plain text
    a = cellxml.select('a')
    b = cellxml.select('b')
    c = cellxml.select('c')
    alpha = cellxml.select('alpha')
    beta = cellxml.select('beta')
    gamma = cellxml.select('gamma')
    if astext:
      return a+b+c+alpha+beta+gamma
    else:
      return [[a,b,c],[alpha,beta,gamma]]
    
  # - - - - - - - - - - - - - - - - -
  def otherStatisticsMultidataset(self, cells, parent=None):
    # More than one dataset
    parent.append('Unit cells for each dataset:')
    dnames = []
    for dataset in self.datasetresultnodes:
      dnames.append(dataset.get('name'))   # dataset names

    dnamelines = []
    for dname in dnames:
      dnamelines.append(dname)
      dnamelines.append("   (angles)")

    celllines = []
    for cell in cells:
      celldata = self.formatCell(cell)  # [[a,b,c],[alpha,beta,gamma]]
      celllines.append(celldata[0])
      celllines.append(celldata[1])

    if len(dnamelines) == len(celllines):
      table1 = parent.addTable(transpose=True)
      for i in range(len(dnamelines)):
        table1.addData(title=dnamelines[i], data=celllines[i])


  # - - - - - - - - - - - - - - - - -
  def otherStatistics(self,parent=None):
    # Other miscellaneous things

    if self.numberofdatasets == 0:
      return

    cells = self.xmlnode.xpath("Result/Dataset/cell")

    if self.numberofdatasets > 1:
      self.otherStatisticsMultidataset(cells, parent)

    else:
      # one dataset, usual case
      dname = (self.datasetresultnodes[0].get('name'))   # dataset name
      parent.append('Dataset:'+dname)
      table = parent.addTable(transpose='true')
      cell = self.formatCell(cells[0])
      table.addData(title='Unit cell: a,b,c', data=cell[0])
      table.addData(title='  alpha,beta,gamma', data=cell[1])

      data = self.getDataList(self.datasetresultnodes,
                              "Mosaicity", 3)
      table.addData(title='Mosaicity', data=data)
      data = self.getDataList(self.datasetresultnodes,
                              'NumberReflections/Overall', 3)
      table.addData(title='Number of reflections', data=data)
      data = self.getDataList(self.datasetresultnodes,
                              'NumberObservations/Overall', 3)
      table.addData(title='Number of observations', data=data)

      outliers = self.xmlnode.xpath('Outliers')

      data = self.getDataList(outliers, 'RejectNumberUnique', 3)
      table.addData(title='Number of rejected outliers', data=data)
      data = self.getDataList(outliers, 'RejectNumberEmax', 3)
      table.addData(title='Number of Emax rejects', data=data)


  # - - - - - - - - - - - - - - - - -
  def ReflectionDataDetails(self,parent=None, infold=True):
    if infold:
      fold = parent.addFold(label="Details of reflection data",brief='InputData')
    else:
      fold = parent
    
    fold.append("Summary of input reflection data")

    table = fold.addTable( select = "ReflectionData", transpose=False )
    for title,select in [ [ "Max resolution", "ResolutionHigh" ],
                          [ "Nreflections", "NumberReflections" ],
                          [ "NObservations", "NumberObservations" ],
                          [ "Nparts", "NumberParts" ],
                          [ "Nbatches", "NumberBatches" ],
                          [ "Ndatasets", "NumberDatasets" ] ]:
      table.addData( title=title , select = select )

    # list of run data
    datasetlist = self.xmlnode.xpath("ReflectionData/Dataset")
    table2 = fold.addTable()

    # List of data wanted
    datasetnames = []
    runnumbers = []
    batchranges = []
    excludedbatches = [] 
    for dataset in datasetlist:  # loop datasets
      runlist = dataset.xpath("Run")
      for run in runlist:  # loop runs
        datasetnames.append(dataset.select("@name"))
        runnumbers.append(run.select("number"))
        batchranges.append(run.select("BatchRange"))
        excludedbatches.append(run.select("ExcludedBatches"))
        
    table2.addData(title="DatasetName", data=datasetnames)
    table2.addData(title="RunNumber", data=runnumbers)
    table2.addData(title="Batch range", data=batchranges)
    table2.addData(title="Excluded batches", data=excludedbatches)

  # - - - - - - - - - - - - - - - - -
  def Graphs(self,parent=None, select="All"):
    'Plot All or selected graphs: selections = "All", "Batch", "notBatch"'
    # A GraphGroup is a group of graphs displayed in the same graph viewer widget
    graphgroup = parent.addFlotGraphGroup(style="width:300px;  height:270px;")
    # Add a Graph to the GraphGroup - add a table of data and plot instructions to the graph

    # Loop over all Graph tables in the program output and add to the GraphGroup
    # The plotting instructions are provided as xml text
    graphlist = self.xmlnode.xpath("CCP4Table[@groupID='Graph']")

    for thisgraph in graphlist:
      graphID = thisgraph.get("id")
      # Batch graph IDs contain either "RotationRange" or "Batch"
      isBatch = False
      if ("RotationRange" in graphID) or ("Batch" in graphID):
        isBatch = True
      usePlotly = isBatch
      plotit = False
      if select == "All": plotit = True
      if select == "Batch" and isBatch: plotit = True
      if select == "notBatch" and not isBatch: plotit = True
      
      #usePlotly = ((thisgraph.get("id") == "Graph-StatsVsBatch") or (thisgraph.get("id") == "Graph-ScalesVsRotationRange"))
      if plotit:
        #print "Graph ", graphID, isBatch, usePlotly
        graph = graphgroup.addFlotGraph( xmlnode=thisgraph, title=thisgraph.get("title") )
        graph = graph.addPimpleData(xmlnode=thisgraph, usePlotly=usePlotly)
      
  # - - - - - - - - - - - - - - - - -
  def ByResolutionGraphs(self,parent=None):
      # A GraphGroup is a group of graphs displayed in the same graph viewer widget
      #      graphgroup = parent.addGraphGroup(style="width:370px;  height:300px;")
      graphgroup = parent.addFlotGraphGroup(style="width:300px;  height:270px;")
      # Add a Graph to the GraphGroup - add a table of data and plot instructions to the graph
        
      # Loop over all Graph tables in the program output and add to the GraphGroup
      # The plotting instructions are provided as xml text
      graphlist = self.xmlnode.xpath("CCP4Table[@id='Graph-CChalf'] | CCP4Table[@id='Graph-StatsVsResolution'] | CCP4Table[@id='Graph-Anisotropy']")
      
      for thisgraph in graphlist:
        #  print "Aimless ByResolutionGraphs thisgraph", thisgraph.get("title")
        graph = graphgroup.addFlotGraph( xmlnode=thisgraph, title=thisgraph.get("title") )
        graph = graph.addPimpleData(xmlnode=thisgraph)

        # - - - - - - - - - - - - - - - - -
  def ByResolutionGraphsMerged(self,parent=None):
      # A GraphGroup is a group of graphs displayed in the same graph viewer widget
      #      graphgroup = parent.addGraphGroup(style="width:370px;  height:300px;")
      byresoDiv = parent.addDiv(
        style="text-align:center;margin:0px auto; padding:3px; border:1px solid black;")
      graphgroup = byresoDiv.addFlotGraphGroup(style="width:300px;  height:270px; border:0px")
      # Add a Graph to the GraphGroup - add a table of data and plot instructions to the graph
        
      # Loop over all Graph tables in the program output and add to the GraphGroup
      # The plotting instructions are provided as xml text
      graphlist = self.xmlnode.xpath("CCP4Table[@id='Graph-StatsVsResolution']")
      graphlist.append(self.xmlnode.xpath("CCP4Table[@id='Graph-Anisotropy']")[0])
      
      for thisgraph in graphlist:
        #print "Aimless ByResolutionGraphsMerged thisgraph", thisgraph.get("title")
        graph = graphgroup.addFlotGraph( xmlnode=thisgraph, title=thisgraph.get("title") )
        graph = graph.addPimpleData(xmlnode=thisgraph)

  # - - - - - - - - - - - - - - - - -
  def ByBatchGraphs(self,parent=None):
      # A GraphGroup is a group of graphs displayed in the same graph viewer widget
##      graphgroup = parent.addGraphGroup(style="width:370px; height:300px;")
      graphgroup = parent.addFlotGraphGroup(style="width:300px;  height:270px;")
      # Add a Graph to the GraphGroup - add a table of data and plot instructions to the graph
      
      # Loop over all Graph tables in the program output and add to the GraphGroup
      # The plotting instructions are provided as xml text
      graphlist = self.xmlnode.xpath(
        "CCP4Table[@id='Graph-StatsVsBatch'] | CCP4Table[@id='Graph-ScalesVsRotationRange']")
      
      for thisgraph in graphlist:
        # print "Aimless ByBatchGraphs thisgraph", thisgraph.get("title")
        graph = graphgroup.addFlotGraph( xmlnode=thisgraph, title=thisgraph.get("title") )
        graph = graph.addPimpleData(xmlnode=thisgraph,usePlotly=True)
      
  # - - - - - - - - - - - - - - - - -
  def formatTable(self, tablein, title, celltitle, datalabel,
                  includenumbers, parent=None):
    # format a table such as a cross-correlation table
    # tablein is list of elements
    # title is a header for the table
    # celltitle is a label for columns & rows
    # datalabel is the tag for the value
    # includenumbers True to add the numbers in brackets
    # <columnsheaders> contains <label>
    # <rows> contains <row> contains <datalabel>[, <Number>]

    columns = tablein.xpath0('columnheaders')
    columnlabels = columns.xpath('label')

    rowlabs = []
    for lab in columnlabels:
      rowlabs.append(lab.text)
    collabs = rowlabs
    
    # truncate columns if needed
    MAXNUMINLINE = 18  #  maximum number / line (all rows)
    ninline = min(len(columnlabels), MAXNUMINLINE)
    if ninline < len(columnlabels):
      collabs = collabs[0:ninline]
      collabs.append(' ...')

    rows = tablein.xpath('row')
    #  true if the table has Number elements in rows
    hasNumber = False
    #print "rows", type(rows)

    rowlines = []
    rowlabels = []
    for row in rows:
      rowdata = row.xpath(datalabel)
      rownumbers = row.xpath('Number')
      #print "rl", row.xpath0('label').text
      rowlabels.append(row.xpath0('label').text)
      line = []
      for i in range(ninline):
        d = rowdata[i]
        lab = ""
        if (d.text != " "):
          lab = d.text
          if includenumbers and (len(rownumbers) > 0):
            n = rownumbers[i]
            lab += " (" + n.text + ")"
            hasNumber = True
        line.append(lab)
      
      if ninline < len(rowdata):
        line.append(' ...')
    
      rowlines.append(line)

    header = title
    if hasNumber:
      header += "<br/>(numbers in brackets)"

    parent.append(header)

    table = parent.addTable(transpose="true")
    table.addData(title=celltitle, data=collabs)

    for rowlabel, line in zip(rowlabels, rowlines):
      #print rowlabel
      table.addData(title=rowlabel, data=line)

  # - - - - - - - - - - - - - - - - -
  def interdatasetAnomalousTable(self,parent=None):
    #  Graph and table of interdataset correlations
    intertable = self.xmlnode.xpath('crosstable[@id="DatasetAnomalousCorrelation"]')
    if intertable != None and len(intertable) != 0:
      self.formatTable(intertable[0],
           "Correlation of DelAnom between datasets", "Dataset", "CC",
                       True, parent)

  # - - - - - - - - - - - - - - - - -
  def interdatasetDispersionTable(self,parent=None):
    #  Table of interdataset correlations
    intertable = self.xmlnode.xpath('crosstable[@id="DatasetDispersiveCorrelation"]')
    if intertable != None and len(intertable) != 0:
      self.formatTable(intertable[0],
          "Correlation of dispersive difference between datasets",
                       "Dataset", "CC", True, parent)

  # - - - - - - - - - - - - - - - - -
  def interRunTable(self,parent=None):
    #  Table of run-run correlations
    intertable = self.xmlnode.xpath('crosstable[@id="RunCorrelation"]')
    if (intertable is not None) and (len(intertable) != 0):
      parent.append('Correlation between runs')
      self.formatTable(intertable[0],
          "", "Run", "CC", False, parent)

  # - - - - - - - - - - - - - - - - -
  def interdatasetAnomalousGraph(self,parent=None):
    #  Graph of interdataset correlations
    graph = self.xmlnode.xpath0("CCP4Table[@id='Graph-AnomalousDifferences']")
    if graph != None:
      agraph = parent.addFlotGraph(xmlnode=graph, title=graph.get("title"))
##                             style="width:370px;" )
      agraph.addPimpleData(xmlnode=graph)

  # - - - - - - - - - - - - - - - - -
  def interdatasetDispersionGraph(self,parent=None):
    #  Graph of interdataset correlations
    graph = self.xmlnode.xpath0("CCP4Table[@id='Graph-DispersiveDifferences']")
    if graph != None:
      agraph = parent.addFlotGraph(xmlnode=graph, title=graph.get("title"))
##                             style="width:370px;" )
      agraph.addPimpleData(xmlnode=graph)

  # - - - - - - - - - - - - - - - - -
  def NormalProbabilityPlot(self,parent=None):
    nmplot = self.xmlnode.xpath0(\
      "//CCP4Table[@title='Normal probability plot']")
    if nmplot is not None:
      graph = parent.addFlotGraph(launcher='Normal probability')
      graph.addPimpleData(xmlnode=nmplot)

  # - - - - - - - - - - - - - - - - -
  def AnomalousNormalProbabilityPlot(self,parent=None):
    nmplot = self.xmlnode.xpath0(\
      "//CCP4Table[@title='Anomalous differences']")
    if nmplot is not None:
      graph = parent.addFlotGraph(launcher='Anomalous Q-Q plot')
      graph.addPimpleData(xmlnode=nmplot)

  # - - - - - - - - - - - - - - - - -
  def AnomalousScatterPlot(self,parent=None):
    nmplot = self.xmlnode.xpath0(\
      "//CCP4Table[@title='DelAnom/RMS scatter plot']")
    if nmplot is not None:
      graph = parent.addFlotGraph(launcher='DelAnom scatterplot')
      graph.addPimpleData(xmlnode=nmplot)

  # - - - - - - - - - - - - - - - - -
  def RoguePlot(self,parent=None):
    nmplot = self.xmlnode.xpath0(\
      "//CCP4Table[@title='Outliers on detector (horizontal rotation axis)']")
    if nmplot is not None:
      graph = parent.addFlotGraph(launcher='Outlier positions')
      graph.addPimpleData(xmlnode=nmplot)

    
############################################################################
if __name__ == "__main__":

  #execfile(os.path.join(os.environ['CCP4I2_TOP'],'bin/ccp4i2.pythonrc'))
  report = aimless_report(xmlFile = sys.argv[1] )
  tree= report.as_etree()
  report.as_html_file(fileName='./test-aimless.html')
  if len(report.errorReport())>0: print 'ERRORS:',r