from lxml import etree
from StringIO import StringIO
from CCP4ReportParser import *
import os
trace = False
class RvapiReport ( Report ) :
CSS_VERSION = '0.1.0'
WATCHED_FILE = 'i2.xml'
SEPARATEDATA = True
RUNNING = True
MAINTAINER = 'jon.agirre@york.ac.uk'
def __init__(self, xmlnode=None, jobStatus=None, jobInfo = {}, **kw ):
Report. __init__(self, xmlnode=xmlnode, jobInfo=jobInfo, jobStatus=jobStatus, cssVersion=self.CSS_VERSION, **kw)
self.jobStatus = jobStatus
jobdir = jobInfo['fileroot']
xml_filename = os.path.join ( jobdir, 'i2.xml' )
if os.path.exists ( xml_filename ) :
text = open( xml_filename ).read ( )
else :
self.createEmptyReport ( )
return
xmlroot_rvapi = etree.fromstring ( text, etree.XMLParser() )
self.inject_CSS ( )
self.parse_rvapi_XML ( xmlroot=xmlroot_rvapi )
def parse_rvapi_XML ( self, xmlroot = None ) :
if trace : print "rvapi report - (re)loading..."
results = self.addResults ( )
div_summary = results.addDiv ( style = "width: 80%; clear: both; padding-left: 6px; " )
if self.jobStatus == "Running" :
div_summary.append ( "The job is currently running. Results will be shown below as soon as they become available." )
all_trees = xmlroot.findall ( "tab/tree" )
if all_trees :
if trace : print "rvapi report - reading trees..."
#print all_trees
for tree in xmlroot.findall ( "tab/tree" ) :
for section in tree.findall ( "section" ) :
tabName = "no name"
tabName = section.find ( "name" ).text
tabOpen = False
if section.find ( "open" ).text == "true" :
tabOpen = True
if not tabName == "no name" :
tabHandle = results.addFold ( label=tabName, initiallyOpen=tabOpen )
# parse sections
self.parse_XML_section ( target_section = section, tab_handle = tabHandle )
initial_section = xmlroot.find ( "tab/section" )
if not initial_section == None :
for text in initial_section.findall( "text" ) :
results.append ( etree.tostring ( text, encoding="us-ascii", method="html" ) + "
" )
else :
if trace : print "rvapi report - skipping trees..."
for section in xmlroot.findall ( "tab/section" ) :
tabName = section.find ( "name" ).text
if tabName == None :
tabName = "no name"
tabOpen = False
if section.find ( "open" ).text == "true" :
tabOpen = True
if tabName == "Job Summary" :
self.parse_XML_section ( target_section = section, tab_handle = div_summary )
elif not tabName == "no name" :
tabHandle = results.addFold ( label=tabName, initiallyOpen=tabOpen )
self.parse_XML_section ( target_section = section, tab_handle = tabHandle )
def parse_XML_section ( self, target_section = None, tab_handle = None ) :
graph_section = target_section.find ( "graph" )
if trace : print "rvapi report - parsing a new section"
if graph_section is not None :
div_left = tab_handle.addDiv ( style="width: 200px; float:left; padding-left:12px; margin-top: 6px; clear:none;" )
graphs = graph_section.findall ( "graphdata" )
div_right = tab_handle.addDiv ( style="clear:none;")
#graph_group = div_right.addFlotGraphGroup ( style="width:400px; float:right; height:310px; border: 0px; margin:10px;" )
for graph_data in graphs :
dataset, data_ids, data_names, ids_names, title = self.parse_dataset ( target_dataset = graph_data )
if trace : print "rvapi report - dataset"
if trace and dataset.size > 0 : print dataset
if trace : print data_ids
if trace : print data_names
if trace : print ids_names
if trace : print title
plots = graph_section.findall( "plot" )
if plots and dataset.size > 0 :
flot_graph = tab_handle.addFlotGraph ( title = title, style="width:400px; float:right; height:310px; border: 0px; margin:10px;", outputXml=True, internalId=title.replace(" ", "") )
index_column = 1
the_data = etree.Element("CCP4Table")
data_node = etree.SubElement ( the_data, "data" )
column_names = ""
data_by_columns = [ ]
for column in dataset.T :
data_name_decoded = ids_names[index_column].replace(" ", " " )
data_list = column.tolist()
column_names += " %s" % data_name_decoded
if type(data_list) is not list :
data_list_list = [ data_list ]
data_by_columns.append (data_list_list )
index_column += 1
continue
for index,value in enumerate(data_list) :
if value == 0.0 and index > 4 :
data_list[index] = '-'
data_by_columns.append ( data_list )
index_column += 1
etree.SubElement ( the_data, "headers" ).text = column_names
data_by_rows = [ list(x) for x in zip(*data_by_columns)]
data_node.text = '\n'.join( (' '.join(str(e) for e in row) for row in data_by_rows) )
for individual_plot in plots :
if trace : print "rvapi report - adding plot"
plot_lines = individual_plot.findall ( "plotline" )
if plot_lines :
if trace : print "rvapi report - we've found plotlines"
dataset_in_use = ""
style = "xy"
for plot_line in plot_lines :
if plot_line.find ( "style" ) is not None :
dataset_in_use = plot_line.get( "dataset" )
if plot_line.find ( "style" ).text == "bars" :
style = "bars"
if len(graphs) > 1 and dataset_in_use != graph_data.get("id") :
#print "continuing...."
continue
po = etree.SubElement (the_data, 'plot' )
plot_name = individual_plot.find ( "title" ).text
xname = individual_plot.find ( "xname" ).text
yname = individual_plot.find ( "yname" ).text
xint = individual_plot.find ( "intx" ).text
yint = individual_plot.find ( "inty" ).text
xmin = individual_plot.find ( "xmin" ).text
ymin = individual_plot.find ( "ymin" ).text
xmax = individual_plot.find ( "xmax" ).text
ymax = individual_plot.find ( "ymax" ).text
cx_ticks = individual_plot.find ( "custom_xvalues" ).text
cx_labels = individual_plot.find ( "custom_xlabels" ).text
if trace : print "rvapi report - adding plot with X[", xname, "], Y[", yname, "], TITLE[", plot_name, "], STYLE[", style, "]"
etree.SubElement(po , 'title').text = plot_name
is_histogram = False
if cx_labels is not None and cx_ticks is not None :
if cx_labels != "" and cx_ticks != "" :
cx_ticks = cx_ticks.lstrip().rstrip()
cx_ticks = cx_ticks.replace(" ", "," )
etree.SubElement( po, 'customXTicks' ).text = cx_ticks
cx_labels = cx_labels.lstrip().rstrip()
cx_labels = cx_labels.replace(" ", "," )
etree.SubElement( po, 'customXLabels' ).text = cx_labels
if style == "bars" :
is_histogram = True
else :
etree.SubElement ( po, 'plottype' ).text = 'xy'
etree.SubElement( po, 'xlabel' ).text = xname
etree.SubElement( po, 'ylabel' ).text = yname
etree.SubElement( po, 'xintegral' ).text = xint
etree.SubElement( po, 'yintegral' ).text = yint
if individual_plot.find ( "xmin" ).get("on") == "true" :
if individual_plot.find ( "xmax" ).get("on") == "true" :
etree.SubElement(po, 'xrange', min = xmin, max = xmax )
if trace : print "rvapi report - using xmin and xmax"
else :
etree.SubElement( po, 'xrange', min = xmin )
if trace : print "rvapi report - using xmin"
elif individual_plot.find ( "xmax" ).get("on") == "true" :
etree.SubElement( po, 'xrange', max = xmax )
if trace : print "rvapi report - using xmax"
if individual_plot.find ( "ymin" ).get("on") == "true" :
if individual_plot.find ( "ymax" ).get("on") == "true" :
etree.SubElement( po, 'yrange', rightaxis='false', min = ymin, max = ymax )
if trace : print "rvapi report - using ymin and ymax"
else :
etree.SubElement( po, 'yrange', rightaxis='false', min = ymin )
if trace : print "rvapi report - using ymin"
elif individual_plot.find ( "ymax" ).get("on") == "true" :
etree.SubElement( po, 'yrange', rightaxis = 'false', max = ymax )
if trace : print "rvapi report - using ymax"
for plot_line in plot_lines :
if trace : print "rvapi report - plotline"
line_dataset = plot_line.get ( "dataset")
line_x = plot_line.get ( "xset" )
line_y = plot_line.get ( "yset" )
is_an_area = plot_line.find ( "fill" ).text
line_color = plot_line.find ( "color" ).text
line_style = plot_line.find ( "style" ).text
line_marker = plot_line.find ( "marker" ).text
line_fill = plot_line.find ( "fill" ).text
line_show = plot_line.find ( "show" ).text
point_size = plot_line.find ( "width" ).text
size_int = 1
ip, dp = divmod ( float(point_size), 1 )
size_int = int (ip)
if is_an_area == "true" :
if trace : print "rvapi report - hey, look, we have one of those grey areas!"
x1 = str ( dataset[:,data_ids[line_x]-1].tolist()[0] )
x2 = str ( dataset[:,data_ids[line_x]-1].tolist()[1] )
y1 = str ( dataset[:,data_ids[line_y]-1].tolist()[0] )
y2 = str ( dataset[:,data_ids[line_y]-1].tolist()[1] )
for element in dataset[:,data_ids[line_x]-1].tolist() :
if not "nan" in str(element) :
x2 = str(element)
etree.SubElement( po, 'polygon', linecolour="#cccccc", fillcolour="#aaaaaa", alpha="0.2" ).text = str( x1 + " " + ymin + " " + x1 + " " + y1 + " " + x2 + " " + y2 + " " + x2 + " " + ymin )
if trace : print "rvapi report - x1: ", x1, " x2: ", x2, " y1: ", y1, " y2: ", y2
xpos = float (x1) + ( float (x2) / 5.0 )
if float (ymax) > 0.0 :
if float (y2) > float (ymax) :
ypos = float (ymax) / 2.1
else :
ypos = float (y2) + ( float (y2) / 45.0 )
else :
ypos = float (y2) + ( float (y2) / 45.0 )
etree.SubElement( po, 'text', xpos = str(xpos), ypos = str(ypos), font="bold 1.25em Helvetica",colour="#aaaaaa" ).text = data_names[line_y]
else :
if line_style == "off" :
linestyle = '.'
else :
linestyle = '-'
if is_histogram :
po_histogram = etree.SubElement( po, 'barchart', col = str(data_ids[line_x]), tcol= str(data_ids[line_y]) )
etree.SubElement(po_histogram, 'width' ).text = '1'
etree.SubElement(po_histogram, 'colour').text = line_color
else :
line = etree.SubElement(po, 'plotline', xcol = str(data_ids[line_x]), ycol = str(data_ids[line_y]) )
etree.SubElement(line, 'linestyle' ).text = linestyle
etree.SubElement(line, 'symbolsize' ).text = "%i" % size_int
if trace : print "rvapi report - plotting with xcol: ", data_ids[line_x], " and ycol: ", data_ids[line_y], " and point size " , size_int
if line_color is not None :
etree.SubElement(line, 'colour' ).text = line_color
etree.SubElement(line, 'fillcolour' ).text = line_color
if trace : print "rvapi report - adding one line for dataset ", line_dataset, ", x: ", line_x, ", y: ", line_y
else :
if trace : print "rvapi report - this plot does not contain any lines -> skipping..."
flot_graph.addPimpleData ( the_data )
else :
div_left = tab_handle.addDiv ( style="width: 90%; padding-left:12px; float: left; clear:none;" )
tab_handle.append ( "
" )
for text in target_section.findall ( "text" ) :
div_left.append ( etree.tostring ( text, encoding="us-ascii", method="html" ) + "
" )
clear_div = tab_handle.addDiv ( style = "clear:both;" )
def parse_dataset ( self, target_dataset = "" ) :
import numpy
from StringIO import StringIO
the_title = ""
data = target_dataset.find ( "data" ).text
the_title = target_dataset.find ( "title" ).text
if len(data) > 4 :
text_field = StringIO ( data )
matrix = numpy.genfromtxt ( text_field, delimiter = " " )
else :
matrix = numpy.empty ( [0, 0] )
ids = {}
names = {}
ids_names = {}
ids_field = target_dataset.find ( "setids" ).text
ids_separator = target_dataset.find ( "setids" ).get( "separator" )
names_field = target_dataset.find ( "setnames" ).text
names_separator = target_dataset.find ( "setnames" ).get( "separator" )
ids_list = ids_field.split ( ids_separator )
names_list = names_field.split ( names_separator )
n_data = 0
for identifier in ids_list :
ids [ identifier.lstrip('\n').rstrip('\n') ] = n_data +1
names [ identifier.lstrip('\n').rstrip('\n') ] = names_list [ n_data ].lstrip('\n').rstrip('\n')
ids_names [ n_data + 1] = names_list [ n_data ].lstrip('\n').rstrip('\n')
n_data = n_data + 1
return matrix,ids,names,ids_names,the_title
def inject_CSS ( self ) :
self.append ( """""" )
def createEmptyReport ( self ) :
results = self.addResults ( )
results.append ( "The job is currently running. Updates will follow shortly.
" )