17 beautify html

tacc_stats_web/apps/tacc_stats/forms.py

@@ -0,0 +1,18 @@
+from django import forms
+from django.forms import ModelForm
+from models import Job
+
+class SearchForm(ModelForm):
+    class Meta:
+        model = Job
+        fields = ('owner', 'begin', 'end', 'hosts', 'acct_id')
+#        widgets = {
+#            'hosts': forms.TextInput(attrs={'size':'40'}),
+#        }
+
+    def __init__(self, *args, **kwargs):
+        super(SearchForm, self).__init__(*args, **kwargs)
+
+        for key in self.fields:
+            self.fields[key].required = False
+

tacc_stats_web/apps/tacc_stats/models.py

@@ -1,6 +1,7 @@
 """The database models of tacc stats"""
 
 from django.db import models
+import time
 
 class System(models.Model):
     """Details about the cluster"""
@@ -20,6 +21,9 @@ class User(models.Model):
     user_name = models.CharField(max_length=128)
     systems = models.ManyToManyField(System)
 
+    def __str__(self):
+        return self.user_name
+
     def __unicode__(self):
         return "User(%s)" % self.user_name
 
@@ -237,15 +241,20 @@ def nr_hosts(self):
         return len(self.hosts.all())
 
     def color(self):
-        ret_val = "LightBlue"
+        ret_val = "background-color: rgba(0%, 0%, 100%, .3);"
         if self.llite_open_work > 3000:
-            ret_val = "red"
+            ret_val = "background-color: rgba(100%, 0%, 0%, .3);"
         elif self.mem_MemUsed > 30*2**30:
-            ret_val = "orange"
+            ret_val = "background-color: rgba(80%, 30%, 0%, .3)"
         elif self.runtime > 3000:
-            ret_val = "LightCoral"
+            ret_val = "background-color: rgba(0%, 100%, 0%, .3)"
         return ret_val
 
+    def timespent(self):
+        return time.strftime('%H:%M:%S', time.gmtime(self.runtime))
+
+    def start_time(self):
+        return time.ctime(self.begin)
 
 class Monitor(models.Model):
     kind = models.CharField(max_length=32)

tacc_stats_web/apps/tacc_stats/urls.py

@@ -1,16 +1,12 @@
 from django.conf.urls.defaults import patterns, url
 from django.views.generic import DetailView, ListView
 from tacc_stats.models import Job
-from tacc_stats.views import index, job_memused_hist, job_timespent_hist, job_mem_heatmap, job_files_open_heatmap, create_heatmap
+from tacc_stats.views import index, job_memused_hist, job_timespent_hist, create_heatmap, search, JobListView
 
 urlpatterns = patterns('',
     url(r'^$', index),
     url(r'^joblist$',
-        ListView.as_view(
-            queryset=Job.objects.order_by('-acct_id')[:200],
-        #    context_object_name='latest_job_list',
-        #    template_name='tacc_stats/index.html',
-        )),
+        JobListView.as_view()),
     url(r'^job/(?P<pk>\d+)/$',
         DetailView.as_view(
             model=Job,
@@ -20,5 +16,6 @@
     url(r'^job_timespent_hist$', job_timespent_hist ),
     url(r'^job_mem_heatmap/(\d+)/$', create_heatmap, {'trait' : 'memory'}),
     url(r'^job_files_opened_heatmap/(\d+)/$', create_heatmap, {'trait' : 'files'}),
-    url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'})
+    url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'}),
+    url(r'^search/$', search ),
 )

tacc_stats_web/apps/tacc_stats/views.py

@@ -3,19 +3,26 @@
 
 from django.http import HttpResponse
 from django.shortcuts import render_to_response
+from django.views.decorators.csrf import csrf_protect
+from django.shortcuts import render
+from django.views.generic import DetailView, ListView
 
 import matplotlib
 matplotlib.use('Agg')
 from matplotlib.backends.backend_agg import FigureCanvasAgg
 from pylab import figure, axes, pie, title, hist, xlabel, ylabel
 from matplotlib import pyplot as PLT
+from matplotlib.colors import LogNorm
 
 import shelve
 import job
 import numpy as NP
+import math
 
 from tacc_stats.models import Job
 
+from forms import SearchForm
+
 SHELVE_DIR = '/home/dmalone/sample-jobs/jobs'
 
 def index(request):
@@ -89,8 +96,8 @@ def _files_open_intensity(job, host):
     for filesystem in job.hosts[host].stats['llite']:
         files_opened = files_opened + job.hosts[host].stats['llite'][filesystem][: , files_opened_index]
 
-    difference = NP.diff(files_opened)
-    intensity = NP.append(0, difference) / difference.max()
+    intensity = NP.diff(files_opened)
+
     return intensity
 
 def _flops_intensity(job, host):
@@ -103,6 +110,7 @@ def _flops_intensity(job, host):
     job -- the job being accessed
     host -- the host being charted
     """
+    RANGER_MAX_FLOPS = 88326000000000
     flops_used = [0] * job.times.size
 
     cpu_data = job.hosts[host].interpret_amd64_pmc_cpu()
@@ -111,8 +119,8 @@ def _flops_intensity(job, host):
         if (key[:4] == 'core'):
             flops_used = flops_used + val['SSEFLOPS']
 
-    difference = NP.diff(flops_used)
-    intensity = NP.append(0, difference) / difference.max()
+    intensity = NP.log(NP.diff(flops_used)) / math.log(RANGER_MAX_FLOPS)
+
     return intensity
 
 def create_subheatmap(intensity, job, host, n, num_hosts):
@@ -135,7 +143,7 @@ def create_subheatmap(intensity, job, host, n, num_hosts):
     intensity = NP.array([intensity]*2, dtype=NP.float64)
 
     PLT.subplot(num_hosts, 1, n)
-    PLT.pcolor(x, NP.array([0, 1]), intensity, cmap=matplotlib.cm.Reds, vmin = 0, vmax = 1)
+    PLT.pcolor(x, NP.array([0, 1]), intensity, cmap=matplotlib.cm.Reds, vmin = 0, vmax = 1, edgecolors='none')
 
     if (n != num_hosts):
         PLT.xticks([])
@@ -196,52 +204,47 @@ def create_heatmap(request, job_id, trait):
     f.set_size_inches(10,num_hosts*.3+1.5)
     return figure_to_response(f)
 
-def job_mem_heatmap(request, job_id):
-    """
-    Creates a heatmap with intensity correlated with the amount of memory used by the job
-    """
-    job_shelf = shelve.open(SHELVE_DIR)
-
-    job = job_shelf[job_id]
-
-    hosts = job.hosts.keys()
-
-    n = 1
-    num_hosts = len(job.hosts)
-    PLT.subplots_adjust(hspace = 0)
-    PLT.suptitle('Memory Used By Host', fontsize = 12)
-
-    for host in hosts:
-        intensity = _memory_intensity(job, host)
-        create_subheatmap(intensity, job, host, n, num_hosts)
-        n += 1
-
-    f = PLT.gcf()
-
-    f.set_size_inches(10,num_hosts*.3+1.5)
-    return figure_to_response(f)
-
-def job_files_open_heatmap(request, job_id):
-    """
-    Creates a heatmap with intensity correlated with the amount of files a job opens
+@csrf_protect
+def search(request):
+    """ 
+    Creates a search form that can be used to navigate through the list 
+    of jobs.
     """
-    job_shelf = shelve.open(SHELVE_DIR)
-
-    job = job_shelf[job_id]
+    if request.method == 'POST':
+        print request.POST
 
-    hosts = job.hosts.keys()
-
-    n = 1
-    num_hosts = len(job.hosts)
-    PLT.subplots_adjust(hspace = 0)
-    PLT.suptitle('Files Used By Host', fontsize = 12)
+        form = SearchForm(request.POST)
+        query = request.POST
 
-    for host in hosts:
-        intensity = _flops_intensity(job, host)
-        create_subheatmap(intensity, job, host, n, num_hosts)
-        n += 1
+        job_list = Job.objects.all()
 
-    f = PLT.gcf()
+        if form["acct_id"].value():
+            job_list = job_list.filter(acct_id = form["acct_id"].value())
+        if form["owner"].value():
+            job_list = job_list.filter(owner = form["owner"].value())
+        if form["begin"].value():
+            job_list = job_list.filter(begin__gte = form["begin"].value())
+        if form["end"].value():
+            job_list = job_list.filter(end__lte = form["end"].value())
 
-    f.set_size_inches(10,num_hosts*.3+1.5)
-    return figure_to_response(f)
+    else:
+        form = SearchForm()
+        job_list = Job.objects.order_by('-begin')[:200]
+
+    return render(request, 'tacc_stats/search.html', {'form' : form, 'job_list' : job_list })
+
+class JobListView(ListView):
+
+    def get_queryset(self):
+        if self.request.method == 'POST':
+            query = self.request.POST
+            return Job.objects.order_by('-begin')[:200]
+            #return Job.objects.filter(
+            #        owner = query.__getitem__("owner"),
+            #        begin = query.__getitem__("begin"),
+            #        end = query.__getitem__("end"),
+            #        hosts = query.__getitem__("hosts"),
+            #        acct_id = query.__getitem__("acct_id")
+            #        )
+        else:
+            return Job.objects.order_by('-begin')[:200]

tacc_stats_web/templates/tacc_stats/index.html

@@ -4,7 +4,8 @@
 </head>
 <body>
 <h1>TACC Stats Web Home</h1>
-<a href="joblist">joblist</a>
+<a href="search">jobsearch</a>
+
 <h2> Charts at a glance </h2>
 
 <img src="job_timespent_hist" alt="Histogram of jobs" />
@@ -16,7 +17,7 @@ <h1>Recent problem jobs</h1>
         <tr><th> ID </th><th> owner </th><th> time spent </th><th> memory used
         </th><th> work open </th><th> irq </th></tr>
     {% for job in job_list %}
-        <tr bgcolor="{{job.color}}">
+        <tr style="{{job.color}}">
             <td><a href="job/{{ job.id }}/">{{ job.acct_id }}</a>
             <td>{{ job.owner }}</td>
             <td align=right>{{ job.runtime }}</td>

tacc_stats_web/templates/tacc_stats/job_detail.html

@@ -10,6 +10,8 @@ <h1>Job {{ job.acct_id }}</h1>
 
     <img src="/tacc_stats/job_files_opened_heatmap/{{job.acct_id}}" alt="Files Opened Heatmap" />
 
+    <img src="/tacc_stats/job_flops_heatmap/{{job.acct_id}}" alt="Flops Performed Heatmap" />
+
     <h2>Accounting Details</h2>
     <table border="1" cellpadding="5">
       <tr><th>Field</th><th>Value</th></tr>

tacc_stats_web/templates/tacc_stats/job_list.html

@@ -5,13 +5,16 @@
 <body>
 {% if job_list %}
     <table border="1" cellpadding="5">
-        <tr><th> ID </th><th> owner </th><th> time spent </th><th> memory used
-        </th><th> work open </th><th> irq </th></tr>
+        <tr><th> name </th><th> owner </th><th> hosts </th>
+	<th> time spent </th><th> begin </th><th> memory used </th>
+	<th> work open </th><th> irq </th></tr>
     {% for job in job_list %}
-        <tr bgcolor="{{job.color}}">
+        <tr style="{{job.color}}">
             <td><a href="job/{{ job.id }}/">{{ job.acct_id }}</a>
             <td>{{ job.owner }}</td>
-            <td align=right>{{ job.runtime }}</td>
+            <td align=right>{{ job.nr_hosts }}</td>
+            <td align=right>{{ job.timespent }}</td>
+            <td align=right>{{ job.start_time }}</td>
             <td align=right>{{ job.mem_MemUsed }}</td>
             <td align=right>{{ job.llite_open_work }}</td>
             <td align=right>{{ job.cpu_irq }}</td>

tacc_stats_web/templates/tacc_stats/search.html

@@ -0,0 +1,28 @@
+<form action="/tacc_stats/search/" method="POST">{% csrf_token %}
+    <table>
+        {{ form.as_table }}
+    </table>
+    <p><input type="submit" value="Submit"></p>
+<form>
+
+{% if job_list %}
+    <table border="1" cellpadding="5">
+        <tr><th> name </th><th> owner </th><th> hosts </th>
+        <th> time spent </th><th> begin </th><th> memory used </th>
+        <th> work open </th><th> irq </th></tr>
+    {% for job in job_list %}
+        <tr style="{{job.color}}">
+            <td><a href="/tacc_stats/job/{{ job.id }}/">{{ job.acct_id }}</a>
+            <td>{{ job.owner }}</td>
+            <td align=right>{{ job.nr_hosts }}</td>
+            <td align=right>{{ job.timespent }}</td>
+            <td align=right>{{ job.start_time }}</td>
+            <td align=right>{{ job.mem_MemUsed }}</td>
+            <td align=right>{{ job.llite_open_work }}</td>
+            <td align=right>{{ job.cpu_irq }}</td>
+        </tr>
+    {% endfor %}
+    </ul>
+{% else %}
+    <p>No jobs are available.</p>
+{% endif %}