From af0a9b5cd7f1ebb2b01b8117ca1e3816d2a4f431 Mon Sep 17 00:00:00 2001 From: Drew Malone Date: Thu, 2 Feb 2012 11:10:05 -0600 Subject: [PATCH 1/3] Editted HTML files and changed models to make it easier to work with in HTML --- tacc_stats_web/apps/tacc_stats/models.py | 17 ++++-- tacc_stats_web/apps/tacc_stats/urls.py | 4 +- tacc_stats_web/apps/tacc_stats/views.py | 60 +++---------------- .../templates/tacc_stats/job_detail.html | 2 + .../templates/tacc_stats/job_list.html | 11 ++-- 5 files changed, 31 insertions(+), 63 deletions(-) diff --git a/tacc_stats_web/apps/tacc_stats/models.py b/tacc_stats_web/apps/tacc_stats/models.py index 8d2d44e..8bdcd43 100644 --- a/tacc_stats_web/apps/tacc_stats/models.py +++ b/tacc_stats_web/apps/tacc_stats/models.py @@ -1,6 +1,7 @@ """The database models of tacc stats""" from django.db import models +import time class System(models.Model): """Details about the cluster""" @@ -20,6 +21,9 @@ class User(models.Model): user_name = models.CharField(max_length=128) systems = models.ManyToManyField(System) + def __str__(self): + return self.user_name + def __unicode__(self): return "User(%s)" % self.user_name @@ -237,15 +241,20 @@ def nr_hosts(self): return len(self.hosts.all()) def color(self): - ret_val = "LightBlue" + ret_val = "background-color: rgba(0%, 0%, 100%, .3);" if self.llite_open_work > 3000: - ret_val = "red" + ret_val = "background-color: rgba(100%, 0%, 0%, .3);" elif self.mem_MemUsed > 30*2**30: - ret_val = "orange" + ret_val = "background-color: rgba(80%, 30%, 0%, .3)" elif self.runtime > 3000: - ret_val = "LightCoral" + ret_val = "background-color: rgba(0%, 100%, 0%, .3)" return ret_val + def timespent(self): + return time.strftime('%H:%M:%S', time.gmtime(self.runtime)) + + def start_time(self): + return time.ctime(self.begin) class Monitor(models.Model): kind = models.CharField(max_length=32) diff --git a/tacc_stats_web/apps/tacc_stats/urls.py b/tacc_stats_web/apps/tacc_stats/urls.py index d41fada..e06f042 100644 --- a/tacc_stats_web/apps/tacc_stats/urls.py +++ b/tacc_stats_web/apps/tacc_stats/urls.py @@ -1,7 +1,7 @@ from django.conf.urls.defaults import patterns, url from django.views.generic import DetailView, ListView from tacc_stats.models import Job -from tacc_stats.views import index, job_memused_hist, job_timespent_hist, job_mem_heatmap, job_files_open_heatmap, create_heatmap +from tacc_stats.views import index, job_memused_hist, job_timespent_hist, create_heatmap urlpatterns = patterns('', url(r'^$', index), @@ -20,5 +20,5 @@ url(r'^job_timespent_hist$', job_timespent_hist ), url(r'^job_mem_heatmap/(\d+)/$', create_heatmap, {'trait' : 'memory'}), url(r'^job_files_opened_heatmap/(\d+)/$', create_heatmap, {'trait' : 'files'}), - url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'}) + url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'}), ) diff --git a/tacc_stats_web/apps/tacc_stats/views.py b/tacc_stats_web/apps/tacc_stats/views.py index 6c00721..0bb184c 100644 --- a/tacc_stats_web/apps/tacc_stats/views.py +++ b/tacc_stats_web/apps/tacc_stats/views.py @@ -9,10 +9,12 @@ from matplotlib.backends.backend_agg import FigureCanvasAgg from pylab import figure, axes, pie, title, hist, xlabel, ylabel from matplotlib import pyplot as PLT +from matplotlib.colors import LogNorm import shelve import job import numpy as NP +import math from tacc_stats.models import Job @@ -103,6 +105,7 @@ def _flops_intensity(job, host): job -- the job being accessed host -- the host being charted """ + RANGER_MAX_FLOPS = 88326000000000 flops_used = [0] * job.times.size cpu_data = job.hosts[host].interpret_amd64_pmc_cpu() @@ -111,8 +114,9 @@ def _flops_intensity(job, host): if (key[:4] == 'core'): flops_used = flops_used + val['SSEFLOPS'] - difference = NP.diff(flops_used) - intensity = NP.append(0, difference) / difference.max() + difference = NP.log(NP.diff(flops_used)) / math.log(RANGER_MAX_FLOPS) + intensity = NP.append(0, difference) + print intensity return intensity def create_subheatmap(intensity, job, host, n, num_hosts): @@ -135,7 +139,7 @@ def create_subheatmap(intensity, job, host, n, num_hosts): intensity = NP.array([intensity]*2, dtype=NP.float64) PLT.subplot(num_hosts, 1, n) - PLT.pcolor(x, NP.array([0, 1]), intensity, cmap=matplotlib.cm.Reds, vmin = 0, vmax = 1) + PLT.pcolor(x, NP.array([0, 1]), intensity, cmap=matplotlib.cm.Reds, vmin = 0, vmax = 1, edgecolors='none') if (n != num_hosts): PLT.xticks([]) @@ -195,53 +199,3 @@ def create_heatmap(request, job_id, trait): f.set_size_inches(10,num_hosts*.3+1.5) return figure_to_response(f) - -def job_mem_heatmap(request, job_id): - """ - Creates a heatmap with intensity correlated with the amount of memory used by the job - """ - job_shelf = shelve.open(SHELVE_DIR) - - job = job_shelf[job_id] - - hosts = job.hosts.keys() - - n = 1 - num_hosts = len(job.hosts) - PLT.subplots_adjust(hspace = 0) - PLT.suptitle('Memory Used By Host', fontsize = 12) - - for host in hosts: - intensity = _memory_intensity(job, host) - create_subheatmap(intensity, job, host, n, num_hosts) - n += 1 - - f = PLT.gcf() - - f.set_size_inches(10,num_hosts*.3+1.5) - return figure_to_response(f) - -def job_files_open_heatmap(request, job_id): - """ - Creates a heatmap with intensity correlated with the amount of files a job opens - """ - job_shelf = shelve.open(SHELVE_DIR) - - job = job_shelf[job_id] - - hosts = job.hosts.keys() - - n = 1 - num_hosts = len(job.hosts) - PLT.subplots_adjust(hspace = 0) - PLT.suptitle('Files Used By Host', fontsize = 12) - - for host in hosts: - intensity = _flops_intensity(job, host) - create_subheatmap(intensity, job, host, n, num_hosts) - n += 1 - - f = PLT.gcf() - - f.set_size_inches(10,num_hosts*.3+1.5) - return figure_to_response(f) diff --git a/tacc_stats_web/templates/tacc_stats/job_detail.html b/tacc_stats_web/templates/tacc_stats/job_detail.html index 79416e2..78abfb6 100644 --- a/tacc_stats_web/templates/tacc_stats/job_detail.html +++ b/tacc_stats_web/templates/tacc_stats/job_detail.html @@ -10,6 +10,8 @@

Job {{ job.acct_id }}

Files Opened Heatmap + Flops Performed Heatmap +

Accounting Details

diff --git a/tacc_stats_web/templates/tacc_stats/job_list.html b/tacc_stats_web/templates/tacc_stats/job_list.html index 816e043..da6fbec 100644 --- a/tacc_stats_web/templates/tacc_stats/job_list.html +++ b/tacc_stats_web/templates/tacc_stats/job_list.html @@ -5,13 +5,16 @@ {% if job_list %}
FieldValue
- + + + {% for job in job_list %} - + - + + + From cae4c5d376ffd0fff56e3a4d9251d7617e576010 Mon Sep 17 00:00:00 2001 From: Drew Malone Date: Fri, 10 Feb 2012 12:48:16 -0600 Subject: [PATCH 2/3] Added Forms and Search Function. --- tacc_stats_web/apps/tacc_stats/urls.py | 9 +-- tacc_stats_web/apps/tacc_stats/views.py | 59 +++++++++++++++++-- .../templates/tacc_stats/index.html | 5 +- 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/tacc_stats_web/apps/tacc_stats/urls.py b/tacc_stats_web/apps/tacc_stats/urls.py index e06f042..3e7de13 100644 --- a/tacc_stats_web/apps/tacc_stats/urls.py +++ b/tacc_stats_web/apps/tacc_stats/urls.py @@ -1,16 +1,12 @@ from django.conf.urls.defaults import patterns, url from django.views.generic import DetailView, ListView from tacc_stats.models import Job -from tacc_stats.views import index, job_memused_hist, job_timespent_hist, create_heatmap +from tacc_stats.views import index, job_memused_hist, job_timespent_hist, create_heatmap, search, JobListView urlpatterns = patterns('', url(r'^$', index), url(r'^joblist$', - ListView.as_view( - queryset=Job.objects.order_by('-acct_id')[:200], - # context_object_name='latest_job_list', - # template_name='tacc_stats/index.html', - )), + JobListView.as_view()), url(r'^job/(?P\d+)/$', DetailView.as_view( model=Job, @@ -21,4 +17,5 @@ url(r'^job_mem_heatmap/(\d+)/$', create_heatmap, {'trait' : 'memory'}), url(r'^job_files_opened_heatmap/(\d+)/$', create_heatmap, {'trait' : 'files'}), url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'}), + url(r'^search/$', search ), ) diff --git a/tacc_stats_web/apps/tacc_stats/views.py b/tacc_stats_web/apps/tacc_stats/views.py index 0bb184c..9c80cbc 100644 --- a/tacc_stats_web/apps/tacc_stats/views.py +++ b/tacc_stats_web/apps/tacc_stats/views.py @@ -3,6 +3,9 @@ from django.http import HttpResponse from django.shortcuts import render_to_response +from django.views.decorators.csrf import csrf_protect +from django.shortcuts import render +from django.views.generic import DetailView, ListView import matplotlib matplotlib.use('Agg') @@ -18,6 +21,8 @@ from tacc_stats.models import Job +from forms import SearchForm + SHELVE_DIR = '/home/dmalone/sample-jobs/jobs' def index(request): @@ -91,8 +96,8 @@ def _files_open_intensity(job, host): for filesystem in job.hosts[host].stats['llite']: files_opened = files_opened + job.hosts[host].stats['llite'][filesystem][: , files_opened_index] - difference = NP.diff(files_opened) - intensity = NP.append(0, difference) / difference.max() + intensity = NP.diff(files_opened) + return intensity def _flops_intensity(job, host): @@ -114,9 +119,8 @@ def _flops_intensity(job, host): if (key[:4] == 'core'): flops_used = flops_used + val['SSEFLOPS'] - difference = NP.log(NP.diff(flops_used)) / math.log(RANGER_MAX_FLOPS) - intensity = NP.append(0, difference) - print intensity + intensity = NP.log(NP.diff(flops_used)) / math.log(RANGER_MAX_FLOPS) + return intensity def create_subheatmap(intensity, job, host, n, num_hosts): @@ -199,3 +203,48 @@ def create_heatmap(request, job_id, trait): f.set_size_inches(10,num_hosts*.3+1.5) return figure_to_response(f) + +@csrf_protect +def search(request): + """ + Creates a search form that can be used to navigate through the list + of jobs. + """ + if request.method == 'POST': + print request.POST + + form = SearchForm(request.POST) + query = request.POST + + job_list = Job.objects.all() + + if form["acct_id"].value(): + job_list = job_list.filter(acct_id = form["acct_id"].value()) + if form["owner"].value(): + job_list = job_list.filter(owner = form["owner"].value()) + if form["begin"].value(): + job_list = job_list.filter(begin__gte = form["begin"].value()) + if form["end"].value(): + job_list = job_list.filter(end__lte = form["end"].value()) + + else: + form = SearchForm() + job_list = Job.objects.order_by('-begin')[:200] + + return render(request, 'tacc_stats/search.html', {'form' : form, 'job_list' : job_list }) + +class JobListView(ListView): + + def get_queryset(self): + if self.request.method == 'POST': + query = self.request.POST + return Job.objects.order_by('-begin')[:200] + #return Job.objects.filter( + # owner = query.__getitem__("owner"), + # begin = query.__getitem__("begin"), + # end = query.__getitem__("end"), + # hosts = query.__getitem__("hosts"), + # acct_id = query.__getitem__("acct_id") + # ) + else: + return Job.objects.order_by('-begin')[:200] diff --git a/tacc_stats_web/templates/tacc_stats/index.html b/tacc_stats_web/templates/tacc_stats/index.html index abab218..56a23e6 100644 --- a/tacc_stats_web/templates/tacc_stats/index.html +++ b/tacc_stats_web/templates/tacc_stats/index.html @@ -4,7 +4,8 @@

TACC Stats Web Home

-joblist +jobsearch +

Charts at a glance

Histogram of jobs @@ -16,7 +17,7 @@

Recent problem jobs

 {% for job in job_list %} - + From 40a85e7eb6b645151714f92ed4de3b0b1da486fe Mon Sep 17 00:00:00 2001 From: Drew Malone Date: Fri, 10 Feb 2012 12:50:54 -0600 Subject: [PATCH 3/3] Added untracked files. --- tacc_stats_web/apps/tacc_stats/forms.py | 18 ++++++++++++ .../templates/tacc_stats/search.html | 28 +++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tacc_stats_web/apps/tacc_stats/forms.py create mode 100644 tacc_stats_web/templates/tacc_stats/search.html diff --git a/tacc_stats_web/apps/tacc_stats/forms.py b/tacc_stats_web/apps/tacc_stats/forms.py new file mode 100644 index 0000000..fb4af12 --- /dev/null +++ b/tacc_stats_web/apps/tacc_stats/forms.py @@ -0,0 +1,18 @@ +from django import forms +from django.forms import ModelForm +from models import Job + +class SearchForm(ModelForm): + class Meta: + model = Job + fields = ('owner', 'begin', 'end', 'hosts', 'acct_id') +# widgets = { +# 'hosts': forms.TextInput(attrs={'size':'40'}), +# } + + def __init__(self, *args, **kwargs): + super(SearchForm, self).__init__(*args, **kwargs) + + for key in self.fields: + self.fields[key].required = False + diff --git a/tacc_stats_web/templates/tacc_stats/search.html b/tacc_stats_web/templates/tacc_stats/search.html new file mode 100644 index 0000000..09cacee --- /dev/null +++ b/tacc_stats_web/templates/tacc_stats/search.html @@ -0,0 +1,28 @@ +
{% csrf_token %} +
ID owner time spent memory used - work open irq
name owner hosts time spent begin memory used work open irq
{{ job.acct_id }} {{ job.owner }}{{ job.runtime }}{{ job.nr_hosts }}{{ job.timespent }}{{ job.start_time }} {{ job.mem_MemUsed }} {{ job.llite_open_work }} {{ job.cpu_irq }}
ID owner time spent memory used work open irq
{{ job.acct_id }} {{ job.owner }} {{ job.runtime }}
+ {{ form.as_table }} +
+

+ + +{% if job_list %} + + + + + {% for job in job_list %} + + + + + + + + + + {% endfor %} + +{% else %} +

No jobs are available.

+{% endif %}
name owner hosts time spent begin memory used work open irq
{{ job.acct_id }} + {{ job.owner }}{{ job.nr_hosts }}{{ job.timespent }}{{ job.start_time }}{{ job.mem_MemUsed }}{{ job.llite_open_work }}{{ job.cpu_irq }}