diff --git a/tacc_stats_web/apps/tacc_stats/forms.py b/tacc_stats_web/apps/tacc_stats/forms.py new file mode 100644 index 0000000..fb4af12 --- /dev/null +++ b/tacc_stats_web/apps/tacc_stats/forms.py @@ -0,0 +1,18 @@ +from django import forms +from django.forms import ModelForm +from models import Job + +class SearchForm(ModelForm): + class Meta: + model = Job + fields = ('owner', 'begin', 'end', 'hosts', 'acct_id') +# widgets = { +# 'hosts': forms.TextInput(attrs={'size':'40'}), +# } + + def __init__(self, *args, **kwargs): + super(SearchForm, self).__init__(*args, **kwargs) + + for key in self.fields: + self.fields[key].required = False + diff --git a/tacc_stats_web/apps/tacc_stats/models.py b/tacc_stats_web/apps/tacc_stats/models.py index 8d2d44e..8bdcd43 100644 --- a/tacc_stats_web/apps/tacc_stats/models.py +++ b/tacc_stats_web/apps/tacc_stats/models.py @@ -1,6 +1,7 @@ """The database models of tacc stats""" from django.db import models +import time class System(models.Model): """Details about the cluster""" @@ -20,6 +21,9 @@ class User(models.Model): user_name = models.CharField(max_length=128) systems = models.ManyToManyField(System) + def __str__(self): + return self.user_name + def __unicode__(self): return "User(%s)" % self.user_name @@ -237,15 +241,20 @@ def nr_hosts(self): return len(self.hosts.all()) def color(self): - ret_val = "LightBlue" + ret_val = "background-color: rgba(0%, 0%, 100%, .3);" if self.llite_open_work > 3000: - ret_val = "red" + ret_val = "background-color: rgba(100%, 0%, 0%, .3);" elif self.mem_MemUsed > 30*2**30: - ret_val = "orange" + ret_val = "background-color: rgba(80%, 30%, 0%, .3)" elif self.runtime > 3000: - ret_val = "LightCoral" + ret_val = "background-color: rgba(0%, 100%, 0%, .3)" return ret_val + def timespent(self): + return time.strftime('%H:%M:%S', time.gmtime(self.runtime)) + + def start_time(self): + return time.ctime(self.begin) class Monitor(models.Model): kind = models.CharField(max_length=32) diff --git a/tacc_stats_web/apps/tacc_stats/urls.py b/tacc_stats_web/apps/tacc_stats/urls.py index d41fada..3e7de13 100644 --- a/tacc_stats_web/apps/tacc_stats/urls.py +++ b/tacc_stats_web/apps/tacc_stats/urls.py @@ -1,16 +1,12 @@ from django.conf.urls.defaults import patterns, url from django.views.generic import DetailView, ListView from tacc_stats.models import Job -from tacc_stats.views import index, job_memused_hist, job_timespent_hist, job_mem_heatmap, job_files_open_heatmap, create_heatmap +from tacc_stats.views import index, job_memused_hist, job_timespent_hist, create_heatmap, search, JobListView urlpatterns = patterns('', url(r'^$', index), url(r'^joblist$', - ListView.as_view( - queryset=Job.objects.order_by('-acct_id')[:200], - # context_object_name='latest_job_list', - # template_name='tacc_stats/index.html', - )), + JobListView.as_view()), url(r'^job/(?P\d+)/$', DetailView.as_view( model=Job, @@ -20,5 +16,6 @@ url(r'^job_timespent_hist$', job_timespent_hist ), url(r'^job_mem_heatmap/(\d+)/$', create_heatmap, {'trait' : 'memory'}), url(r'^job_files_opened_heatmap/(\d+)/$', create_heatmap, {'trait' : 'files'}), - url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'}) + url(r'^job_flops_heatmap/(\d+)/$', create_heatmap, {'trait' : 'flops'}), + url(r'^search/$', search ), ) diff --git a/tacc_stats_web/apps/tacc_stats/views.py b/tacc_stats_web/apps/tacc_stats/views.py index 6c00721..9c80cbc 100644 --- a/tacc_stats_web/apps/tacc_stats/views.py +++ b/tacc_stats_web/apps/tacc_stats/views.py @@ -3,19 +3,26 @@ from django.http import HttpResponse from django.shortcuts import render_to_response +from django.views.decorators.csrf import csrf_protect +from django.shortcuts import render +from django.views.generic import DetailView, ListView import matplotlib matplotlib.use('Agg') from matplotlib.backends.backend_agg import FigureCanvasAgg from pylab import figure, axes, pie, title, hist, xlabel, ylabel from matplotlib import pyplot as PLT +from matplotlib.colors import LogNorm import shelve import job import numpy as NP +import math from tacc_stats.models import Job +from forms import SearchForm + SHELVE_DIR = '/home/dmalone/sample-jobs/jobs' def index(request): @@ -89,8 +96,8 @@ def _files_open_intensity(job, host): for filesystem in job.hosts[host].stats['llite']: files_opened = files_opened + job.hosts[host].stats['llite'][filesystem][: , files_opened_index] - difference = NP.diff(files_opened) - intensity = NP.append(0, difference) / difference.max() + intensity = NP.diff(files_opened) + return intensity def _flops_intensity(job, host): @@ -103,6 +110,7 @@ def _flops_intensity(job, host): job -- the job being accessed host -- the host being charted """ + RANGER_MAX_FLOPS = 88326000000000 flops_used = [0] * job.times.size cpu_data = job.hosts[host].interpret_amd64_pmc_cpu() @@ -111,8 +119,8 @@ def _flops_intensity(job, host): if (key[:4] == 'core'): flops_used = flops_used + val['SSEFLOPS'] - difference = NP.diff(flops_used) - intensity = NP.append(0, difference) / difference.max() + intensity = NP.log(NP.diff(flops_used)) / math.log(RANGER_MAX_FLOPS) + return intensity def create_subheatmap(intensity, job, host, n, num_hosts): @@ -135,7 +143,7 @@ def create_subheatmap(intensity, job, host, n, num_hosts): intensity = NP.array([intensity]*2, dtype=NP.float64) PLT.subplot(num_hosts, 1, n) - PLT.pcolor(x, NP.array([0, 1]), intensity, cmap=matplotlib.cm.Reds, vmin = 0, vmax = 1) + PLT.pcolor(x, NP.array([0, 1]), intensity, cmap=matplotlib.cm.Reds, vmin = 0, vmax = 1, edgecolors='none') if (n != num_hosts): PLT.xticks([]) @@ -196,52 +204,47 @@ def create_heatmap(request, job_id, trait): f.set_size_inches(10,num_hosts*.3+1.5) return figure_to_response(f) -def job_mem_heatmap(request, job_id): - """ - Creates a heatmap with intensity correlated with the amount of memory used by the job - """ - job_shelf = shelve.open(SHELVE_DIR) - - job = job_shelf[job_id] - - hosts = job.hosts.keys() - - n = 1 - num_hosts = len(job.hosts) - PLT.subplots_adjust(hspace = 0) - PLT.suptitle('Memory Used By Host', fontsize = 12) - - for host in hosts: - intensity = _memory_intensity(job, host) - create_subheatmap(intensity, job, host, n, num_hosts) - n += 1 - - f = PLT.gcf() - - f.set_size_inches(10,num_hosts*.3+1.5) - return figure_to_response(f) - -def job_files_open_heatmap(request, job_id): - """ - Creates a heatmap with intensity correlated with the amount of files a job opens +@csrf_protect +def search(request): + """ + Creates a search form that can be used to navigate through the list + of jobs. """ - job_shelf = shelve.open(SHELVE_DIR) - - job = job_shelf[job_id] + if request.method == 'POST': + print request.POST - hosts = job.hosts.keys() - - n = 1 - num_hosts = len(job.hosts) - PLT.subplots_adjust(hspace = 0) - PLT.suptitle('Files Used By Host', fontsize = 12) + form = SearchForm(request.POST) + query = request.POST - for host in hosts: - intensity = _flops_intensity(job, host) - create_subheatmap(intensity, job, host, n, num_hosts) - n += 1 + job_list = Job.objects.all() - f = PLT.gcf() + if form["acct_id"].value(): + job_list = job_list.filter(acct_id = form["acct_id"].value()) + if form["owner"].value(): + job_list = job_list.filter(owner = form["owner"].value()) + if form["begin"].value(): + job_list = job_list.filter(begin__gte = form["begin"].value()) + if form["end"].value(): + job_list = job_list.filter(end__lte = form["end"].value()) - f.set_size_inches(10,num_hosts*.3+1.5) - return figure_to_response(f) + else: + form = SearchForm() + job_list = Job.objects.order_by('-begin')[:200] + + return render(request, 'tacc_stats/search.html', {'form' : form, 'job_list' : job_list }) + +class JobListView(ListView): + + def get_queryset(self): + if self.request.method == 'POST': + query = self.request.POST + return Job.objects.order_by('-begin')[:200] + #return Job.objects.filter( + # owner = query.__getitem__("owner"), + # begin = query.__getitem__("begin"), + # end = query.__getitem__("end"), + # hosts = query.__getitem__("hosts"), + # acct_id = query.__getitem__("acct_id") + # ) + else: + return Job.objects.order_by('-begin')[:200] diff --git a/tacc_stats_web/templates/tacc_stats/index.html b/tacc_stats_web/templates/tacc_stats/index.html index abab218..56a23e6 100644 --- a/tacc_stats_web/templates/tacc_stats/index.html +++ b/tacc_stats_web/templates/tacc_stats/index.html @@ -4,7 +4,8 @@

TACC Stats Web Home

-joblist +jobsearch +

Charts at a glance

Histogram of jobs @@ -16,7 +17,7 @@

Recent problem jobs

ID owner time spent memory used work open irq {% for job in job_list %} - + {{ job.acct_id }} {{ job.owner }} {{ job.runtime }} diff --git a/tacc_stats_web/templates/tacc_stats/job_detail.html b/tacc_stats_web/templates/tacc_stats/job_detail.html index 79416e2..78abfb6 100644 --- a/tacc_stats_web/templates/tacc_stats/job_detail.html +++ b/tacc_stats_web/templates/tacc_stats/job_detail.html @@ -10,6 +10,8 @@

Job {{ job.acct_id }}

Files Opened Heatmap + Flops Performed Heatmap +

Accounting Details

diff --git a/tacc_stats_web/templates/tacc_stats/job_list.html b/tacc_stats_web/templates/tacc_stats/job_list.html index 816e043..da6fbec 100644 --- a/tacc_stats_web/templates/tacc_stats/job_list.html +++ b/tacc_stats_web/templates/tacc_stats/job_list.html @@ -5,13 +5,16 @@ {% if job_list %}
FieldValue
- + + + {% for job in job_list %} - + - + + + diff --git a/tacc_stats_web/templates/tacc_stats/search.html b/tacc_stats_web/templates/tacc_stats/search.html new file mode 100644 index 0000000..09cacee --- /dev/null +++ b/tacc_stats_web/templates/tacc_stats/search.html @@ -0,0 +1,28 @@ +{% csrf_token %} +
ID owner time spent memory used - work open irq
name owner hosts time spent begin memory used work open irq
{{ job.acct_id }} {{ job.owner }}{{ job.runtime }}{{ job.nr_hosts }}{{ job.timespent }}{{ job.start_time }} {{ job.mem_MemUsed }} {{ job.llite_open_work }} {{ job.cpu_irq }}
+ {{ form.as_table }} +
+

+ + +{% if job_list %} + + + + + {% for job in job_list %} + + + + + + + + + + {% endfor %} + +{% else %} +

No jobs are available.

+{% endif %}
name owner hosts time spent begin memory used work open irq
{{ job.acct_id }} + {{ job.owner }}{{ job.nr_hosts }}{{ job.timespent }}{{ job.start_time }}{{ job.mem_MemUsed }}{{ job.llite_open_work }}{{ job.cpu_irq }}