[ARVADOS] updated: 26321ce3b54a0696982d55dabec7ff8082b146b9
git at public.curoverse.com
git at public.curoverse.com
Mon Nov 17 03:53:47 EST 2014
Summary of changes:
.../app/assets/javascripts/application.js | 1 +
apps/workbench/app/assets/javascripts/event_log.js | 203 +++++++++++----------
apps/workbench/app/helpers/jobs_helper.rb | 4 +-
.../vendor/assets/javascripts/jquery.number.min.js | 2 +
4 files changed, 115 insertions(+), 95 deletions(-)
create mode 100644 apps/workbench/vendor/assets/javascripts/jquery.number.min.js
via 26321ce3b54a0696982d55dabec7ff8082b146b9 (commit)
via 07ad3d1e604624893a945d08666046cc69568dab (commit)
from 73a8436194e335cfa3e208b2cc86350a2c35ae57 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 26321ce3b54a0696982d55dabec7ff8082b146b9
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Mon Nov 17 09:53:42 2014 +0100
4233: catch and ignore log processing errors
diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index d94c001..7bef6f4 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -61,113 +61,115 @@ $(document).on('ajax:complete ready', function() {
window.jobGraphData = [];
window.jobGraphSeries = [];
window.jobGraphMaxima = {};
-
- TODO: make this more robust: many type conversions etc problems could theoretically go wrong in here
*/
function processLogLineForChart( logLine ) {
var recreate = false;
var rescale = false;
- var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
- if( match ) {
- // the timestamp comes first
- var timestamp = match[1].replace('_','T');
- // for the series use the first word after 'crunchstat:'
- var series = match[5];
- // and append the task number (the 4th term)
- series += '-' + match[4]
- if( $.inArray( series, jobGraphSeries) < 0 ) {
- jobGraphSeries.push(series);
- jobGraphMaxima[series] = null;
- recreate = true;
- }
- var intervalData = match[7].trim().split(' ');
- var dt = parseFloat(intervalData[0]);
- var dsum = 0.0;
- for(var i=2; i < intervalData.length; i += 2 ) {
- dsum += parseFloat(intervalData[i]);
- }
- var datum = dsum/dt;
- if( datum !== 0 && ( jobGraphMaxima[series] === null || jobGraphMaxima[series] < datum ) ) {
- if( isJobSeriesRescalable(series) ) {
- // use old maximum to get a scale conversion
- var scaleConversion = jobGraphMaxima[series]/datum;
- // set new maximum and rescale the series
- jobGraphMaxima[series] = datum;
- rescaleJobGraphSeries( series, scaleConversion );
+ try {
+ var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
+ if( match ) {
+ // the timestamp comes first
+ var timestamp = match[1].replace('_','T');
+ // for the series use the first word after 'crunchstat:'
+ var series = match[5];
+ // and append the task number (the 4th term)
+ series += '-' + match[4]
+ if( $.inArray( series, jobGraphSeries) < 0 ) {
+ jobGraphSeries.push(series);
+ jobGraphMaxima[series] = null;
+ recreate = true;
+ }
+ var intervalData = match[7].trim().split(' ');
+ var dt = parseFloat(intervalData[0]);
+ var dsum = 0.0;
+ for(var i=2; i < intervalData.length; i += 2 ) {
+ dsum += parseFloat(intervalData[i]);
}
- // and special calculation for cpus
- if( /^cpu-/.test(series) ) {
- // divide the stat by the number of cpus
- var cpuCountMatch = match[6].match(/(\d+) cpus/);
- if( cpuCountMatch ) {
- datum = datum / cpuCountMatch[1];
+ var datum = dsum/dt;
+ if( datum !== 0 && ( jobGraphMaxima[series] === null || jobGraphMaxima[series] < datum ) ) {
+ if( isJobSeriesRescalable(series) ) {
+ // use old maximum to get a scale conversion
+ var scaleConversion = jobGraphMaxima[series]/datum;
+ // set new maximum and rescale the series
+ jobGraphMaxima[series] = datum;
+ rescaleJobGraphSeries( series, scaleConversion );
+ }
+ // and special calculation for cpus
+ if( /^cpu-/.test(series) ) {
+ // divide the stat by the number of cpus
+ var cpuCountMatch = match[6].match(/(\d+) cpus/);
+ if( cpuCountMatch ) {
+ datum = datum / cpuCountMatch[1];
+ }
}
}
- }
- // scale
- var scaledDatum = null;
- if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
- scaledDatum = datum/jobGraphMaxima[series]
- } else {
- scaledDatum = datum;
- }
- // identify x axis point, searching from the end of the array (most recent)
- var found = false;
- for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
- if( jobGraphData[i]['t'] === timestamp ) {
- found = true;
- jobGraphData[i][series] = scaledDatum;
- jobGraphData[i]['raw-'+series] = match[7];
- break;
- } else if( jobGraphData[i]['t'] < timestamp ) {
- // we've gone far enough back in time and this data is supposed to be sorted
- break;
+ // scale
+ var scaledDatum = null;
+ if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
+ scaledDatum = datum/jobGraphMaxima[series]
+ } else {
+ scaledDatum = datum;
}
- }
- // index counter from previous loop will have gone one too far, so add one
- var insertAt = i+1;
- if(!found) {
- // create a new x point for this previously unrecorded timestamp
- var entry = { 't': timestamp };
- entry[series] = scaledDatum;
- entry['raw-'+series] = match[7];
- jobGraphData.splice( insertAt, 0, entry );
- var shifted = [];
- // now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
- while( jobGraphData.length > 0
- && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 10*60000 < Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
- shifted.push(jobGraphData.shift());
+ // identify x axis point, searching from the end of the array (most recent)
+ var found = false;
+ for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
+ if( jobGraphData[i]['t'] === timestamp ) {
+ found = true;
+ jobGraphData[i][series] = scaledDatum;
+ jobGraphData[i]['raw-'+series] = match[7];
+ break;
+ } else if( jobGraphData[i]['t'] < timestamp ) {
+ // we've gone far enough back in time and this data is supposed to be sorted
+ break;
+ }
}
- if( shifted.length > 0 ) {
- // from those that we dropped, are any of them maxima? if so we need to rescale
- jobGraphSeries.forEach( function(series) {
- // test that every shifted entry in this series was either not a number (in which case we don't care)
- // or else approximately (to 2 decimal places) smaller than the scaled maximum (i.e. 1),
- // because otherwise we just scrolled off something that was a maximum point
- // and so we need to recalculate a new maximum point by looking at all remaining displayed points in the series
- if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null
- && !shifted.every( function(e) { return( !$.isNumeric(e[series]) || e[series].toFixed(2) < 1.0 ) } ) ) {
- // check the remaining displayed points and find the new (scaled) maximum
- var seriesMax = null;
- jobGraphData.forEach( function(entry) {
- if( $.isNumeric(entry[series]) && (seriesMax === null || entry[series] > seriesMax)) {
- seriesMax = entry[series];
+ // index counter from previous loop will have gone one too far, so add one
+ var insertAt = i+1;
+ if(!found) {
+ // create a new x point for this previously unrecorded timestamp
+ var entry = { 't': timestamp };
+ entry[series] = scaledDatum;
+ entry['raw-'+series] = match[7];
+ jobGraphData.splice( insertAt, 0, entry );
+ var shifted = [];
+ // now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
+ while( jobGraphData.length > 0
+ && (Date.parse( jobGraphData[0]['t'] ).valueOf() + 10*60000 < Date.parse( jobGraphData[jobGraphData.length-1]['t'] ).valueOf()) ) {
+ shifted.push(jobGraphData.shift());
+ }
+ if( shifted.length > 0 ) {
+ // from those that we dropped, are any of them maxima? if so we need to rescale
+ jobGraphSeries.forEach( function(series) {
+ // test that every shifted entry in this series was either not a number (in which case we don't care)
+ // or else approximately (to 2 decimal places) smaller than the scaled maximum (i.e. 1),
+ // because otherwise we just scrolled off something that was a maximum point
+ // and so we need to recalculate a new maximum point by looking at all remaining displayed points in the series
+ if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null
+ && !shifted.every( function(e) { return( !$.isNumeric(e[series]) || e[series].toFixed(2) < 1.0 ) } ) ) {
+ // check the remaining displayed points and find the new (scaled) maximum
+ var seriesMax = null;
+ jobGraphData.forEach( function(entry) {
+ if( $.isNumeric(entry[series]) && (seriesMax === null || entry[series] > seriesMax)) {
+ seriesMax = entry[series];
+ }
+ });
+ if( seriesMax !== null && seriesMax !== 0 ) {
+ // set new actual maximum using the new maximum as the conversion conversion and rescale the series
+ jobGraphMaxima[series] *= seriesMax;
+ var scaleConversion = 1/seriesMax;
+ rescaleJobGraphSeries( series, scaleConversion );
+ }
+ else {
+ // we no longer have any data points displaying for this series
+ jobGraphMaxima[series] = null;
}
- });
- if( seriesMax !== null && seriesMax !== 0 ) {
- // set new actual maximum using the new maximum as the conversion conversion and rescale the series
- jobGraphMaxima[series] *= seriesMax;
- var scaleConversion = 1/seriesMax;
- rescaleJobGraphSeries( series, scaleConversion );
- }
- else {
- // we no longer have any data points displaying for this series
- jobGraphMaxima[series] = null;
}
- }
- });
+ });
+ }
}
}
+ } catch( err ) {
+ console.log( 'Ignoring error trying to process log line: ' + err);
}
return recreate;
}
commit 07ad3d1e604624893a945d08666046cc69568dab
Author: Phil Hodgson <bitbucket at philhodgson.net>
Date: Mon Nov 17 09:49:23 2014 +0100
4233: better labelling includes raw interval data and number formatting
diff --git a/apps/workbench/app/assets/javascripts/application.js b/apps/workbench/app/assets/javascripts/application.js
index c39df63..796053b 100644
--- a/apps/workbench/app/assets/javascripts/application.js
+++ b/apps/workbench/app/assets/javascripts/application.js
@@ -25,6 +25,7 @@
//= require wiselinks
//= require raphael
//= require morris
+//= require jquery.number.min
//= require_tree .
jQuery(function($){
diff --git a/apps/workbench/app/assets/javascripts/event_log.js b/apps/workbench/app/assets/javascripts/event_log.js
index a763d32..d94c001 100644
--- a/apps/workbench/app/assets/javascripts/event_log.js
+++ b/apps/workbench/app/assets/javascripts/event_log.js
@@ -61,11 +61,12 @@ $(document).on('ajax:complete ready', function() {
window.jobGraphData = [];
window.jobGraphSeries = [];
window.jobGraphMaxima = {};
+
+ TODO: make this more robust: many type conversions etc problems could theoretically go wrong in here
*/
function processLogLineForChart( logLine ) {
var recreate = false;
var rescale = false;
- // TODO: make this more robust: anything could go wrong in here
var match = logLine.match(/(\S+) (\S+) (\S+) (\S+) stderr crunchstat: (\S+) (.*) -- interval (.*)/);
if( match ) {
// the timestamp comes first
@@ -104,19 +105,19 @@ function processLogLineForChart( logLine ) {
}
}
// scale
- // FIXME: what about negative numbers?
var scaledDatum = null;
if( isJobSeriesRescalable(series) && jobGraphMaxima[series] !== null && jobGraphMaxima[series] !== 0 ) {
scaledDatum = datum/jobGraphMaxima[series]
} else {
scaledDatum = datum;
}
- // identify x axis point
+ // identify x axis point, searching from the end of the array (most recent)
var found = false;
for( var i = jobGraphData.length - 1; i >= 0; i-- ) {
if( jobGraphData[i]['t'] === timestamp ) {
found = true;
jobGraphData[i][series] = scaledDatum;
+ jobGraphData[i]['raw-'+series] = match[7];
break;
} else if( jobGraphData[i]['t'] < timestamp ) {
// we've gone far enough back in time and this data is supposed to be sorted
@@ -129,6 +130,7 @@ function processLogLineForChart( logLine ) {
// create a new x point for this previously unrecorded timestamp
var entry = { 't': timestamp };
entry[series] = scaledDatum;
+ entry['raw-'+series] = match[7];
jobGraphData.splice( insertAt, 0, entry );
var shifted = [];
// now let's see about "scrolling" the graph, dropping entries that are too old (>10 minutes)
@@ -176,6 +178,7 @@ function createJobGraph(elementName) {
element: elementName,
data: jobGraphData,
ymax: 1.0,
+ yLabelFormat: function () { return ''; },
xkey: 't',
ykeys: jobGraphSeries,
labels: jobGraphSeries,
@@ -196,7 +199,19 @@ function createJobGraph(elementName) {
if( isJobSeriesRescalable( series ) ) {
datum *= jobGraphMaxima[series];
}
- s += datum.toFixed(2);
+ if( parseFloat(datum) !== 0 ) {
+ if( /^cpu-/.test(series) ){
+ datum = $.number(datum * 100, 1) + '%';
+ } else if( datum < 10 ) {
+ datum = $.number(datum, 2);
+ } else {
+ datum = $.number(datum);
+ }
+ datum += ' (' + options.data[index]['raw-'+series] + ')';
+ }
+ s += datum;
+ } else {
+ s += '-';
}
s += "</div> ";
}
diff --git a/apps/workbench/app/helpers/jobs_helper.rb b/apps/workbench/app/helpers/jobs_helper.rb
index ec6fccf..aca0999 100644
--- a/apps/workbench/app/helpers/jobs_helper.rb
+++ b/apps/workbench/app/helpers/jobs_helper.rb
@@ -24,9 +24,9 @@ module JobsHelper
filters += extra_filters if extra_filters
last_entry = Log.order('id DESC').limit(1).filter(filters).results.first
if last_entry
- filters += [["event_at", ">=", last_entry.event_at - 3.minutes]]
+ filters += [["event_at", ">=", last_entry.event_at - 5.minutes]]
Log.order('id DESC')
- .limit(10000)
+ .limit(250)
.filter(filters)
.results
else
diff --git a/apps/workbench/vendor/assets/javascripts/jquery.number.min.js b/apps/workbench/vendor/assets/javascripts/jquery.number.min.js
new file mode 100644
index 0000000..4fce02b
--- /dev/null
+++ b/apps/workbench/vendor/assets/javascripts/jquery.number.min.js
@@ -0,0 +1,2 @@
+/*! jQuery number 2.1.5 (c) github.com/teamdf/jquery-number | opensource.teamdf.com/license */
+(function(e){"use strict";function t(e,t){if(this.createTextRange){var n=this.createTextRange();n.collapse(true);n.moveStart("character",e);n.moveEnd("character",t-e);n.select()}else if(this.setSelectionRange){this.focus();this.setSelectionRange(e,t)}}function n(e){var t=this.value.length;e=e.toLowerCase()=="start"?"Start":"End";if(document.selection){var n=document.selection.createRange(),r,i,s;r=n.duplicate();r.expand("textedit");r.setEndPoint("EndToEnd",n);i=r.text.length-n.text.length;s=i+n.text.length;return e=="Start"?i:s}else if(typeof this["selection"+e]!="undefined"){t=this["selection"+e]}return t}var r={codes:{46:127,188:44,109:45,190:46,191:47,192:96,220:92,222:39,221:93,219:91,173:45,187:61,186:59,189:45,110:46},shifts:{96:"~",49:"!",50:"@",51:"#",52:"$",53:"%",54:"^",55:"&",56:"*",57:"(",48:")",45:"_",61:"+",91:"{",93:"}",92:"|",59:":",39:'"',44:"<",46:">",47:"?"}};e.fn.number=function(i,s,o,u){u=typeof u==="undefined"?",":u;o=typeof o==="undefined"?".":o;s=typeof s==="undefined"?0:s;var a="\\u"+("0000"+o.charCodeAt(0).toString(16)).slice(-4),f=new RegExp("[^"+a+"0-9]","g"),l=new RegExp(a,"g");if(i===true){if(this.is("input:text")){return this.on({"keydown.format":function(i){var a=e(this),f=a.data("numFormat"),l=i.keyCode?i.keyCode:i.which,c="",h=n.apply(this,["start"]),p=n.apply(this,["end"]),d="",v=false;if(r.codes.hasOwnProperty(l)){l=r.codes[l]}if(!i.shiftKey&&l>=65&&l<=90){l+=32}else if(!i.shiftKey&&l>=69&&l<=105){l-=48}else if(i.shiftKey&&r.shifts.hasOwnProperty(l)){c=r.shifts[l]}if(c=="")c=String.fromCharCode(l);if(l!=8&&l!=45&&l!=127&&c!=o&&!c.match(/[0-9]/)){var m=i.keyCode?i.keyCode:i.which;if(m==46||m==8||m==127||m==9||m==27||m==13||(m==65||m==82||m==80||m==83||m==70||m==72||m==66||m==74||m==84||m==90||m==61||m==173||m==48)&&(i.ctrlKey||i.metaKey)===true||(m==86||m==67||m==88)&&(i.ctrlKey||i.metaKey)===true||m>=35&&m<=39||m>=112&&m<=123){return}i.preventDefault();return false}if(h==0&&p==this.value.length||a.val()==0){if(l==8){h=p=1;this.value="";f.init=s>0?-1:0;f.c=s>0?-(s+1):0;t.apply(this,[0,0])}else if(c==o){h=p=1;this.value="0"+o+(new Array(s+1)).join("0");f.init=s>0?1:0;f.c=s>0?-(s+1):0}else if(l==45){h=p=2;this.value="-0"+o+(new Array(s+1)).join("0");f.init=s>0?1:0;f.c=s>0?-(s+1):0;t.apply(this,[2,2])}else{f.init=s>0?-1:0;f.c=s>0?-s:0}}else{f.c=p-this.value.length}f.isPartialSelection=h==p?false:true;if(s>0&&c==o&&h==this.value.length-s-1){f.c++;f.init=Math.max(0,f.init);i.preventDefault();v=this.value.length+f.c}else if(l==45&&(h!=0||this.value.indexOf("-")==0)){i.preventDefault()}else if(c==o){f.init=Math.max(0,f.init);i.preventDefault()}else if(s>0&&l==127&&h==this.value.length-s-1){i.preventDefault()}else if(s>0&&l==8&&h==this.value.length-s){i.preventDefault();f.c--;v=this.value.length+f.c}else if(s>0&&l==127&&h>this.value.length-s-1){if(this.value==="")return;if(this.value.slice(h,h+1)!="0"){d=this.value.slice(0,h)+"0"+this.value.slice(h+1);a.val(d)}i.preventDefault();v=this.value.length+f.c}else if(s>0&&l==8&&h>this.value.length-s){if(this.value==="")return;if(this.value.slice(h-1,h)!="0"){d=this.value.slice(0,h-1)+"0"+this.value.slice(h);a.val(d)}i.preventDefault();f.c--;v=this.value.length+f.c}else if(l==127&&this.value.slice(h,h+1)==u){i.preventDefault()}else if(l==8&&this.value.slice(h-1,h)==u){i.preventDefault();f.c--;v=this.value.length+f.c}else if(s>0&&h==p&&this.value.length>s+1&&h>this.value.length-s-1&&isFinite(+c)&&!i.metaKey&&!i.ctrlKey&&!i.altKey&&c.length===1){if(p===this.value.length){d=this.value.slice(0,h-1)}else{d=this.value.slice(0,h)+this.value.slice(h+1)}this.value=d;v=h}if(v!==false){t.apply(this,[v,v])}a.data("numFormat",f)},"keyup.format":function(r){var i=e(this),o=i.data("numFormat"),u=r.keyCode?r.keyCode:r.which,a=n.apply(this,["start"]),f=n.apply(this,["end"]),l;if(a===0&&f===0&&(u===189||u===109)){i.val("-"+i.val());a=1;o.c=1-this.value.length;o.init=1;i.data("numFormat",o);l=this.value.length+o.c;t.apply(this,[l,l])}if(this.value===""||(u<48||u>57)&&(u<96||u>105)&&u!==8&&u!==46&&u!==110)return;i.val(i.val());if(s>0){if(o.init<1){a=this.value.length-s-(o.init<0?1:0);o.c=a-this.value.length;o.init=1;i.data("numFormat",o)}else if(a>this.value.length-s&&u!=8){o.c++;i.data("numFormat",o)}}if(u==46&&!o.isPartialSelection){o.c++;i.data("numFormat",o)}l=this.value.length+o.c;t.apply(this,[l,l])},"paste.format":function(t){var n=e(this),r=t.originalEvent,i=null;if(window.clipboardData&&window.clipboardData.getData){i=window.clipboardData.getData("Text")}else if(r.clipboardData&&r.clipboardData.getData){i=r.clipboardData.getData("text/plain")}n.val(i);t.preventDefault();return false}}).each(function(){var t=e(this).data("numFormat",{c:-(s+1),decimals:s,thousands_sep:u,dec_point:o,regex_dec_num:f,regex_dec:l,init:this.value.indexOf(".")?true:false});if(this.value==="")return;t.val(t.val())})}else{return this.each(function(){var t=e(this),n=+t.text().replace(f,"").replace(l,".");t.number(!isFinite(n)?0:+n,s,o,u)})}}return this.text(e.number.apply(window,arguments))};var i=null,s=null;if(e.isPlainObject(e.valHooks.text)){if(e.isFunction(e.valHooks.text.get))i=e.valHooks.text.get;if(e.isFunction(e.valHooks.text.set))s=e.valHooks.text.set}else{e.valHooks.text={}}e.valHooks.text.get=function(t){var n=e(t),r,s,o=n.data("numFormat");if(!o){if(e.isFunction(i)){return i(t)}else{return undefined}}else{if(t.value==="")return"";r=+t.value.replace(o.regex_dec_num,"").replace(o.regex_dec,".");return(t.value.indexOf("-")===0?"-":"")+(isFinite(r)?r:0)}};e.valHooks.text.set=function(t,n){var r=e(t),i=r.data("numFormat");if(!i){if(e.isFunction(s)){return s(t,n)}else{return undefined}}else{var o=e.number(n,i.decimals,i.dec_point,i.thousands_sep);return t.value=o}};e.number=function(e,t,n,r){r=typeof r==="undefined"?",":r;n=typeof n==="undefined"?".":n;t=!isFinite(+t)?0:Math.abs(t);var i="\\u"+("0000"+n.charCodeAt(0).toString(16)).slice(-4);var s="\\u"+("0000"+r.charCodeAt(0).toString(16)).slice(-4);e=(e+"").replace(".",n).replace(new RegExp(s,"g"),"").replace(new RegExp(i,"g"),".").replace(new RegExp("[^0-9+-Ee.]","g"),"");var o=!isFinite(+e)?0:+e,u="",a=function(e,t){var n=Math.pow(10,t);return""+Math.round(e*n)/n};u=(t?a(o,t):""+Math.round(o)).split(".");if(u[0].length>3){u[0]=u[0].replace(/\B(?=(?:\d{3})+(?!\d))/g,r)}if((u[1]||"").length<t){u[1]=u[1]||"";u[1]+=(new Array(t-u[1].length+1)).join("0")}return u.join(n)}})(jQuery)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list