[ARVADOS] updated: 04d8e74e9b6476b2a87c5a66c07566f041c97b37
Git user
git at public.curoverse.com
Tue Sep 20 20:23:31 EDT 2016
Summary of changes:
.../{log_viewer.js => job_log_viewer.js} | 51 +----
.../workbench/app/assets/javascripts/log_viewer.js | 244 +--------------------
apps/workbench/app/views/jobs/_show_log.html.erb | 2 +-
.../app/views/work_units/_show_log.html.erb | 121 ++++++++--
build/run-build-packages.sh | 2 +-
sdk/cwl/arvados_cwl/pathmapper.py | 2 +
sdk/cwl/arvados_cwl/runner.py | 37 +++-
sdk/cwl/setup.py | 2 +-
sdk/python/arvados/collection.py | 25 ++-
services/api/test/fixtures/container_requests.yml | 2 +-
services/keep-balance/keep-balance.service | 13 ++
11 files changed, 183 insertions(+), 318 deletions(-)
copy apps/workbench/app/assets/javascripts/{log_viewer.js => job_log_viewer.js} (82%)
create mode 100644 services/keep-balance/keep-balance.service
via 04d8e74e9b6476b2a87c5a66c07566f041c97b37 (commit)
via 878275d3bb35add83cdbe45c93bec353e9f8e7e8 (commit)
via 6802d84dd40245f2c605364dcd6d42849c409324 (commit)
via dcf24627def74e4357619b8128336c01783cafdf (commit)
via d9eec0b9bf52b428b5506617590d87b2d92eb246 (commit)
via 8cb96c1b23d02be8fac545d8c05a167454c1a11d (commit)
via 13af0befe3b6c8d77f9a9d4037f960700e60ec5e (commit)
via 8a442154b136d9dcfc038cd61ef889ece0052edc (commit)
via 511e36db2a5ed0380752d15e9c60364483d0e9f0 (commit)
via b810854ddeccc8fff3ac718e8085a4cafb7f6b86 (commit)
via d63f3ade27bb506da049164789679f9fed171005 (commit)
via 17e4fbd0495b595b5cd959b389798201bebe8929 (commit)
from e3de0a09618c8aa46833357e6eacabe15bb68f6f (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 04d8e74e9b6476b2a87c5a66c07566f041c97b37
Author: radhika <radhika at curoverse.com>
Date: Tue Sep 20 20:22:09 2016 -0400
9514: Refactor job log related bit out of log_viewer.js into job_log_viewer.js
diff --git a/apps/workbench/app/assets/javascripts/log_viewer.js b/apps/workbench/app/assets/javascripts/job_log_viewer.js
similarity index 82%
copy from apps/workbench/app/assets/javascripts/log_viewer.js
copy to apps/workbench/app/assets/javascripts/job_log_viewer.js
index 0e12f9c..4fa038a 100644
--- a/apps/workbench/app/assets/javascripts/log_viewer.js
+++ b/apps/workbench/app/assets/javascripts/job_log_viewer.js
@@ -6,7 +6,7 @@ function newTaskState() {
"nodes": []};
}
-function addToLogViewer(logViewer, lines, taskState) {
+function addToJobLogViewer(logViewer, lines, taskState) {
var re = /((\d\d\d\d)-(\d\d)-(\d\d))_((\d\d):(\d\d):(\d\d)) ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}) (\d+) (\d+)? (.*)/;
var items = [];
@@ -231,52 +231,3 @@ function generateJobOverview(id, logViewer, taskState) {
$(id).html(html);
}
-
-function gotoPage(n, logViewer, page, id) {
- if (n < 0) { return; }
- if (n*page > logViewer.matchingItems.length) { return; }
- logViewer.page_offset = n;
- logViewer.show(n*page, page);
-}
-
-function updatePaging(id, logViewer, page) {
- var p = "";
- var i = logViewer.matchingItems.length;
- var n;
- for (n = 0; (n*page) < i; n += 1) {
- if (n == logViewer.page_offset) {
- p += "<span class='log-viewer-page-num'>" + (n+1) + "</span> ";
- } else {
- p += "<a href=\"#\" class='log-viewer-page-num log-viewer-page-" + n + "'>" + (n+1) + "</a> ";
- }
- }
- $(id).html(p);
- for (n = 0; (n*page) < i; n += 1) {
- (function(n) {
- $(".log-viewer-page-" + n).on("click", function() {
- gotoPage(n, logViewer, page, id);
- return false;
- });
- })(n);
- }
-
- if (logViewer.page_offset == 0) {
- $(".log-viewer-page-up").addClass("text-muted");
- } else {
- $(".log-viewer-page-up").removeClass("text-muted");
- }
-
- if (logViewer.page_offset == (n-1)) {
- $(".log-viewer-page-down").addClass("text-muted");
- } else {
- $(".log-viewer-page-down").removeClass("text-muted");
- }
-}
-
-function nextPage(logViewer, page, id) {
- gotoPage(logViewer.page_offset+1, logViewer, page, id);
-}
-
-function prevPage(logViewer, page, id) {
- gotoPage(logViewer.page_offset-1, logViewer, page, id);
-}
diff --git a/apps/workbench/app/assets/javascripts/log_viewer.js b/apps/workbench/app/assets/javascripts/log_viewer.js
index 0e12f9c..58bde9a 100644
--- a/apps/workbench/app/assets/javascripts/log_viewer.js
+++ b/apps/workbench/app/assets/javascripts/log_viewer.js
@@ -1,237 +1,3 @@
-function newTaskState() {
- return {"complete_count": 0,
- "failure_count": 0,
- "task_count": 0,
- "incomplete_count": 0,
- "nodes": []};
-}
-
-function addToLogViewer(logViewer, lines, taskState) {
- var re = /((\d\d\d\d)-(\d\d)-(\d\d))_((\d\d):(\d\d):(\d\d)) ([a-z0-9]{5}-[a-z0-9]{5}-[a-z0-9]{15}) (\d+) (\d+)? (.*)/;
-
- var items = [];
- var count = logViewer.items.length;
- for (var a in lines) {
- var v = lines[a].match(re);
- if (v != null) {
-
- var ts = new Date(Date.UTC(v[2], v[3]-1, v[4], v[6], v[7], v[8]));
-
- v11 = v[11];
- if (typeof v[11] === 'undefined') {
- v11 = "";
- } else {
- v11 = Number(v11);
- }
-
- var message = v[12];
- var type = "";
- var node = "";
- var slot = "";
- if (v11 !== "") {
- if (!taskState.hasOwnProperty(v11)) {
- taskState[v11] = {};
- taskState.task_count += 1;
- }
-
- if (/^stderr /.test(message)) {
- message = message.substr(7);
- if (/^crunchstat: /.test(message)) {
- type = "crunchstat";
- message = message.substr(12);
- } else if (/^srun: /.test(message) || /^slurmd/.test(message)) {
- type = "task-dispatch";
- } else {
- type = "task-print";
- }
- } else {
- var m;
- if (m = /^success in (\d+) second/.exec(message)) {
- taskState[v11].outcome = "success";
- taskState[v11].runtime = Number(m[1]);
- taskState.complete_count += 1;
- }
- else if (m = /^failure \(\#\d+, (temporary|permanent)\) after (\d+) second/.exec(message)) {
- taskState[v11].outcome = "failure";
- taskState[v11].runtime = Number(m[2]);
- taskState.failure_count += 1;
- if (m[1] == "permanent") {
- taskState.incomplete_count += 1;
- }
- }
- else if (m = /^child \d+ started on ([^.]*)\.(\d+)/.exec(message)) {
- taskState[v11].node = m[1];
- taskState[v11].slot = m[2];
- if (taskState.nodes.indexOf(m[1], 0) == -1) {
- taskState.nodes.push(m[1]);
- }
- for (var i in items) {
- if (i > 0) {
- if (items[i].taskid === v11) {
- items[i].node = m[1];
- items[i].slot = m[2];
- }
- }
- }
- }
- type = "task-dispatch";
- }
- node = taskState[v11].node;
- slot = taskState[v11].slot;
- } else {
- type = "crunch";
- }
-
- items.push({
- id: count,
- ts: ts,
- timestamp: ts.toLocaleDateString() + " " + ts.toLocaleTimeString(),
- taskid: v11,
- node: node,
- slot: slot,
- message: message.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>'),
- type: type
- });
- count += 1;
- } else {
- console.log("Did not parse line " + a + ": " + lines[a]);
- }
- }
- logViewer.add(items);
-}
-
-function sortById(a, b, opt) {
- a = a.values();
- b = b.values();
-
- if (a["id"] > b["id"]) {
- return 1;
- }
- if (a["id"] < b["id"]) {
- return -1;
- }
- return 0;
-}
-
-function sortByTask(a, b, opt) {
- var aa = a.values();
- var bb = b.values();
-
- if (aa["taskid"] === "" && bb["taskid"] !== "") {
- return -1;
- }
- if (aa["taskid"] !== "" && bb["taskid"] === "") {
- return 1;
- }
-
- if (aa["taskid"] !== "" && bb["taskid"] !== "") {
- if (aa["taskid"] > bb["taskid"]) {
- return 1;
- }
- if (aa["taskid"] < bb["taskid"]) {
- return -1;
- }
- }
-
- return sortById(a, b, opt);
-}
-
-function sortByNode(a, b, opt) {
- var aa = a.values();
- var bb = b.values();
-
- if (aa["node"] === "" && bb["node"] !== "") {
- return -1;
- }
- if (aa["node"] !== "" && bb["node"] === "") {
- return 1;
- }
-
- if (aa["node"] !== "" && bb["node"] !== "") {
- if (aa["node"] > bb["node"]) {
- return 1;
- }
- if (aa["node"] < bb["node"]) {
- return -1;
- }
- }
-
- if (aa["slot"] !== "" && bb["slot"] !== "") {
- if (aa["slot"] > bb["slot"]) {
- return 1;
- }
- if (aa["slot"] < bb["slot"]) {
- return -1;
- }
- }
-
- return sortById(a, b, opt);
-}
-
-
-function dumbPluralize(n, s, p) {
- if (typeof p === 'undefined') {
- p = "s";
- }
- if (n == 0 || n > 1) {
- return n + " " + (s + p);
- } else {
- return n + " " + s;
- }
-}
-
-function generateJobOverview(id, logViewer, taskState) {
- var html = "";
-
- if (logViewer.items.length > 2) {
- var first = logViewer.items[1];
- var last = logViewer.items[logViewer.items.length-1];
- var duration = (last.values().ts.getTime() - first.values().ts.getTime()) / 1000;
-
- var hours = 0;
- var minutes = 0;
- var seconds;
-
- if (duration >= 3600) {
- hours = Math.floor(duration / 3600);
- duration -= (hours * 3600);
- }
- if (duration >= 60) {
- minutes = Math.floor(duration / 60);
- duration -= (minutes * 60);
- }
- seconds = duration;
-
- var tcount = taskState.task_count;
-
- html += "<p>";
- html += "Started at " + first.values().timestamp + ". ";
- html += "Ran " + dumbPluralize(tcount, " task") + " over ";
- if (hours > 0) {
- html += dumbPluralize(hours, " hour");
- }
- if (minutes > 0) {
- html += " " + dumbPluralize(minutes, " minute");
- }
- if (seconds > 0) {
- html += " " + dumbPluralize(seconds, " second");
- }
-
- html += " using " + dumbPluralize(taskState.nodes.length, " node");
-
- html += ". " + dumbPluralize(taskState.complete_count, "task") + " completed";
- html += ", " + dumbPluralize(taskState.incomplete_count, "task") + " incomplete";
- html += " (" + dumbPluralize(taskState.failure_count, " failure") + ")";
-
- html += ". Finished at " + last.values().timestamp + ".";
- html += "</p>";
- } else {
- html = "<p>Job log is empty or failed to load.</p>";
- }
-
- $(id).html(html);
-}
-
function gotoPage(n, logViewer, page, id) {
if (n < 0) { return; }
if (n*page > logViewer.matchingItems.length) { return; }
@@ -280,3 +46,13 @@ function nextPage(logViewer, page, id) {
function prevPage(logViewer, page, id) {
gotoPage(logViewer.page_offset-1, logViewer, page, id);
}
+
+function addToLogViewer(logViewer, lines) {
+ var items = [];
+ for (var a in lines) {
+ items.push({
+ message: lines[a].replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>')
+ });
+ }
+ logViewer.add(items);
+}
diff --git a/apps/workbench/app/views/jobs/_show_log.html.erb b/apps/workbench/app/views/jobs/_show_log.html.erb
index 02ad2b7..448e04c 100644
--- a/apps/workbench/app/views/jobs/_show_log.html.erb
+++ b/apps/workbench/app/views/jobs/_show_log.html.erb
@@ -98,7 +98,7 @@ var makeFilter = function() {
return load_log();
}
logViewer.filter();
- addToLogViewer(logViewer, data.split("\n"), taskState);
+ addToJobLogViewer(logViewer, data.split("\n"), taskState);
logViewer.filter(makeFilter());
content_range_hdr = jqxhr.getResponseHeader('Content-Range');
var v = content_range_hdr && content_range_hdr.match(/bytes \d+-(\d+)\/(.+)/);
diff --git a/apps/workbench/app/views/work_units/_show_log.html.erb b/apps/workbench/app/views/work_units/_show_log.html.erb
index 6a0916f..f9b6241 100644
--- a/apps/workbench/app/views/work_units/_show_log.html.erb
+++ b/apps/workbench/app/views/work_units/_show_log.html.erb
@@ -9,16 +9,111 @@
</div>
<% end %>
-<%# Show log in terminal window %>
-<h4>Recent logs</h4>
-<div id="event_log_div"
- class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window"
- data-object-uuids="<%= wu.log_object_uuids.join(' ') %>"
- ><%= wu.live_log_lines(Rails.configuration.running_job_log_records_to_fetch).join("\n") %>
-</div>
-
-<%# Applying a long throttle suppresses the auto-refresh of this
- partial that would normally be triggered by arv-log-event. %>
-<div class="arv-log-refresh-control"
- data-load-throttle="86486400000" <%# 1001 nights %>>
-</div>
+<% live_log_lines = wu.live_log_lines(Rails.configuration.running_job_log_records_to_fetch).join("\n") %>
+<% if !render_log or (live_log_lines.size > 0) %>
+ <%# Still running, or recently finished and logs are still available from logs table %>
+ <%# Show recent logs in terminal window %>
+ <h4>Recent logs</h4>
+ <div id="event_log_div"
+ class="arv-log-event-listener arv-log-event-handler-append-logs arv-job-log-window"
+ data-object-uuids="<%= wu.log_object_uuids.join(' ') %>"
+ ><%= live_log_lines %>
+ </div>
+
+ <%# Applying a long throttle suppresses the auto-refresh of this
+ partial that would normally be triggered by arv-log-event. %>
+ <div class="arv-log-refresh-control"
+ data-load-throttle="86486400000" <%# 1001 nights %>>
+ </div>
+<% elsif render_log[:log] %>
+ <%# Retrieve finished log from keep and show %>
+ <script>
+ (function() {
+ var pagesize = 1000;
+ var logViewer = new List('log-viewer', {
+ valueNames: [ 'message'],
+ page: pagesize
+ });
+
+ logViewer.page_offset = 0;
+ logViewer.on("updated", function() { updatePaging(".log-viewer-paging", logViewer, pagesize) } );
+ $(".log-viewer-page-up").on("click", function() { prevPage(logViewer, pagesize, ".log-viewer-paging"); return false; });
+ $(".log-viewer-page-down").on("click", function() { nextPage(logViewer, pagesize, ".log-viewer-paging"); return false; });
+
+ <% logcollection = render_log[:log] %>
+ var log_size = <%= logcollection.files[0][2] %>
+ var log_maxbytes = <%= Rails.configuration.log_viewer_max_bytes %>;
+ var logcollection_url = '<%=j url_for logcollection %>/<%=j logcollection.files[0][1] %>';
+ var headers = {};
+ if (log_size > log_maxbytes) {
+ headers['Range'] = 'bytes=0-' + log_maxbytes;
+ }
+ var ajax_opts = { dataType: 'text', headers: headers };
+ load_log();
+
+ function load_log() {
+ $.ajax(logcollection_url, ajax_opts).done(done).fail(fail);
+ }
+ function done(data, status, jqxhr) {
+ if (jqxhr.getResponseHeader('Content-Type').indexOf('application/json') === 0) {
+ // The browser won't allow a redirect-with-cookie response
+ // because keep-web isn't same-origin with us. Instead, we
+ // assure keep-web it's OK to respond with the content
+ // immediately by setting the token in the request body
+ // instead and adding disposition=attachment.
+ logcollection_url = JSON.parse(data).href;
+ var queryAt = logcollection_url.indexOf('?api_token=');
+ if (queryAt >= 0) {
+ ajax_opts.method = 'POST';
+ ajax_opts.data = {
+ api_token: logcollection_url.slice(queryAt+11),
+ disposition: 'attachment',
+ };
+ logcollection_url = logcollection_url.slice(0, queryAt);
+ }
+ return load_log();
+ }
+ addToLogViewer(logViewer, data.split("\n"));
+ content_range_hdr = jqxhr.getResponseHeader('Content-Range');
+ var v = content_range_hdr && content_range_hdr.match(/bytes \d+-(\d+)\/(.+)/);
+ short_log = v && (v[2] == '*' || parseInt(v[1]) + 1 < v[2]);
+ if (jqxhr.status == 206 && short_log) {
+ $("#log-viewer-overview").html(
+ '<p>Showing only ' + data.length + ' bytes of this log.</p>'
+ );
+ }
+
+ $("#log-viewer .spinner").detach();
+ }
+ function fail(jqxhr, status, error) {
+ // TODO: tell the user about the error
+ $("#log-viewer .spinner").detach();
+ }
+ })();
+ </script>
+
+ <div id="log-viewer">
+ <p id="log-viewer-overview"></p>
+
+ <div class="h3">Log</div>
+ <div class="smart-scroll" data-smart-scroll-padding-bottom="50" style="margin-bottom: 0px">
+ <table class="log-viewer-table">
+ <tbody class="list">
+ <tr>
+ <td class="message"></td>
+ </tr>
+ </tbody>
+ </table>
+
+ <% if logcollection %>
+ <div class="spinner spinner-32px"></div>
+ <% end %>
+ </div>
+
+ <div class="log-viewer-paging-div" style="margin-bottom: -15px">
+ <a href="#" class="log-viewer-page-up"><span class='glyphicon glyphicon-arrow-up'></span></a>
+ <span class="log-viewer-paging"></span>
+ <a href="#" class="log-viewer-page-down"><span class='glyphicon glyphicon-arrow-down'></span></a>
+ </div>
+ </div>
+<% end %>
diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml
index 49a1fae..78d9fa7 100644
--- a/services/api/test/fixtures/container_requests.yml
+++ b/services/api/test/fixtures/container_requests.yml
@@ -184,7 +184,7 @@ cr_for_failed:
cwd: test
output_path: test
command: ["echo", "hello"]
- container_uuid: zzzzz-dz642-failedcntnr
+ container_uuid: zzzzz-dz642-failedcontainr1
runtime_constraints:
vcpus: 1
ram: 123
commit 878275d3bb35add83cdbe45c93bec353e9f8e7e8
Merge: e3de0a0 6802d84
Author: radhika <radhika at curoverse.com>
Date: Tue Sep 20 20:01:44 2016 -0400
Merge branch 'master' into 9514-delete-old-container-logs
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list