[ARVADOS] updated: d1c72b300d50f071106a320a807f465c251a812e
git at public.curoverse.com
git at public.curoverse.com
Wed Feb 10 10:00:48 EST 2016
Summary of changes:
apps/workbench/Gemfile.lock | 3 -
apps/workbench/app/models/arvados_api_client.rb | 5 +-
backports/python-llfuse/fpm-info.sh | 3 +
crunch_scripts/crunchutil/subst.py | 2 +-
crunch_scripts/run-command | 17 +-
doc/_includes/_install_git.liquid | 8 +-
doc/_includes/_install_redhat_postgres_auth.liquid | 11 +
doc/_includes/_install_ruby_and_bundler.liquid | 2 +-
doc/_includes/_install_runit.liquid | 13 +
doc/install/install-api-server.html.textile.liquid | 27 +-
.../install-arv-git-httpd.html.textile.liquid | 17 +-
.../install-compute-node.html.textile.liquid | 21 +-
.../install-crunch-dispatch.html.textile.liquid | 22 +-
doc/install/install-keep-web.html.textile.liquid | 4 +-
doc/install/install-keepproxy.html.textile.liquid | 6 +-
doc/install/install-keepstore.html.textile.liquid | 4 +-
.../install-shell-server.html.textile.liquid | 16 +-
doc/install/install-sso.html.textile.liquid | 15 +-
.../install-workbench-app.html.textile.liquid | 15 +-
sdk/cli/bin/crunch-job | 2 +-
sdk/python/arvados/events.py | 63 ++--
services/api/Gemfile | 2 +-
services/api/Gemfile.lock | 4 +-
.../app/controllers/arvados/v1/nodes_controller.rb | 2 +-
services/api/app/models/arvados_model.rb | 4 +
services/api/fpm-info.sh | 2 +-
services/api/lib/crunch_dispatch.rb | 5 +-
services/api/lib/load_param.rb | 26 +-
services/api/test/fixtures/nodes.yml | 11 +
.../functional/arvados/v1/nodes_controller_test.rb | 38 +++
.../api/test/functional/arvados/v1/query_test.rb | 68 +++++
services/crunchstat/crunchstat.go | 4 +-
services/fuse/arvados_fuse/command.py | 106 ++++---
services/fuse/arvados_fuse/fusedir.py | 24 ++
services/fuse/fpm-info.sh | 1 +
services/fuse/setup.py | 2 +-
services/fuse/tests/integration_test.py | 4 +-
services/fuse/tests/mount_test_base.py | 31 +-
services/fuse/tests/test_command_args.py | 28 +-
services/fuse/tests/test_exec.py | 60 ++++
services/keepstore/azure_blob_volume.go | 19 +-
services/keepstore/handler_test.go | 103 +++++++
services/keepstore/handlers.go | 53 +++-
services/keepstore/keepstore.go | 10 +
services/keepstore/s3_volume.go | 13 +-
services/keepstore/trash_worker.go | 2 +-
services/keepstore/volume.go | 18 +-
services/keepstore/volume_generic_test.go | 8 +-
services/keepstore/volume_test.go | 7 +-
services/keepstore/volume_unix.go | 14 +-
services/keepstore/volume_unix_test.go | 2 +-
services/login-sync/bin/arvados-login-sync | 3 +-
services/nodemanager/arvnodeman/clientactor.py | 20 +-
.../nodemanager/arvnodeman/computenode/__init__.py | 64 ++++
.../arvnodeman/computenode/dispatch/__init__.py | 159 +++++-----
.../arvnodeman/computenode/dispatch/slurm.py | 8 +-
.../arvnodeman/computenode/driver/__init__.py | 56 +++-
.../arvnodeman/computenode/driver/ec2.py | 3 +
.../arvnodeman/computenode/driver/gce.py | 1 +
services/nodemanager/arvnodeman/daemon.py | 103 +++++--
services/nodemanager/arvnodeman/jobqueue.py | 3 +-
services/nodemanager/arvnodeman/launcher.py | 56 ++--
services/nodemanager/arvnodeman/nodelist.py | 5 +-
services/nodemanager/tests/test_clientactor.py | 4 +-
.../nodemanager/tests/test_computenode_dispatch.py | 21 +-
.../tests/test_computenode_driver_azure.py | 10 +
.../tests/test_computenode_driver_gce.py | 10 +
services/nodemanager/tests/test_daemon.py | 1 +
services/nodemanager/tests/testutil.py | 8 +
tools/crunchstat-summary/MANIFEST.in | 1 +
tools/crunchstat-summary/bin/crunchstat-summary | 9 +-
.../crunchstat_summary/__init__.py | 4 +
.../crunchstat_summary/chartjs.js | 27 ++
.../crunchstat_summary/chartjs.py | 70 +++++
.../crunchstat_summary/command.py | 50 +++-
.../crunchstat_summary/reader.py | 69 +++++
.../crunchstat_summary/summarizer.py | 332 ++++++++++++++++++---
tools/crunchstat-summary/setup.py | 2 +
.../tests/crunchstat_error_messages.txt | 9 +
.../tests/logfile_20151204190335.txt.gz.report | 3 +
.../tests/logfile_20151210063411.txt.gz.report | 2 +
.../tests/logfile_20151210063439.txt.gz.report | 2 +
tools/crunchstat-summary/tests/test_examples.py | 158 +++++++++-
83 files changed, 1781 insertions(+), 439 deletions(-)
create mode 100644 doc/_includes/_install_redhat_postgres_auth.liquid
create mode 100644 doc/_includes/_install_runit.liquid
create mode 100644 services/api/test/functional/arvados/v1/query_test.rb
create mode 100644 services/fuse/fpm-info.sh
create mode 100644 services/fuse/tests/test_exec.py
create mode 100644 tools/crunchstat-summary/crunchstat_summary/chartjs.js
create mode 100644 tools/crunchstat-summary/crunchstat_summary/chartjs.py
create mode 100644 tools/crunchstat-summary/crunchstat_summary/reader.py
create mode 100644 tools/crunchstat-summary/tests/crunchstat_error_messages.txt
discards f9d868590efb81cee078d19d3be91ef297634499 (commit)
via d1c72b300d50f071106a320a807f465c251a812e (commit)
via baeb7dbe5929012dea22985b11ae4c5584f76891 (commit)
via 19199a75e41004ea776622c305c3ca43e5367bf2 (commit)
via ddd02b0a536cbc0e80b77df43939e30f294126b5 (commit)
via 090b515aa10dba597b20a96797f17688f582a529 (commit)
via fcbb743e3de63e93280f2fbeedea49f98430d26f (commit)
via e8b7fbbd711836c59824327364e15ef2253a5848 (commit)
via 1fc6d7713baabfe85b49191e156b6c093d22b69f (commit)
via d4807ea755bcd122388189815b0b6d5aaa295509 (commit)
via 3f5b70e78c4008e0de876e53fc7fbbc0671a6937 (commit)
via 1293a55ede38d16119b04a6584ca9da143ea3725 (commit)
via 6c950260b253d28e78579b493c8eb3eabe0add11 (commit)
via d7e8f7c787b7706937f95c3ed2a5086616d48514 (commit)
via 3e7037e6383f57b0d1b4b627cd9feb27f05af13d (commit)
via 5bda5bf3aedc0621abeed901a608adb7db6030e6 (commit)
via f6bb371a170d0a74db6abf9df0f65aabe08d7cf9 (commit)
via a13c14cfc7fabe4f4da48edd57a086d2d8953a03 (commit)
via 78b9f61dcab9bd571952b8c9e8052b643588c52b (commit)
via 66da04e142bb80ff25d65900292bf99c7e4252f2 (commit)
via 4186924dbc103d080c47ae85bcfcd38d87314767 (commit)
via 48369a1e78683cec26b7154d5edc5f3450fec245 (commit)
via 3a2802166901386eba4c3bc9bf877a18346ba03c (commit)
via b63c1041dd34fe496920fd035249e50edc88b095 (commit)
via 45f37fe38bd3cb7ca1ff9a7c47791ec73888f383 (commit)
via 8417bf754051966915b93c362624ebdcd492d660 (commit)
via 64e0a60330d73309c02e75d012249a28a32e1b9d (commit)
via 0d6070d3b62fc367dbb02e8ead5fe25dce012156 (commit)
via ea5a4de11a0f379c6fa78be288bc403da53995a1 (commit)
via 8709ef45c968e79b5e5978759484b994d4789aea (commit)
via fd5a11b21af83870c4122afdf94844dd4cf63cc6 (commit)
via cda9a7d3e3601a7a8bb721261c5b26210e9f629f (commit)
via ca4e92183dfca9f226898790a09239935e17eba9 (commit)
via be2197781a206ba7cbf5bbfbf61a70375fb10d68 (commit)
via e9901a3f94eab93f9482b7cb3d7e2c1b50216f08 (commit)
via 73812c5a32581a7f72922dcd2e448e4d4ba4d918 (commit)
via e0e0c312ddf1f60736cc1ca7488bb36f8618440c (commit)
via c7e3518e66369b931821b802160bd014cf82b218 (commit)
via ce8f8ab963a12ca7f89c79cb7d6f5d19fe15b03c (commit)
via a3dfad4606eb8cff8bb66533a38c47d0d8c158fa (commit)
via 1dbf2e031ecbe21aefebe8b297fb5d5e222234a7 (commit)
via 48f61a5af0d0e6617ea545d716c5e0b132790198 (commit)
via b26951c6f488f28cf1ea5aac65c65311d0b573a4 (commit)
via 95773bc4ac9e01126a80a96fdbe0bf37d1478767 (commit)
via 4ede474fb3be39474fdd2a075d81dd351e6afefc (commit)
via e9dcedd8dda3de8360e34bfca536dd606d558d34 (commit)
via b983edff6a9053b686275af980ce336283fd3c27 (commit)
via dd6bbea1497114eed99285753bc163972717dfbb (commit)
via aea45ff7ad85ec23a903faf16386d43e0d614f83 (commit)
via 6570eec0115d7973cce4df10857631cfe6bd11c5 (commit)
via 24d4409c087eeebe67ba54af7a6c19745a60365b (commit)
via 6d349aac4810e7b0039e519860d0eda8345c9dfe (commit)
via 40103b455d0df2bc47eaba422a72bf6ec4fd6808 (commit)
via 7b4f98f54598aefc54f88a491dba64793cf749ae (commit)
via 0171406e1dda9662e47c19ffabdd4eb66dd2fc61 (commit)
via b445db93ece1069c949f0c02d0564e578e453d12 (commit)
via 179b6580522095c6e190bd000c71002fc6564a2e (commit)
via 630eac929f942fa997a4efcb9e7cd2d88414d0c2 (commit)
via a9335a762f70e30affdb259e2ff487f27963f1c8 (commit)
via 9416b3013a959ef51f12829023d12986a234e029 (commit)
via 9024a5b5eb247e3a552dbb92a61f7c916c0d4349 (commit)
via 9eb25cddc8bf1d9768d0bb9dae71ac91754e3480 (commit)
via bbd84f0a5594fdbf21a967572d4c2b33d7987f2b (commit)
via 678e366229d4082d5977c7dadf94224d5d834fb7 (commit)
via 4abdedc4baddd6ed80b180b8820fba19068c1c3a (commit)
via df0f52924d6e843bcdbecd96a9b398b825b61a85 (commit)
via efe49059b1cdfc9f91504bcbdc96656b4b844839 (commit)
via c174caa6acbfb4927631160da728ec4d1f8ad410 (commit)
via 0c3ec9229478a836d6e8fe86369cf4dffdb58183 (commit)
via 00a94def60fcf314e4a8cf525f82ca4f9952ed78 (commit)
via cc79feaa11ec7436bf5f94ac9e3db4ed76bdc4c9 (commit)
via bb7d7bd789d2fb71bc383fd33c74d084f6a82120 (commit)
via 2174d59b367e5b2000472cf8b521a155b952b543 (commit)
via daa3cc42efa55ad17df2a151aebaac515d10cc52 (commit)
via ce669fb969b7669dc1f672214f96404fcd141455 (commit)
via 0d4bee15a0e69b08799bff869eb3f1ca122f107f (commit)
via b32e371420fff5763c13c79cd4327692e2a2e1bd (commit)
via 7faa4aa742a7562c75e2fc2cabc7bf2ea8e856a8 (commit)
via 85c71a36173550f14fb1d5f4092f2050ec8dc033 (commit)
via fc3b155fb7ad82cf97cdb81a02a9cfaebe4389ff (commit)
via a4b58ddc7b8ec80010dfa30dc6c4ac1ce69858da (commit)
via 9aa6066eef7a67809f33871447d4312af6f22749 (commit)
via fdd28d6f009c605c61f4444dc2d9d142c3a1d395 (commit)
via d9ab654450fd23c043beab5c19c99806a301e51c (commit)
via 56922b056a746e79c53b43186dbeff7e8a856546 (commit)
via e001d5f3e5cece6429c9b1305f98fdad89f451d0 (commit)
via b59cedb067aa5cf27045d1ad15fe9cea49502520 (commit)
via aa720c29abcf039a965231decb6a40d00e479437 (commit)
via 72e1f8eebd7eb9ad5f6b4d0923f06cfc2b57b9d4 (commit)
via 12b1501e8ee2edc932ceb908ab69189712a761b4 (commit)
via 3b31f110db82c93c3aade294f50bbb0218c74697 (commit)
via 6ac861fd4a44cb39371a66d96b77ad7a41c85d3b (commit)
via 33e39bb6053e1cf5bf849a29e41fb74b31a22ef7 (commit)
via f50626706f9177137472fcd37472d3437b7c7a57 (commit)
via 2b6837ef2b57204b7bf4fd5c4373283892999268 (commit)
via a4d0d55ac333e8060d13a600d8aa8f5443760b29 (commit)
via 2a6e0785f8fb675038f867aebc65033dc8a85211 (commit)
via 0d3411abc645a8cd6fef4b960070ecf054ac16f2 (commit)
via 20fe67d073424f5c277fbd13557ffe5ae2b15fd9 (commit)
via 0343fe988fdc50bdfb1ac34204c1a4998fc7c446 (commit)
via 4b78fb11974d8bcb0b9e4ecd0162d6a938026c73 (commit)
via b8fbd409f84d647726a25ee7fd2d73660a2bfa82 (commit)
via d9b556e32bd91635a57d5f7ab0844eb3ad12de2b (commit)
via 87a3cc47267fad4cc769d9d866754c072a83c35e (commit)
via a82be0d9ab0438fa778e80c0aea3a07e8ba6a929 (commit)
This update added new revisions after undoing existing revisions. That is
to say, the old revision is not a strict subset of the new revision. This
situation occurs when you --force push a change and generate a repository
containing something like this:
* -- * -- B -- O -- O -- O (f9d868590efb81cee078d19d3be91ef297634499)
\
N -- N -- N (d1c72b300d50f071106a320a807f465c251a812e)
When this happens we assume that you've already had alert emails for all
of the O revisions, and so we here report only the revisions in the N
branch from the common base, B.
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d1c72b300d50f071106a320a807f465c251a812e
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jan 22 15:02:21 2016 -0500
7263: Avoid getting stuck processing stderr for one task for a long time.
Do not sleep(0.1) unless pipes are idle.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index baaf795..f153ec8 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1057,12 +1057,14 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
check_refresh_wanted();
check_squeue();
update_progress_stats();
- select (undef, undef, undef, 0.1);
}
elsif (time - $progress_stats_updated >= 30 || $progress_is_dirty)
{
update_progress_stats();
}
+ if (!$gotsome) {
+ select (undef, undef, undef, 0.1);
+ }
$working_slot_count = scalar(grep { $_->{node}->{fail_count} == 0 &&
$_->{node}->{hold_count} < 4 } @slot);
if (($thisround_failed_multiple >= 8 && $thisround_succeeded == 0) ||
@@ -1432,15 +1434,21 @@ sub readfrompipes
foreach my $job (keys %reader)
{
my $buf;
- while (0 < sysread ($reader{$job}, $buf, 8192))
+ if (0 < sysread ($reader{$job}, $buf, 65536))
{
print STDERR $buf if $ENV{CRUNCH_DEBUG};
$jobstep[$job]->{stderr_at} = time;
$jobstep[$job]->{stderr} .= $buf;
+
+ # Consume everything up to the last \n
preprocess_stderr ($job);
+
if (length ($jobstep[$job]->{stderr}) > 16384)
{
- substr ($jobstep[$job]->{stderr}, 0, 8192) = "";
+ # If we get a lot of stderr without a newline, chop off the
+ # front to avoid letting our buffer grow indefinitely.
+ substr ($jobstep[$job]->{stderr},
+ 0, length($jobstep[$job]->{stderr}) - 8192) = "";
}
$gotsome = 1;
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list