[ARVADOS] updated: 58a8ba13bfb03270a2cbcfd10b8e3865b5c76b36
Git user
git at public.curoverse.com
Thu Jul 21 11:44:21 EDT 2016
Summary of changes:
.../app/assets/javascripts/infinite_scroll.js | 1 +
.../app/controllers/application_controller.rb | 22 ++
.../app/controllers/projects_controller.rb | 16 +-
.../app/controllers/work_units_controller.rb | 37 +++
apps/workbench/app/models/pipeline_instance.rb | 2 +-
apps/workbench/app/models/proxy_work_unit.rb | 6 +
apps/workbench/app/models/work_unit.rb | 4 +
.../application/_delete_object_button.html.erb | 2 +-
.../views/container_requests/_show_log.html.erb | 2 +-
.../views/container_requests/_show_status.html.erb | 2 +-
.../app/views/containers/_show_log.html.erb | 2 +-
.../app/views/containers/_show_status.html.erb | 2 +-
.../workbench/app/views/jobs/_show_status.html.erb | 2 +-
.../pipeline_instances/_show_components.html.erb | 2 +-
.../app/views/projects/_show_dashboard.html.erb | 20 +-
apps/workbench/app/views/users/profile.html.erb | 6 +-
.../app/views/work_unit/_show_status.html.erb | 6 -
.../_component_detail.html.erb | 2 +-
.../{work_unit => work_units}/_progress.html.erb | 0
.../views/work_units/_show_all_processes.html.erb | 51 ++++
.../work_units/_show_all_processes_rows.html.erb | 23 ++
.../{work_unit => work_units}/_show_child.html.erb | 6 +-
.../_show_component.html.erb | 6 +-
.../{work_unit => work_units}/_show_log.html.erb | 0
.../app/views/work_units/_show_output.html.erb | 13 +
.../_show_outputs.html.erb | 0
.../app/views/work_units/_show_status.html.erb | 6 +
apps/workbench/app/views/work_units/index.html.erb | 1 +
.../workbench}/config/initializers/time_format.rb | 0
apps/workbench/config/routes.rb | 1 +
.../test/controllers/projects_controller_test.rb | 28 +-
.../test/controllers/work_units_controller_test.rb | 68 +++++
.../test/integration/application_layout_test.rb | 3 +-
.../test/integration/user_profile_test.rb | 2 +-
apps/workbench/test/integration/work_units_test.rb | 58 ++++
build/run-build-packages.sh | 6 +-
build/run-tests.sh | 13 +-
crunch_scripts/cwl-runner | 26 +-
doc/_config.yml | 9 +-
doc/_includes/_pipeline_deprecation_notice.liquid | 3 +
doc/sdk/cli/install.html.textile.liquid | 26 +-
doc/sdk/python/sdk-python.html.textile.liquid | 6 +-
doc/user/cwl/bwa-mem/bwa-mem-input-local.yml | 14 +
doc/user/cwl/bwa-mem/bwa-mem-input.yml | 14 +
doc/user/cwl/bwa-mem/bwa-mem-template.yml | 4 +
doc/user/cwl/bwa-mem/bwa-mem.cwl | 45 ++++
doc/user/cwl/cwl-runner.html.textile.liquid | 297 +++++++++++++++++++++
.../vm-login-with-webshell.html.textile.liquid | 2 +-
doc/user/topics/arv-copy.html.textile.liquid | 10 +-
.../crunch-tools-overview.html.textile.liquid | 2 +
...nning-pipeline-command-line.html.textile.liquid | 2 +-
.../topics/tutorial-parallel.html.textile.liquid | 2 +
.../running-external-program.html.textile.liquid | 4 +-
.../tutorial-firstscript.html.textile.liquid | 2 +
.../tutorial-submit-job.html.textile.liquid | 2 +
sdk/cwl/arvados_cwl/__init__.py | 9 +
sdk/cwl/arvados_cwl/arvjob.py | 8 +-
sdk/cwl/arvados_cwl/fsaccess.py | 1 -
sdk/cwl/arvados_cwl/pathmapper.py | 14 +
sdk/cwl/arvados_cwl/runner.py | 20 +-
sdk/cwl/setup.py | 2 +-
sdk/cwl/test_with_arvbox.sh | 13 +-
sdk/go/arvados/client.go | 12 +-
sdk/python/arvados/collection.py | 1 -
sdk/python/arvados/keep.py | 16 +-
sdk/python/tests/run_test_server.py | 13 +-
sdk/python/tests/test_keep_client.py | 4 +-
.../crunch-dispatch-slurm/crunch-dispatch-slurm.go | 5 +-
.../crunch-dispatch-slurm_test.go | 3 +-
services/keepstore/s3_volume.go | 31 ++-
services/keepstore/s3_volume_test.go | 47 ++--
tools/arvbox/bin/arvbox | 27 +-
tools/arvbox/lib/arvbox/docker/Dockerfile.demo | 8 +-
73 files changed, 949 insertions(+), 176 deletions(-)
create mode 100644 apps/workbench/app/controllers/work_units_controller.rb
delete mode 100644 apps/workbench/app/views/work_unit/_show_status.html.erb
rename apps/workbench/app/views/{work_unit => work_units}/_component_detail.html.erb (97%)
rename apps/workbench/app/views/{work_unit => work_units}/_progress.html.erb (100%)
create mode 100644 apps/workbench/app/views/work_units/_show_all_processes.html.erb
create mode 100644 apps/workbench/app/views/work_units/_show_all_processes_rows.html.erb
rename apps/workbench/app/views/{work_unit => work_units}/_show_child.html.erb (89%)
rename apps/workbench/app/views/{work_unit => work_units}/_show_component.html.erb (89%)
rename apps/workbench/app/views/{work_unit => work_units}/_show_log.html.erb (100%)
create mode 100644 apps/workbench/app/views/work_units/_show_output.html.erb
rename apps/workbench/app/views/{work_unit => work_units}/_show_outputs.html.erb (100%)
create mode 100644 apps/workbench/app/views/work_units/_show_status.html.erb
create mode 100644 apps/workbench/app/views/work_units/index.html.erb
copy {services/api => apps/workbench}/config/initializers/time_format.rb (100%)
create mode 100644 apps/workbench/test/controllers/work_units_controller_test.rb
create mode 100644 apps/workbench/test/integration/work_units_test.rb
create mode 100644 doc/_includes/_pipeline_deprecation_notice.liquid
create mode 100755 doc/user/cwl/bwa-mem/bwa-mem-input-local.yml
create mode 100755 doc/user/cwl/bwa-mem/bwa-mem-input.yml
create mode 100644 doc/user/cwl/bwa-mem/bwa-mem-template.yml
create mode 100755 doc/user/cwl/bwa-mem/bwa-mem.cwl
create mode 100644 doc/user/cwl/cwl-runner.html.textile.liquid
via 58a8ba13bfb03270a2cbcfd10b8e3865b5c76b36 (commit)
via e32bc8e5fd44845b7e098cb78605a8e15c687e24 (commit)
via 0f493521472fd70820ceb56e5d5375c66bc85262 (commit)
via d5f8e84b64d3ad294db793fb99f5b45c3250d5d3 (commit)
via 7926ff7028d577a1c77eab0c8bd94d3959146b1a (commit)
via e6cd7c31ee28851d3ead992437fc93f2fc73ef92 (commit)
via 4ed794e2803ef9812dbc7ae938a8e17b3b9714c1 (commit)
via 5b1176df5257cd16350fb4da5a6d6157c58a0a0f (commit)
via a78a5994fcbe41e1903a93f334d5a75e6e58db04 (commit)
via c0eb54eaffd1dac69f4ea73742a69a2473669538 (commit)
via 779c5400a4f376489f4abc8564f890f309ae3f20 (commit)
via df75e59337899044a55767e10f69717b520c3ead (commit)
via 5fd2ff4941d2cf6b06c082e9496a1b6ef723bf39 (commit)
via e41e6ac3f0039ecb50e762e58d85cc1fffdf6eda (commit)
via 30565edcd7effd719539939541bcb223b8d1bca3 (commit)
via c7609245998c63028b5fb84cf07752b49b735932 (commit)
via e94c1f08e958ed96d7cd153ea4668baed35961ed (commit)
via 38ad14c0cb5e400e66904a4af1c432f46df8a170 (commit)
via 42490568db4cf4bc65fc436b41cfcffb8eadd8d1 (commit)
via 7719a08f3cdcc3a7e3468a26e6f49c849061a42d (commit)
via 931236cc8c9bea9c796aed23c1636abce3ee4534 (commit)
via de4d6cb4a482d07fbeb2c73366e6717441a26e17 (commit)
via f02f9915ecccc36bc13ad7ae34567be26e55a339 (commit)
via 3a0aa1db801154916f50b1b299d5100945a3e1df (commit)
via 5cf55ab0cfe55cbd803b61736276044f7008bfa4 (commit)
via f17319c645b30aaf976000134a18433d3a8adb19 (commit)
via 077afbcc57d173a47ed1c799a480ba7728ee22bc (commit)
via 0d4726056080f8c865135e0f576ccc15d4f388b9 (commit)
via 1d40c96c8688cd6f74eab1c7dffa857d91be31c4 (commit)
via 72f92ad04b77bf781f27ff355b4cc91d7f36ec0f (commit)
via 7a041a2b0068e02bbb9743407d915522a1e2d1dc (commit)
via 133c1473a92a3f4d5d49c0b4e94a4656f913a3e8 (commit)
via 4e78269414737d1e693dad494e796ef46de1ea8a (commit)
via c211862a0f01b5073f59621ea3fe395d7f53390c (commit)
via 5605f8a67514037f5e7ca0e7780b23d6772317e4 (commit)
via ec2af1a4d13bbbdeedb83295d6177f7317cd8302 (commit)
via bb5430392e6af88d2a65fa060d9069630a4a7dc2 (commit)
via ea4ac61f1abd5ac3add332227751f22cc6eb329f (commit)
via eeddbb297de457834c943d16c49ee0a1192cc2a4 (commit)
via b827b267317419039e436dfa3b8c302b9ca095b8 (commit)
via 9e3cf124be0a38ae835ab934077c8febf4e2daba (commit)
via 353bc51027bdf085c8b638cbde9dbb38e1fd0262 (commit)
via cc79dd8213409bc57dd731687c9f67c9a84808bb (commit)
via 5603cefb38fbd2fc6493e8ccee587629e7af089e (commit)
via f3685f741ad7eca0fcaa2b11dc550f0ef5acd3ac (commit)
via 3f69ccbaaa9b9c16bd0ea2c063aa3a7aa91e6b84 (commit)
via 51b150baaa5148228156af65ccd6676e65935c41 (commit)
via 78baf73e784e01fb44d1b16aefaf6e295bb4d5b4 (commit)
from 90c663a9f4272db22532a07a6488a2ec4061284c (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 58a8ba13bfb03270a2cbcfd10b8e3865b5c76b36
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 21 11:44:17 2016 -0400
8555: Fix EmptyTrash so it can clean up old races.
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 5b00d4a..8ded160 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -564,14 +564,14 @@ func (v *S3Volume) EmptyTrash() {
if !v.isKeepBlock(loc) {
continue
}
- recent, err := v.Bucket.Head("recent/"+loc, nil)
+ trashT, err := time.Parse(time.RFC3339, trash.LastModified)
if err != nil {
- log.Printf("warning: %s: EmptyTrash: cannot delete trash %q with no corresponding recent/* marker", v, trash.Key)
+ log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err)
continue
}
- trashT, err := time.Parse(time.RFC3339, trash.LastModified)
+ recent, err := v.Bucket.Head("recent/"+loc, nil)
if err != nil {
- log.Printf("warning: %s: EmptyTrash: %q: parse %q: %s", v, trash.Key, trash.LastModified, err)
+ log.Printf("warning: %s: EmptyTrash: cannot delete trash %q with no corresponding recent/* marker", v, trash.Key)
continue
}
recentT, err := v.lastModified(recent)
@@ -580,8 +580,27 @@ func (v *S3Volume) EmptyTrash() {
continue
}
if trashT.Sub(recentT) < blobSignatureTTL {
- v.fixRace(loc)
- continue
+ if age := now.Sub(recentT); age >= blobSignatureTTL - v.raceWindow {
+ // recent/loc is too old to protect
+ // loc from being Trashed again during
+ // the raceWindow that starts if we
+ // delete trash/X now.
+ //
+ // Note this means (trashCheckInterval
+ // < blobSignatureTTL - raceWindow) is
+ // necessary to avoid starvation.
+ log.Printf("notice: %s: EmptyTrash: detected old race for %q, calling fixRace + Touch", v, loc)
+ v.fixRace(loc)
+ v.Touch(loc)
+ continue
+ } else if _, err := v.Bucket.Head(loc, nil); os.IsNotExist(err) {
+ log.Printf("notice: %s: EmptyTrash: detected recent race for %q, calling fixRace", v, loc)
+ v.fixRace(loc)
+ continue
+ } else if err != nil {
+ log.Printf("warning: %s: EmptyTrash: HEAD %q: %s", v, loc, err)
+ continue
+ }
}
if now.Sub(trashT) < trashLifetime {
continue
diff --git a/services/keepstore/s3_volume_test.go b/services/keepstore/s3_volume_test.go
index a40b84f..59402dc 100644
--- a/services/keepstore/s3_volume_test.go
+++ b/services/keepstore/s3_volume_test.go
@@ -142,11 +142,12 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
canGetAfterTrash bool
canUntrash bool
haveTrashAfterEmpty bool
+ freshAfterEmpty bool
}{
{
"No related objects",
none, none, none,
- false, false, false, false, false},
+ false, false, false, false, false, false},
{
// Stored by older version, or there was a
// race between EmptyTrash and Put: Trash is a
@@ -154,56 +155,59 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
// old
"No recent/X",
t0.Add(-48 * time.Hour), none, none,
- true, true, true, false, false},
+ true, true, true, false, false, false},
{
"Not trash; old enough to trash",
t0.Add(-24 * time.Hour), t0.Add(-2 * time.Hour), none,
- true, true, false, false, false},
+ true, true, false, false, false, false},
{
"Not trash; not old enough to trash",
t0.Add(-24 * time.Hour), t0.Add(-30 * time.Minute), none,
- true, true, true, false, false},
+ true, true, true, false, false, false},
{
"Trash + not-trash: recent race between Trash and Put",
t0.Add(-24 * time.Hour), t0.Add(-3 * time.Minute), t0.Add(-2 * time.Minute),
- true, true, true, true, true},
+ true, true, true, true, true, false},
{
"Trash + not-trash, nearly eligible for deletion, prone to Trash race",
t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
- true, false, true, true, true},
+ true, false, true, true, true, false},
{
"Trash + not-trash, eligible for deletion, prone to Trash race",
t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-61 * time.Minute),
- true, false, true, true, false},
- // FIXME: old trash never gets deleted!
- // {
- // "Not trash; old race between Trash and Put, or incomplete Trash",
- // t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour),
- // true, false, true, true, false},
+ true, false, true, true, false, false},
+ {
+ "Trash + not-trash, unsafe to empty; old race between Put and unfinished Trash",
+ t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour), t0.Add(-12 * time.Hour),
+ true, false, true, true, true, true},
+ {
+ "Trash + not-trash, was unsafe to empty, but since made safe by fixRace+Touch",
+ t0.Add(-time.Second), t0.Add(-time.Second), t0.Add(-12 * time.Hour),
+ true, true, true, true, false, false},
{
"Trash operation was interrupted",
t0.Add(-24 * time.Hour), t0.Add(-24 * time.Hour), t0.Add(-12 * time.Hour),
- true, false, true, true, false},
+ true, false, true, true, false, false},
{
"Trash, not yet eligible for deletion",
none, t0.Add(-12 * time.Hour), t0.Add(-time.Minute),
- false, false, false, true, true},
+ false, false, false, true, true, false},
{
"Trash, not yet eligible for deletion, prone to races",
none, t0.Add(-12 * time.Hour), t0.Add(-59 * time.Minute),
- false, false, false, true, true},
+ false, false, false, true, true, false},
{
"Trash, eligible for deletion",
none, t0.Add(-12 * time.Hour), t0.Add(-2 * time.Hour),
- false, false, false, true, false},
+ false, false, false, true, false, false},
{
"Erroneously trashed during a race, detected before trashLifetime",
none, t0.Add(-30 * time.Minute), t0.Add(-29 * time.Minute),
- true, false, true, true, true},
+ true, false, true, true, true, false},
{
"Erroneously trashed during a race, rescue during EmptyTrash despite reaching trashLifetime",
none, t0.Add(-90 * time.Minute), t0.Add(-89 * time.Minute),
- true, false, true, true, true},
+ true, false, true, true, true, false},
} {
c.Log("Scenario: ", test.label)
var loc string
@@ -245,6 +249,13 @@ func (s *StubbedS3Suite) TestBackendStates(c *check.C) {
v.EmptyTrash()
_, err = v.Bucket.Head("trash/"+loc, nil)
c.Check(err == nil, check.Equals, test.haveTrashAfterEmpty)
+ if test.freshAfterEmpty {
+ t, err := v.Mtime(loc)
+ c.Check(err, check.IsNil)
+ // new mtime must be current (with an
+ // allowance for 1s timestamp precision)
+ c.Check(t.After(t0.Add(-time.Second)), check.Equals, true)
+ }
}
}
commit e32bc8e5fd44845b7e098cb78605a8e15c687e24
Merge: 90c663a 0f49352
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 21 09:21:09 2016 -0400
Merge branch 'master' into 8555-s3-trash
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list