[ARVADOS] updated: 1.2.0-114-g3596aff09
Git user
git at public.curoverse.com
Tue Sep 25 14:51:47 EDT 2018
Summary of changes:
build/package-build-dockerfiles/Makefile | 6 +-
build/package-build-dockerfiles/centos7/Dockerfile | 3 +-
build/package-build-dockerfiles/debian8/Dockerfile | 3 +-
.../package-build-dockerfiles/debian9/D39DC0E3.asc | 692 ------------
build/package-build-dockerfiles/debian9/Dockerfile | 4 +-
.../ubuntu1404/Dockerfile | 3 +-
.../ubuntu1604/Dockerfile | 3 +-
.../ubuntu1804/Dockerfile | 3 +-
build/run-build-packages.sh | 9 +-
doc/_config.yml | 1 +
doc/architecture/Arvados_federation.odg | Bin 0 -> 16162 bytes
doc/architecture/federation.html.textile.liquid | 111 ++
doc/images/arvados_federation.svg | 1133 ++++++++++++++++++++
doc/install/install-api-server.html.textile.liquid | 39 +-
lib/controller/federation.go | 152 ++-
lib/controller/federation_test.go | 195 +++-
lib/controller/proxy.go | 9 +-
sdk/go/arvados/container.go | 43 +
sdk/go/arvados/fs_collection.go | 30 +-
sdk/go/arvados/fs_collection_test.go | 14 +-
sdk/go/arvadostest/fixtures.go | 3 +-
.../api/app/controllers/application_controller.rb | 8 +-
services/api/app/models/container.rb | 14 +
services/api/app/models/container_request.rb | 49 +-
services/api/config/application.default.yml | 7 +
.../api/config/initializers/legacy_jobs_api.rb | 16 +
services/api/lib/enable_jobs_api.rb | 39 +
services/api/test/unit/container_test.rb | 42 +-
services/api/test/unit/job_test.rb | 28 +
29 files changed, 1820 insertions(+), 839 deletions(-)
delete mode 100644 build/package-build-dockerfiles/debian9/D39DC0E3.asc
create mode 100644 doc/architecture/Arvados_federation.odg
create mode 100644 doc/architecture/federation.html.textile.liquid
create mode 100644 doc/images/arvados_federation.svg
create mode 100644 services/api/config/initializers/legacy_jobs_api.rb
create mode 100644 services/api/lib/enable_jobs_api.rb
via 3596aff0954f405b06799814585d834502d0d76a (commit)
via a1e2cc595d148ea36f0f65c6750650a5eb034405 (commit)
via 4058b87cfdc07fe25da31f59577f835656e97816 (commit)
via 959fc42ba11fbe4f803511ffcb8945bae537ff21 (commit)
via 4edb33f16f8f111fa478dda908c37ea2a5723dd9 (commit)
via 27d98c4c36f840686d5b7414500c7b006e80c114 (commit)
via 1dd4997ebfe9e1bce4073aed9abb37f89ae90845 (commit)
via b23972dec86c459c2e89fbb0c021153db54e7efa (commit)
via a2d7b1241424ea22e5ee81818d6562f92ee2731d (commit)
via a629657158ed8690e9c4d44acea47bf0d8ba3794 (commit)
via ab92b051d6f09f1b3e670ecd39e8c96ed34570db (commit)
via a0789339c1fa67ab8b3f71b4fa6a0087165c078d (commit)
via 49e18f8c16bda6b6bf56c301b440c9d56d9bcb72 (commit)
via a78dce7bd57c1a6822e1949be4a5f13cf2447909 (commit)
via ba54c26be4078cb32d78947837f630a4404a93ae (commit)
via 494cc8312cff496bab3653d5b2ab1f00ac43a40c (commit)
via 720e76bb1d82d5a5448ce395df634310ceee473e (commit)
via b5f589dce8110139073388f11f1ed2259e272a96 (commit)
via 985bfe119f34bbf39a7007bee1fc1c03db9ab8bf (commit)
via 9b1e23489c659655134a7e208a012d5d8d05bd07 (commit)
via 898cd195f8b5916ed9c3a83dec212e858bfa610f (commit)
via 5d8eb6ead9e2ca32f424a8385979485a902cf09f (commit)
via 7d8906675774ce1c176b3f17c47f64ab43eea5cb (commit)
from e7b3a06c0176f74082db9b436f773af2003cb824 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3596aff0954f405b06799814585d834502d0d76a
Merge: a1e2cc595 4058b87cf
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 25 14:51:32 2018 -0400
10181: Merge branch 'master' into 10181-incremental-log
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
commit a1e2cc595d148ea36f0f65c6750650a5eb034405
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue Sep 25 14:50:44 2018 -0400
10181: Keep CR logs synchronized with container logs while running.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index e8a70499a..3bb9cbf8e 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -39,6 +39,7 @@ class Container < ArvadosModel
before_save :update_secret_mounts_md5
before_save :scrub_secret_mounts
before_save :clear_runtime_status_when_queued
+ after_save :update_cr_logs
after_save :handle_completed
after_save :propagate_priority
after_commit { UpdatePriority.run_update_thread }
@@ -495,6 +496,19 @@ class Container < ArvadosModel
end
end
+ def update_cr_logs
+ # If self.final?, this update is superfluous: the final log/output
+ # update will be done when handle_completed calls finalize! on
+ # each requesting CR.
+ return if self.final? || !self.log_changed?
+ leave_modified_by_user_alone do
+ ContainerRequest.where(container_uuid: self.uuid).each do |cr|
+ cr.update_collections(container: self, collections: ['log'])
+ cr.save!
+ end
+ end
+ end
+
def assign_auth
if self.auth_uuid_changed?
return errors.add :auth_uuid, 'is readonly'
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index f430ae918..bbec42108 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -123,11 +123,13 @@ class ContainerRequest < ArvadosModel
# Finalize the container request after the container has
# finished/cancelled.
def finalize!
- out_coll = nil
- log_coll = nil
- c = Container.find_by_uuid(container_uuid)
- ['output', 'log'].each do |out_type|
- pdh = c.send(out_type)
+ update_collections(container: Container.find_by_uuid(container_uuid))
+ update_attributes!(state: Final)
+ end
+
+ def update_collections(container:, collections: ['log', 'output'])
+ collections.each do |out_type|
+ pdh = container.send(out_type)
next if pdh.nil?
coll_name = "Container #{out_type} for request #{uuid}"
trash_at = nil
@@ -141,24 +143,25 @@ class ContainerRequest < ArvadosModel
end
manifest = Collection.where(portable_data_hash: pdh).first.manifest_text
- coll = Collection.new(owner_uuid: owner_uuid,
- manifest_text: manifest,
- portable_data_hash: pdh,
- name: coll_name,
- trash_at: trash_at,
- delete_at: trash_at,
- properties: {
- 'type' => out_type,
- 'container_request' => uuid,
- })
- coll.save_with_unique_name!
- if out_type == 'output'
- out_coll = coll.uuid
- else
- log_coll = coll.uuid
+ coll_uuid = self.send(out_type + '_uuid')
+ coll = coll_uuid.nil? ? nil : Collection.where(uuid: coll_uuid).first
+ if !coll
+ coll = Collection.new(
+ owner_uuid: self.owner_uuid,
+ name: coll_name,
+ properties: {
+ 'type' => out_type,
+ 'container_request' => uuid,
+ })
end
+ coll.assign_attributes(
+ portable_data_hash: pdh,
+ manifest_text: manifest,
+ trash_at: trash_at,
+ delete_at: trash_at)
+ coll.save_with_unique_name!
+ self.send(out_type + '_uuid=', coll.uuid)
end
- update_attributes!(state: Final, output_uuid: out_coll, log_uuid: log_coll)
end
def self.full_text_searchable_columns
@@ -312,6 +315,10 @@ class ContainerRequest < ArvadosModel
permitted.push :container_count
end
+ if current_user.andand.is_admin
+ permitted.push :log_uuid
+ end
+
when Final
if self.state_was == Committed
# "Cancel" means setting priority=0, state=Committed
diff --git a/services/api/test/unit/container_test.rb b/services/api/test/unit/container_test.rb
index 0c5e2e7ad..03e7850a5 100644
--- a/services/api/test/unit/container_test.rb
+++ b/services/api/test/unit/container_test.rb
@@ -653,22 +653,48 @@ class ContainerTest < ActiveSupport::TestCase
end
test "locked_by_uuid can update log when locked/running, and output when running" do
- c, _ = minimal_new
+ logcoll = collections(:real_log_collection)
+ c, cr1 = minimal_new
+ cr2 = ContainerRequest.new(DEFAULT_ATTRS)
+ cr2.state = ContainerRequest::Committed
+ act_as_user users(:active) do
+ cr2.save!
+ end
+ assert_equal cr1.container_uuid, cr2.container_uuid
+
+ logpdh_time1 = logcoll.portable_data_hash
+
set_user_from_auth :dispatch1
c.lock
assert_equal c.locked_by_uuid, Thread.current[:api_client_authorization].uuid
- assert c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
- assert c.update_attributes(state: Container::Running)
+ c.update_attributes!(log: logpdh_time1)
+ c.update_attributes!(state: Container::Running)
+ cr1.reload
+ cr2.reload
+ cr1log_uuid = cr1.log_uuid
+ cr2log_uuid = cr2.log_uuid
+ assert_not_nil cr1log_uuid
+ assert_not_nil cr2log_uuid
+ assert_not_equal logcoll.uuid, cr1log_uuid
+ assert_not_equal logcoll.uuid, cr2log_uuid
+ assert_not_equal cr1log_uuid, cr2log_uuid
+
+ logcoll.update_attributes!(manifest_text: logcoll.manifest_text + ". acbd18db4cc2f85cedef654fccc4a4d8+3 0:3:foo.txt\n")
+ logpdh_time2 = logcoll.portable_data_hash
assert c.update_attributes(output: collections(:collection_owned_by_active).portable_data_hash)
- assert c.update_attributes(log: nil)
- assert c.update_attributes(log: collections(:real_log_collection).portable_data_hash)
- assert c.update_attributes(state: Container::Complete, log: collections(:real_log_collection).portable_data_hash)
+ assert c.update_attributes(log: logpdh_time2)
+ assert c.update_attributes(state: Container::Complete, log: logcoll.portable_data_hash)
c.reload
- assert_equal c.output, collections(:collection_owned_by_active).portable_data_hash
- assert_equal c.log, collections(:real_log_collection).portable_data_hash
+ assert_equal collections(:collection_owned_by_active).portable_data_hash, c.output
+ assert_equal logpdh_time2, c.log
refute c.update_attributes(output: nil)
refute c.update_attributes(log: nil)
+ cr1.reload
+ cr2.reload
+ assert_equal cr1log_uuid, cr1.log_uuid
+ assert_equal cr2log_uuid, cr2.log_uuid
+ assert_equal [logpdh_time2], Collection.where(uuid: [cr1log_uuid, cr2log_uuid]).to_a.collect(&:portable_data_hash).uniq
end
test "auth_uuid can set output, progress on running container -- but not state, log" do
commit 27d98c4c36f840686d5b7414500c7b006e80c114
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Sat Sep 22 02:07:22 2018 -0400
10181: Fix race in MarshalManifest.
Ensure child nodes stay locked between sync and marshal. Otherwise
concurrent file writes can add new memSegments that can't be
marshaled.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index bcf2f4810..1fe4d8f56 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -548,9 +548,10 @@ func (dn *dirnode) Child(name string, replace func(inode) (inode, error)) (inode
return dn.treenode.Child(name, replace)
}
-// sync flushes in-memory data (for all files in the tree rooted at
-// dn) to persistent storage. Caller must hold dn.Lock().
-func (dn *dirnode) sync() error {
+// sync flushes in-memory data (for the children with the given names,
+// which must be children of dn) to persistent storage. Caller must
+// have write lock on dn and the named children.
+func (dn *dirnode) sync(names []string) error {
type shortBlock struct {
fn *filenode
idx int
@@ -586,19 +587,11 @@ func (dn *dirnode) sync() error {
return nil
}
- names := make([]string, 0, len(dn.inodes))
- for name := range dn.inodes {
- names = append(names, name)
- }
- sort.Strings(names)
-
for _, name := range names {
fn, ok := dn.inodes[name].(*filenode)
if !ok {
continue
}
- fn.Lock()
- defer fn.Unlock()
for idx, seg := range fn.segments {
seg, ok := seg.(*memSegment)
if !ok {
@@ -636,18 +629,19 @@ func (dn *dirnode) marshalManifest(prefix string) (string, error) {
var subdirs string
var blocks []string
- if err := dn.sync(); err != nil {
- return "", err
- }
-
names := make([]string, 0, len(dn.inodes))
- for name, node := range dn.inodes {
+ for name := range dn.inodes {
names = append(names, name)
+ }
+ sort.Strings(names)
+ for _, name := range names {
+ node := dn.inodes[name]
node.Lock()
defer node.Unlock()
}
- sort.Strings(names)
-
+ if err := dn.sync(names); err != nil {
+ return "", err
+ }
for _, name := range names {
switch node := dn.inodes[name].(type) {
case *dirnode:
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index ac3bbbbad..96347737f 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -491,15 +491,19 @@ func (s *CollectionFSSuite) TestConcurrentWriters(c *check.C) {
c.Assert(err, check.IsNil)
defer f.Close()
for i := 0; i < 6502; i++ {
- switch rand.Int() & 3 {
- case 0:
+ r := rand.Uint32()
+ switch {
+ case r%11 == 0:
+ _, err := s.fs.MarshalManifest(".")
+ c.Check(err, check.IsNil)
+ case r&3 == 0:
f.Truncate(int64(rand.Intn(64)))
- case 1:
+ case r&3 == 1:
f.Seek(int64(rand.Intn(64)), io.SeekStart)
- case 2:
+ case r&3 == 2:
_, err := f.Write([]byte("beep boop"))
c.Check(err, check.IsNil)
- case 3:
+ case r&3 == 3:
_, err := ioutil.ReadAll(f)
c.Check(err, check.IsNil)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list