[ARVADOS] updated: 1.3.0-2269-g3c0a5f77f
Git user
git at public.arvados.org
Tue Jun 9 17:38:52 UTC 2020
Summary of changes:
AUTHORS | 2 +
apps/workbench/Gemfile.lock | 6 +--
lib/dispatchcloud/container/queue.go | 2 +-
sdk/cwl/arvados_cwl/runner.py | 3 +-
services/api/Gemfile.lock | 6 +--
services/keepproxy/keepproxy.go | 6 +++
services/keepproxy/keepproxy_test.go | 75 ++++++++++++++++++++++++++++--------
7 files changed, 75 insertions(+), 25 deletions(-)
via 3c0a5f77f2bea2206aa3fe9d5ffa8245c85c9cbc (commit)
via 49e0afcce46dcba1e1402f2524b4b88f6522d216 (commit)
via 5a39bdfd6afdd0f0423adf76d26eeba7f3daf0cc (commit)
via 132c3152aac95f67f439bad68aceab9f60ad5aaa (commit)
via 7f9c18ecb14a435bf29b38a3a63450ef88d33784 (commit)
from b405ab07d342827e2bcb70ab4072d30eded90537 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 3c0a5f77f2bea2206aa3fe9d5ffa8245c85c9cbc
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Fri Jun 5 15:13:44 2020 -0300
16492: Upgrades websocket-extensions gem to fix a security alert.
We really don't use this gem as it's requested by actioncable from Rails 5
that we don't even load at runtime. But, as we cannot remove actioncable
from our Gemfile.lock files because of some other gems depending on 'rails',
we upgrade it to make GitHub happy.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index e722fa241..2420fee24 100644
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -315,7 +315,7 @@ GEM
json (>= 1.8.0)
websocket-driver (0.6.5)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.3)
+ websocket-extensions (0.1.5)
xpath (2.1.0)
nokogiri (~> 1.3)
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 9f3a5fb2b..c8a1a27b7 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -273,7 +273,7 @@ GEM
json (>= 1.8.0)
websocket-driver (0.6.5)
websocket-extensions (>= 0.1.0)
- websocket-extensions (0.1.3)
+ websocket-extensions (0.1.5)
PLATFORMS
ruby
commit 49e0afcce46dcba1e1402f2524b4b88f6522d216
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon May 11 11:08:01 2020 -0400
16339: Fix $schemas rewriting bug
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/cwl/arvados_cwl/runner.py b/sdk/cwl/arvados_cwl/runner.py
index 2239e0f9d..4c2e6697d 100644
--- a/sdk/cwl/arvados_cwl/runner.py
+++ b/sdk/cwl/arvados_cwl/runner.py
@@ -42,6 +42,7 @@ import schema_salad.validate as validate
import arvados.collection
from .util import collectionUUID
import ruamel.yaml as yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
import arvados_cwl.arvdocker
from .pathmapper import ArvPathMapper, trim_listing, collection_pdh_pattern, collection_uuid_pattern
@@ -392,7 +393,7 @@ def upload_dependencies(arvrunner, name, document_loader,
discovered_secondaryfiles[mapper.mapper(d).resolved] = discovered[d]
if "$schemas" in workflowobj:
- sch = []
+ sch = CommentedSeq()
for s in workflowobj["$schemas"]:
sch.append(mapper.mapper(s).resolved)
workflowobj["$schemas"] = sch
commit 5a39bdfd6afdd0f0423adf76d26eeba7f3daf0cc
Author: Ward Vandewege <ward at jhvc.com>
Date: Mon Apr 13 17:57:03 2020 -0400
16328: If config.yml is available, use the keepstores defined there instead of
the legacy autodiscover mechanism via the API server.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at jhvc.com>
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index 79aa992aa..188173274 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -116,6 +116,12 @@ func run(logger log.FieldLogger, cluster *arvados.Cluster) error {
return fmt.Errorf("Error setting up arvados client %v", err)
}
+ // If a config file is available, use the keepstores defined there
+ // instead of the legacy autodiscover mechanism via the API server
+ for k := range cluster.Services.Keepstore.InternalURLs {
+ arv.KeepServiceURIs = append(arv.KeepServiceURIs, k.String())
+ }
+
if cluster.SystemLogs.LogLevel == "debug" {
keepclient.DebugPrintf = log.Printf
}
diff --git a/services/keepproxy/keepproxy_test.go b/services/keepproxy/keepproxy_test.go
index aa3235680..94ed05bff 100644
--- a/services/keepproxy/keepproxy_test.go
+++ b/services/keepproxy/keepproxy_test.go
@@ -40,6 +40,12 @@ var _ = Suite(&ServerRequiredSuite{})
// Tests that require the Keep server running
type ServerRequiredSuite struct{}
+// Gocheck boilerplate
+var _ = Suite(&ServerRequiredConfigYmlSuite{})
+
+// Tests that require the Keep servers running as defined in config.yml
+type ServerRequiredConfigYmlSuite struct{}
+
// Gocheck boilerplate
var _ = Suite(&NoKeepServerSuite{})
@@ -83,6 +89,21 @@ func (s *ServerRequiredSuite) TearDownSuite(c *C) {
arvadostest.StopAPI()
}
+func (s *ServerRequiredConfigYmlSuite) SetUpSuite(c *C) {
+ arvadostest.StartAPI()
+ // config.yml defines 4 keepstores
+ arvadostest.StartKeep(4, false)
+}
+
+func (s *ServerRequiredConfigYmlSuite) SetUpTest(c *C) {
+ arvadostest.ResetEnv()
+}
+
+func (s *ServerRequiredConfigYmlSuite) TearDownSuite(c *C) {
+ arvadostest.StopKeep(4)
+ arvadostest.StopAPI()
+}
+
func (s *NoKeepServerSuite) SetUpSuite(c *C) {
arvadostest.StartAPI()
// We need API to have some keep services listed, but the
@@ -99,12 +120,17 @@ func (s *NoKeepServerSuite) TearDownSuite(c *C) {
arvadostest.StopAPI()
}
-func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
+func runProxy(c *C, bogusClientToken bool, loadKeepstoresFromConfig bool) *keepclient.KeepClient {
cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
c.Assert(err, Equals, nil)
cluster, err := cfg.GetCluster("")
c.Assert(err, Equals, nil)
+ if !loadKeepstoresFromConfig {
+ // Do not load Keepstore InternalURLs from the config file
+ cluster.Services.Keepstore.InternalURLs = make(map[arvados.URL]arvados.ServiceInstance)
+ }
+
cluster.Services.Keepproxy.InternalURLs = map[arvados.URL]arvados.ServiceInstance{arvados.URL{Host: ":0"}: arvados.ServiceInstance{}}
listener = nil
@@ -131,7 +157,7 @@ func runProxy(c *C, bogusClientToken bool) *keepclient.KeepClient {
}
func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
- runProxy(c, false)
+ runProxy(c, false, false)
defer closeListener()
req, err := http.NewRequest("POST",
@@ -158,7 +184,7 @@ func (s *ServerRequiredSuite) TestResponseViaHeader(c *C) {
}
func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
sr := map[string]string{
@@ -176,7 +202,7 @@ func (s *ServerRequiredSuite) TestLoopDetection(c *C) {
}
func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
// Set up fake keepstore to record request headers
@@ -203,7 +229,7 @@ func (s *ServerRequiredSuite) TestStorageClassesHeader(c *C) {
}
func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
content := []byte("TestDesiredReplicas")
@@ -220,7 +246,7 @@ func (s *ServerRequiredSuite) TestDesiredReplicas(c *C) {
}
func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
content := []byte("TestPutWrongContentLength")
@@ -259,7 +285,7 @@ func (s *ServerRequiredSuite) TestPutWrongContentLength(c *C) {
}
func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
router.(*proxyHandler).timeout = time.Nanosecond
@@ -286,7 +312,7 @@ func (s *ServerRequiredSuite) TestManyFailedPuts(c *C) {
}
func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -363,7 +389,7 @@ func (s *ServerRequiredSuite) TestPutAskGet(c *C) {
}
func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
- kc := runProxy(c, true)
+ kc := runProxy(c, true, false)
defer closeListener()
hash := fmt.Sprintf("%x+3", md5.Sum([]byte("bar")))
@@ -389,7 +415,7 @@ func (s *ServerRequiredSuite) TestPutAskGetForbidden(c *C) {
}
func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
- runProxy(c, false)
+ runProxy(c, false, false)
defer closeListener()
{
@@ -420,7 +446,7 @@ func (s *ServerRequiredSuite) TestCorsHeaders(c *C) {
}
func (s *ServerRequiredSuite) TestPostWithoutHash(c *C) {
- runProxy(c, false)
+ runProxy(c, false, false)
defer closeListener()
{
@@ -463,7 +489,22 @@ func (s *ServerRequiredSuite) TestStripHint(c *C) {
// With a valid but non-existing prefix (expect "\n")
// With an invalid prefix (expect error)
func (s *ServerRequiredSuite) TestGetIndex(c *C) {
- kc := runProxy(c, false)
+ getIndexWorker(c, false)
+}
+
+// Test GetIndex
+// Uses config.yml
+// Put one block, with 2 replicas
+// With no prefix (expect the block locator, twice)
+// With an existing prefix (expect the block locator, twice)
+// With a valid but non-existing prefix (expect "\n")
+// With an invalid prefix (expect error)
+func (s *ServerRequiredConfigYmlSuite) TestGetIndex(c *C) {
+ getIndexWorker(c, true)
+}
+
+func getIndexWorker(c *C, useConfig bool) {
+ kc := runProxy(c, false, useConfig)
defer closeListener()
// Put "index-data" blocks
@@ -526,7 +567,7 @@ func (s *ServerRequiredSuite) TestGetIndex(c *C) {
}
func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
hash, _, err := kc.PutB([]byte("shareddata"))
c.Check(err, IsNil)
@@ -539,7 +580,7 @@ func (s *ServerRequiredSuite) TestCollectionSharingToken(c *C) {
}
func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
// Put a test block
@@ -576,7 +617,7 @@ func (s *ServerRequiredSuite) TestPutAskGetInvalidToken(c *C) {
}
func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
// Point keepproxy at a non-existent keepstore
@@ -602,7 +643,7 @@ func (s *ServerRequiredSuite) TestAskGetKeepProxyConnectionError(c *C) {
}
func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
hash := fmt.Sprintf("%x", md5.Sum([]byte("foo")))
@@ -625,7 +666,7 @@ func (s *NoKeepServerSuite) TestAskGetNoKeepServerError(c *C) {
}
func (s *ServerRequiredSuite) TestPing(c *C) {
- kc := runProxy(c, false)
+ kc := runProxy(c, false, false)
defer closeListener()
rtr := MakeRESTRouter(kc, 10*time.Second, arvadostest.ManagementToken)
commit 132c3152aac95f67f439bad68aceab9f60ad5aaa
Author: Lucas Di Pentima <lucas at di-pentima.com.ar>
Date: Tue Apr 21 13:55:10 2020 -0300
16341: Upgrades rack on API & Workbench.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas at di-pentima.com.ar>
diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index 2af9c8b16..e722fa241 100644
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -214,7 +214,7 @@ GEM
multi_json (~> 1.0)
websocket-driver (>= 0.2.0)
public_suffix (4.0.3)
- rack (2.0.7)
+ rack (2.2.2)
rack-mini-profiler (1.0.2)
rack (>= 1.2.0)
rack-test (0.6.3)
@@ -375,4 +375,4 @@ DEPENDENCIES
uglifier (~> 2.0)
BUNDLED WITH
- 1.11
+ 1.16.6
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 24d5ad5b6..9f3a5fb2b 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -180,7 +180,7 @@ GEM
pg (1.1.4)
power_assert (1.1.4)
public_suffix (4.0.3)
- rack (2.0.7)
+ rack (2.2.2)
rack-test (0.6.3)
rack (>= 1.0)
rails (5.0.7.2)
@@ -317,4 +317,4 @@ DEPENDENCIES
uglifier (~> 2.0)
BUNDLED WITH
- 1.11
+ 1.16.6
commit 7f9c18ecb14a435bf29b38a3a63450ef88d33784
Author: Dante Tsang <waichuntwc at gmail.com>
Date: Mon Apr 20 18:23:56 2020 +0800
Fixed FetchAll method for Container queue in order to make use of preemptible instance
Updated AUTHORS file
Arvados-DCO-1.1-Signed-off-by: Dante Tsang <dante at dantetsang.com>
diff --git a/AUTHORS b/AUTHORS
index 436a504c3..93bdb37dc 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -19,3 +19,5 @@ Thomas Mooney <tmooney at genome.wustl.edu>
Chen Chen <aflyhorse at gmail.com>
Veritas Genetics, Inc. <*@veritasgenetics.com>
Curii Corporation, Inc. <*@curii.com>
+Dante Tsang <dante at dantetsang.com>
+Codex Genetics Ltd <info at codexgenetics.com>
\ No newline at end of file
diff --git a/lib/dispatchcloud/container/queue.go b/lib/dispatchcloud/container/queue.go
index d128c265f..45b346383 100644
--- a/lib/dispatchcloud/container/queue.go
+++ b/lib/dispatchcloud/container/queue.go
@@ -382,7 +382,7 @@ func (cq *Queue) poll() (map[string]*arvados.Container, error) {
*next[upd.UUID] = upd
}
}
- selectParam := []string{"uuid", "state", "priority", "runtime_constraints", "container_image", "mounts"}
+ selectParam := []string{"uuid", "state", "priority", "runtime_constraints", "container_image", "mounts", "scheduling_parameters"}
limitParam := 1000
mine, err := cq.fetchAll(arvados.ResourceListParams{
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list