[ARVADOS] updated: 1.3.0-854-gf481cb302
Git user
git at public.curoverse.com
Thu May 9 17:05:46 UTC 2019
Summary of changes:
.../container_requests_controller_test.rb | 2 +-
lib/cmdtest/leakcheck.go | 2 +-
lib/controller/federation/conn.go | 5 ++-
lib/controller/router/request.go | 28 +++++++++++++-
lib/controller/router/request_test.go | 29 ++++++++++++++
lib/controller/router/response.go | 27 +++++++++++++
lib/controller/router/router.go | 2 +-
lib/controller/router/router_test.go | 45 +++++++++++++++++++++-
lib/controller/rpc/conn.go | 11 ++++--
sdk/go/arvados/api.go | 1 +
sdk/go/arvados/api_client_authorization.go | 8 ++--
sdk/go/arvados/collection.go | 35 +++++++++--------
sdk/go/arvados/group.go | 6 +--
sdk/go/arvados/link.go | 14 +++----
sdk/go/arvados/log.go | 16 ++++----
sdk/go/arvados/node.go | 4 +-
sdk/go/arvados/resource_list.go | 2 +-
sdk/go/arvados/resource_list_test.go | 11 ++++++
sdk/go/arvados/user.go | 6 +--
sdk/go/arvados/workflow.go | 14 +++----
sdk/go/arvadostest/fixtures.go | 4 +-
services/api/test/fixtures/collections.yml | 26 ++++++-------
services/api/test/fixtures/container_requests.yml | 2 +-
services/keep-balance/collection_test.go | 2 +-
24 files changed, 224 insertions(+), 78 deletions(-)
create mode 100644 lib/controller/router/request_test.go
via f481cb302b4a2f243badf4d76fcbf8724e49781e (commit)
via 5369e269c05c832b1ba54e24546c8d0e1f11b7aa (commit)
via 422c2a7593ca4a4a990f34694f27492836e3454e (commit)
via aab6fc6506dac95c8281a38b988dd7136255a67d (commit)
via 17f96aa64f76766c58a16071d0e02deb2710d24a (commit)
via bb49b690681b72d2763c89b22e2af06845ca351e (commit)
via 04cfcf38689e1a8b23fd4d35d9d5d6b75bc21f5d (commit)
via b8456e13fb1395f67914a58c8ac2db80f25660b0 (commit)
via dc9a6a001080c982c3957a02cf76aa17502aab7b (commit)
via 199c4797fe3b3626a7548e4d4147c4729bd72d5a (commit)
via 58a959a40145a6b5ecc7c29002ceb4d5f7b33904 (commit)
via 0704c11d4818f1eee71671d82493e1d01cc76709 (commit)
via 7c57f91650bc33a35067ba9b74123a90aadfc39f (commit)
via 92a42f3f715254305e9fe0f4a47cbc9ccc03fe54 (commit)
from a207f61584e69c429b3bcca46aa2a54999076dd0 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit f481cb302b4a2f243badf4d76fcbf8724e49781e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu May 9 13:04:56 2019 -0400
14287: Fix debug log level.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 4a6f9b5af..cd66e90d2 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -189,7 +189,7 @@ func (rtr *router) addRoutes(cluster *arvados.Cluster) {
ctx = arvados.ContextWithRequestID(ctx, req.Header.Get("X-Request-Id"))
resp, err := route.exec(ctx, opts)
if err != nil {
- ctxlog.FromContext(ctx).WithError(err).Infof("returning error response for %#v", err)
+ ctxlog.FromContext(ctx).WithError(err).Debugf("returning error response for %#v", err)
rtr.sendError(w, err)
return
}
commit 5369e269c05c832b1ba54e24546c8d0e1f11b7aa
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed May 8 11:45:22 2019 -0400
14287: Remove omitempty tags so zeroes/nulls appear in responses.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/api_client_authorization.go b/sdk/go/arvados/api_client_authorization.go
index 17cff235d..7c17cdef0 100644
--- a/sdk/go/arvados/api_client_authorization.go
+++ b/sdk/go/arvados/api_client_authorization.go
@@ -6,10 +6,10 @@ package arvados
// APIClientAuthorization is an arvados#apiClientAuthorization resource.
type APIClientAuthorization struct {
- UUID string `json:"uuid,omitempty"`
- APIToken string `json:"api_token,omitempty"`
- ExpiresAt string `json:"expires_at,omitempty"`
- Scopes []string `json:"scopes,omitempty"`
+ UUID string `json:"uuid"`
+ APIToken string `json:"api_token"`
+ ExpiresAt string `json:"expires_at"`
+ Scopes []string `json:"scopes"`
}
// APIClientAuthorizationList is an arvados#apiClientAuthorizationList resource.
diff --git a/sdk/go/arvados/collection.go b/sdk/go/arvados/collection.go
index f374eea07..136159a7e 100644
--- a/sdk/go/arvados/collection.go
+++ b/sdk/go/arvados/collection.go
@@ -15,23 +15,24 @@ import (
// Collection is an arvados#collection resource.
type Collection struct {
- UUID string `json:"uuid,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
- TrashAt *time.Time `json:"trash_at,omitempty"`
- ManifestText string `json:"manifest_text"`
- UnsignedManifestText string `json:"unsigned_manifest_text,omitempty"`
- Name string `json:"name,omitempty"`
- CreatedAt *time.Time `json:"created_at,omitempty"`
- ModifiedAt *time.Time `json:"modified_at,omitempty"`
- PortableDataHash string `json:"portable_data_hash,omitempty"`
- ReplicationConfirmed *int `json:"replication_confirmed,omitempty"`
- ReplicationConfirmedAt *time.Time `json:"replication_confirmed_at,omitempty"`
- ReplicationDesired *int `json:"replication_desired,omitempty"`
- StorageClassesDesired []string `json:"storage_classes_desired,omitempty"`
- StorageClassesConfirmed []string `json:"storage_classes_confirmed,omitempty"`
- StorageClassesConfirmedAt *time.Time `json:"storage_classes_confirmed_at,omitempty"`
- DeleteAt *time.Time `json:"delete_at,omitempty"`
- IsTrashed bool `json:"is_trashed,omitempty"`
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ TrashAt *time.Time `json:"trash_at"`
+ ManifestText string `json:"manifest_text"`
+ UnsignedManifestText string `json:"unsigned_manifest_text"`
+ Name string `json:"name"`
+ CreatedAt *time.Time `json:"created_at"`
+ ModifiedAt *time.Time `json:"modified_at"`
+ PortableDataHash string `json:"portable_data_hash"`
+ ReplicationConfirmed *int `json:"replication_confirmed"`
+ ReplicationConfirmedAt *time.Time `json:"replication_confirmed_at"`
+ ReplicationDesired *int `json:"replication_desired"`
+ StorageClassesDesired []string `json:"storage_classes_desired"`
+ StorageClassesConfirmed []string `json:"storage_classes_confirmed"`
+ StorageClassesConfirmedAt *time.Time `json:"storage_classes_confirmed_at"`
+ DeleteAt *time.Time `json:"delete_at"`
+ IsTrashed bool `json:"is_trashed"`
+ Properties map[string]interface{} `json:"properties"`
}
func (c Collection) resourceName() string {
diff --git a/sdk/go/arvados/group.go b/sdk/go/arvados/group.go
index 6b5718a6c..bf2fe72ff 100644
--- a/sdk/go/arvados/group.go
+++ b/sdk/go/arvados/group.go
@@ -6,9 +6,9 @@ package arvados
// Group is an arvados#group record
type Group struct {
- UUID string `json:"uuid,omitempty"`
- Name string `json:"name,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
+ UUID string `json:"uuid"`
+ Name string `json:"name"`
+ OwnerUUID string `json:"owner_uuid"`
GroupClass string `json:"group_class"`
}
diff --git a/sdk/go/arvados/link.go b/sdk/go/arvados/link.go
index dee13556e..fbd699f30 100644
--- a/sdk/go/arvados/link.go
+++ b/sdk/go/arvados/link.go
@@ -7,13 +7,13 @@ package arvados
// Link is an arvados#link record
type Link struct {
UUID string `json:"uuid,omiempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
- Name string `json:"name,omitempty"`
- LinkClass string `json:"link_class,omitempty"`
- HeadUUID string `json:"head_uuid,omitempty"`
- HeadKind string `json:"head_kind,omitempty"`
- TailUUID string `json:"tail_uuid,omitempty"`
- TailKind string `json:"tail_kind,omitempty"`
+ OwnerUUID string `json:"owner_uuid"`
+ Name string `json:"name"`
+ LinkClass string `json:"link_class"`
+ HeadUUID string `json:"head_uuid"`
+ HeadKind string `json:"head_kind"`
+ TailUUID string `json:"tail_uuid"`
+ TailKind string `json:"tail_kind"`
}
// UserList is an arvados#userList resource.
diff --git a/sdk/go/arvados/log.go b/sdk/go/arvados/log.go
index 6f72bf7c6..6f72634e5 100644
--- a/sdk/go/arvados/log.go
+++ b/sdk/go/arvados/log.go
@@ -10,14 +10,14 @@ import (
// Log is an arvados#log record
type Log struct {
- ID uint64 `json:"id,omitempty"`
- UUID string `json:"uuid,omitempty"`
- ObjectUUID string `json:"object_uuid,omitempty"`
- ObjectOwnerUUID string `json:"object_owner_uuid,omitempty"`
- EventType string `json:"event_type,omitempty"`
- EventAt *time.Time `json:"event,omitempty"`
- Properties map[string]interface{} `json:"properties,omitempty"`
- CreatedAt *time.Time `json:"created_at,omitempty"`
+ ID uint64 `json:"id"`
+ UUID string `json:"uuid"`
+ ObjectUUID string `json:"object_uuid"`
+ ObjectOwnerUUID string `json:"object_owner_uuid"`
+ EventType string `json:"event_type"`
+ EventAt *time.Time `json:"event"`
+ Properties map[string]interface{} `json:"properties"`
+ CreatedAt *time.Time `json:"created_at"`
}
// LogList is an arvados#logList resource.
diff --git a/sdk/go/arvados/node.go b/sdk/go/arvados/node.go
index cc844fe82..97466eb8a 100644
--- a/sdk/go/arvados/node.go
+++ b/sdk/go/arvados/node.go
@@ -12,10 +12,10 @@ type Node struct {
Domain string `json:"domain"`
Hostname string `json:"hostname"`
IPAddress string `json:"ip_address"`
- LastPingAt *time.Time `json:"last_ping_at,omitempty"`
+ LastPingAt *time.Time `json:"last_ping_at"`
SlotNumber int `json:"slot_number"`
Status string `json:"status"`
- JobUUID string `json:"job_uuid,omitempty"`
+ JobUUID string `json:"job_uuid"`
Properties NodeProperties `json:"properties"`
}
diff --git a/sdk/go/arvados/user.go b/sdk/go/arvados/user.go
index 3a36e5eba..27d2b28a4 100644
--- a/sdk/go/arvados/user.go
+++ b/sdk/go/arvados/user.go
@@ -6,11 +6,11 @@ package arvados
// User is an arvados#user record
type User struct {
- UUID string `json:"uuid,omitempty"`
+ UUID string `json:"uuid"`
IsActive bool `json:"is_active"`
IsAdmin bool `json:"is_admin"`
- Username string `json:"username,omitempty"`
- Email string `json:"email,omitempty"`
+ Username string `json:"username"`
+ Email string `json:"email"`
}
// UserList is an arvados#userList resource.
diff --git a/sdk/go/arvados/workflow.go b/sdk/go/arvados/workflow.go
index 09c8c71e8..5ddc8732d 100644
--- a/sdk/go/arvados/workflow.go
+++ b/sdk/go/arvados/workflow.go
@@ -8,13 +8,13 @@ import "time"
// Workflow is an arvados#workflow resource.
type Workflow struct {
- UUID string `json:"uuid,omitempty"`
- OwnerUUID string `json:"owner_uuid,omitempty"`
- Name string `json:"name,omitempty"`
- Description string `json:"description,omitempty"`
- Definition string `json:"definition,omitempty"`
- CreatedAt *time.Time `json:"created_at,omitempty"`
- ModifiedAt *time.Time `json:"modified_at,omitempty"`
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ Name string `json:"name"`
+ Description string `json:"description"`
+ Definition string `json:"definition"`
+ CreatedAt *time.Time `json:"created_at"`
+ ModifiedAt *time.Time `json:"modified_at"`
}
// WorkflowList is an arvados#workflowList resource.
commit 422c2a7593ca4a4a990f34694f27492836e3454e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed May 8 11:43:33 2019 -0400
14287: Ensure timestamps in responses have 9 digits of nanoseconds.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/response.go b/lib/controller/router/response.go
index 65e0159fa..ddbeee666 100644
--- a/lib/controller/router/response.go
+++ b/lib/controller/router/response.go
@@ -7,11 +7,15 @@ package router
import (
"encoding/json"
"net/http"
+ "strings"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/httpserver"
)
+const rfc3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
type responseOptions struct {
Select []string
}
@@ -41,6 +45,29 @@ func (rtr *router) sendResponse(w http.ResponseWriter, resp interface{}, opts re
}
tmp = selected
}
+ // Format non-nil timestamps as rfc3339NanoFixed (by default
+ // they will have been encoded to time.RFC3339Nano, which
+ // omits trailing zeroes).
+ for k, v := range tmp {
+ if !strings.HasSuffix(k, "_at") {
+ continue
+ }
+ switch tv := v.(type) {
+ case *time.Time:
+ if tv == nil {
+ break
+ }
+ tmp[k] = tv.Format(rfc3339NanoFixed)
+ case time.Time:
+ tmp[k] = tv.Format(rfc3339NanoFixed)
+ case string:
+ t, err := time.Parse(time.RFC3339Nano, tv)
+ if err != nil {
+ break
+ }
+ tmp[k] = t.Format(rfc3339NanoFixed)
+ }
+ }
json.NewEncoder(w).Encode(tmp)
}
commit aab6fc6506dac95c8281a38b988dd7136255a67d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed May 8 11:36:29 2019 -0400
14287: Fix accepting JSON-encoded params in request body.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/request.go b/lib/controller/router/request.go
index 8ea253e6c..aa2cd636c 100644
--- a/lib/controller/router/request.go
+++ b/lib/controller/router/request.go
@@ -6,6 +6,7 @@ package router
import (
"encoding/json"
+ "fmt"
"io"
"mime"
"net/http"
@@ -66,9 +67,15 @@ func (rtr *router) loadRequestParams(req *http.Request, attrsKey string) (map[st
// as foo=["bar","baz"]?
}
}
- if ct, _, err := mime.ParseMediaType(req.Header.Get("Content-Type")); err != nil && ct == "application/json" {
+
+ // Decode body as JSON if Content-Type request header is
+ // missing or application/json.
+ mt := req.Header.Get("Content-Type")
+ if ct, _, err := mime.ParseMediaType(mt); err != nil && mt != "" {
+ return nil, fmt.Errorf("error parsing media type %q: %s", mt, err)
+ } else if (ct == "application/json" || mt == "") && req.ContentLength != 0 {
jsonParams := map[string]interface{}{}
- err := json.NewDecoder(req.Body).Decode(jsonParams)
+ err := json.NewDecoder(req.Body).Decode(&jsonParams)
if err != nil {
return nil, httpError(http.StatusBadRequest, err)
}
diff --git a/lib/controller/router/request_test.go b/lib/controller/router/request_test.go
new file mode 100644
index 000000000..cffdccc90
--- /dev/null
+++ b/lib/controller/router/request_test.go
@@ -0,0 +1,29 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package router
+
+import (
+ "bytes"
+ "net/http/httptest"
+
+ check "gopkg.in/check.v1"
+)
+
+func (s *RouterSuite) TestAttrsInBody(c *check.C) {
+ for _, body := range []string{
+ `{"foo":"bar"}`,
+ `{"model_name": {"foo":"bar"}}`,
+ } {
+ c.Logf("body: %s", body)
+ req := httptest.NewRequest("POST", "https://an.example/ctrl", bytes.NewBufferString(body))
+ req.Header.Set("Content-Type", "application/json")
+ params, err := s.rtr.loadRequestParams(req, "model_name")
+ c.Assert(err, check.IsNil)
+ c.Logf("params: %#v", params)
+ c.Check(params, check.NotNil)
+ c.Assert(params["attrs"], check.FitsTypeOf, map[string]interface{}{})
+ c.Check(params["attrs"].(map[string]interface{})["foo"], check.Equals, "bar")
+ }
+}
commit 17f96aa64f76766c58a16071d0e02deb2710d24a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed May 8 11:34:58 2019 -0400
14287: Fix PDH check to ignore additional hints.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index a08ec48f4..ad46d8788 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -211,7 +211,10 @@ func (conn *Conn) CollectionGet(ctx context.Context, options arvados.GetOptions)
if err != nil {
return err
}
- if pdh := portableDataHash(c.ManifestText); pdh != options.UUID {
+ // options.UUID is either hash+size or
+ // hash+size+hints; only hash+size need to
+ // match the computed PDH.
+ if pdh := portableDataHash(c.ManifestText); pdh != options.UUID && !strings.HasPrefix(options.UUID, pdh+"+") {
ctxlog.FromContext(ctx).Warnf("bad portable data hash %q received from remote %q (expected %q)", pdh, remoteID, options.UUID)
return notFoundError{}
}
commit bb49b690681b72d2763c89b22e2af06845ca351e
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 15:49:57 2019 -0400
14287: Fix PDHs and manifests in test fixtures.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/apps/workbench/test/controllers/container_requests_controller_test.rb b/apps/workbench/test/controllers/container_requests_controller_test.rb
index 6e96839e2..6940a0907 100644
--- a/apps/workbench/test/controllers/container_requests_controller_test.rb
+++ b/apps/workbench/test/controllers/container_requests_controller_test.rb
@@ -138,7 +138,7 @@ class ContainerRequestsControllerTest < ActionController::TestCase
assert_includes @response.body, "href=\"\/collections/fa7aeb5140e2848d39b416daeef4ffc5+45/foobar\?" # locator on command
assert_includes @response.body, "href=\"\/collections/fa7aeb5140e2848d39b416daeef4ffc5+45/foo" # mount input1
assert_includes @response.body, "href=\"\/collections/fa7aeb5140e2848d39b416daeef4ffc5+45/bar" # mount input2
- assert_includes @response.body, "href=\"\/collections/1fd08fc162a5c6413070a8bd0bffc818+150" # mount workflow
+ assert_includes @response.body, "href=\"\/collections/f9ddda46bb293b6847da984e3aa735db+290" # mount workflow
assert_includes @response.body, "href=\"#Log\""
assert_includes @response.body, "href=\"#Provenance\""
end
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index f2f457448..37633ed23 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -33,7 +33,7 @@ const (
ASubprojectUUID = "zzzzz-j7d0g-axqo7eu9pwvna1x"
FooAndBarFilesInDirUUID = "zzzzz-4zz18-foonbarfilesdir"
- FooAndBarFilesInDirPDH = "6bbac24198d09a93975f60098caf0bdf+62"
+ FooAndBarFilesInDirPDH = "870369fc72738603c2fad16664e50e2d+58"
Dispatch1Token = "kwi8oowusvbutahacwk2geulqewy5oaqmpalczfna4b6bb0hfw"
Dispatch1AuthUUID = "zzzzz-gj3su-k9dvestay1plssr"
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index c84e479e4..5024ecc96 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -129,7 +129,7 @@ w_a_z_file_version_1:
multilevel_collection_1:
uuid: zzzzz-4zz18-pyw8yp9g3pr7irn
current_version_uuid: zzzzz-4zz18-pyw8yp9g3pr7irn
- portable_data_hash: 1fd08fc162a5c6413070a8bd0bffc818+150
+ portable_data_hash: f9ddda46bb293b6847da984e3aa735db+290
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -143,7 +143,7 @@ multilevel_collection_2:
uuid: zzzzz-4zz18-45xf9hw1sxkhl6q
current_version_uuid: zzzzz-4zz18-45xf9hw1sxkhl6q
# All of this collection's files are deep in subdirectories.
- portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
+ portable_data_hash: 8591cc5caeca80fc62fd529ba1d63bf3+118
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -394,7 +394,7 @@ unique_expired_collection:
unique_expired_collection2:
uuid: zzzzz-4zz18-mto52zx1s7sn3jr
current_version_uuid: zzzzz-4zz18-mto52zx1s7sn3jr
- portable_data_hash: 4ad199f90029935844dc3f098f4fca2b+49
+ portable_data_hash: 64a2bed1ef0f40fe3a7d39bcf2584cb8+50
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -404,7 +404,7 @@ unique_expired_collection2:
is_trashed: true
trash_at: 2001-01-01T00:00:00Z
delete_at: 2038-01-01T00:00:00Z
- manifest_text: ". 29d7797f1888013986899bc9083783fa+3 0:3:expired\n"
+ manifest_text: ". 29d7797f1888013986899bc9083783fa+3 0:3:expired2\n"
name: unique_expired_collection2
# a collection with a log file that can be parsed by the log viewer
@@ -474,14 +474,14 @@ collection_with_files_in_subdir:
uuid: zzzzz-4zz18-filesinsubdir00
current_version_uuid: zzzzz-4zz18-filesinsubdir00
name: collection_files_in_subdir
- portable_data_hash: 85877ca2d7e05498dd3d109baf2df106+95
+ portable_data_hash: 7eb64275355980ebc93411b44050c137+281
owner_uuid: zzzzz-tpzed-user1withloadab
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
modified_by_user_uuid: zzzzz-tpzed-user1withloadab
modified_at: 2014-02-03T17:22:54Z
updated_at: 2014-02-03T17:22:54Z
- manifest_text: ". 85877ca2d7e05498dd3d109baf2df106+95 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64 0:32:file1_in_subdir4.txt 32:32:file2_in_subdir4.txt"
+ manifest_text: ". 85877ca2d7e05498dd3d109baf2df106+95 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64 0:32:file1_in_subdir4.txt 32:32:file2_in_subdir4.txt\n"
graph_test_collection1:
uuid: zzzzz-4zz18-bv31uwvy3neko22
@@ -722,7 +722,7 @@ collection_with_one_property:
collection_with_repeated_filenames_and_contents_in_two_dirs_1:
uuid: zzzzz-4zz18-duplicatenames1
current_version_uuid: zzzzz-4zz18-duplicatenames1
- portable_data_hash: f3a67fad3a19c31c658982fb8158fa58+144
+ portable_data_hash: ce437b12aa73ab34f7af5227f556c9e6+142
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -748,7 +748,7 @@ collection_with_repeated_filenames_and_contents_in_two_dirs_2:
foo_and_bar_files_in_dir:
uuid: zzzzz-4zz18-foonbarfilesdir
current_version_uuid: zzzzz-4zz18-foonbarfilesdir
- portable_data_hash: 6bbac24198d09a93975f60098caf0bdf+62
+ portable_data_hash: 870369fc72738603c2fad16664e50e2d+58
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -801,7 +801,7 @@ collection_with_several_unsupported_file_types:
collection_not_readable_by_active:
uuid: zzzzz-4zz18-cd42uwvy3neko21
current_version_uuid: zzzzz-4zz18-cd42uwvy3neko21
- portable_data_hash: bb89eb5140e2848d39b416daeef4ffc5+45
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
owner_uuid: zzzzz-tpzed-000000000000000
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -814,7 +814,7 @@ collection_not_readable_by_active:
collection_to_remove_and_rename_files:
uuid: zzzzz-4zz18-a21ux3541sxa8sf
current_version_uuid: zzzzz-4zz18-a21ux3541sxa8sf
- portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
+ portable_data_hash: 21aed8fd508bd6263704b673455949ba+57
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -843,7 +843,7 @@ collection_with_tags_owned_by_active:
trashed_collection_to_test_name_conflict_on_untrash:
uuid: zzzzz-4zz18-trashedcolnamec
current_version_uuid: zzzzz-4zz18-trashedcolnamec
- portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
+ portable_data_hash: 21aed8fd508bd6263704b673455949ba+57
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -859,7 +859,7 @@ trashed_collection_to_test_name_conflict_on_untrash:
same_name_as_trashed_coll_to_test_name_conflict_on_untrash:
uuid: zzzzz-4zz18-namesameastrash
current_version_uuid: zzzzz-4zz18-namesameastrash
- portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
+ portable_data_hash: 21aed8fd508bd6263704b673455949ba+57
owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
@@ -872,7 +872,7 @@ same_name_as_trashed_coll_to_test_name_conflict_on_untrash:
collection_in_trashed_subproject:
uuid: zzzzz-4zz18-trashedproj2col
current_version_uuid: zzzzz-4zz18-trashedproj2col
- portable_data_hash: 80cf6dd2cf079dd13f272ec4245cb4a8+48
+ portable_data_hash: 21aed8fd508bd6263704b673455949ba+57
owner_uuid: zzzzz-j7d0g-trashedproject2
created_at: 2014-02-03T17:22:54Z
modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
diff --git a/services/api/test/fixtures/container_requests.yml b/services/api/test/fixtures/container_requests.yml
index dea98887e..ea86dca17 100644
--- a/services/api/test/fixtures/container_requests.yml
+++ b/services/api/test/fixtures/container_requests.yml
@@ -322,7 +322,7 @@ completed_with_input_mounts:
basename: bar
class: File
location: "keep:fa7aeb5140e2848d39b416daeef4ffc5+45/bar"
- /var/lib/cwl/workflow.json: "keep:1fd08fc162a5c6413070a8bd0bffc818+150"
+ /var/lib/cwl/workflow.json: "keep:f9ddda46bb293b6847da984e3aa735db+290"
uncommitted:
uuid: zzzzz-xvhdp-cr4uncommittedc
commit 04cfcf38689e1a8b23fd4d35d9d5d6b75bc21f5d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 15:24:36 2019 -0400
14287: Fix unparsed formatting directive.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/cmdtest/leakcheck.go b/lib/cmdtest/leakcheck.go
index c132f1b36..ba4c3c123 100644
--- a/lib/cmdtest/leakcheck.go
+++ b/lib/cmdtest/leakcheck.go
@@ -43,7 +43,7 @@ func LeakCheck(c *check.C) func() {
os.Stdout, os.Stderr = stdout, stderr
for i, tmpfile := range tmpfiles {
- c.Log("checking %s", i)
+ c.Logf("checking %s", i)
_, err := tmpfile.Seek(0, io.SeekStart)
c.Assert(err, check.IsNil)
leaked, err := ioutil.ReadAll(tmpfile)
commit b8456e13fb1395f67914a58c8ac2db80f25660b0
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 15:10:52 2019 -0400
14287: Propagate order param.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index 4cdf7c0e1..a1c790680 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -42,6 +42,7 @@ type ListOptions struct {
Filters []Filter `json:"filters"`
Limit int `json:"limit"`
Offset int `json:"offset"`
+ Order string `json:"order"`
}
type CreateOptions struct {
commit dc9a6a001080c982c3957a02cf76aa17502aab7b
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 13:50:52 2019 -0400
14287: Test timestamp precision is maintained by response munging.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 9b04c2672..2e0c52b79 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -12,6 +12,7 @@ import (
"net/http/httptest"
"os"
"testing"
+ "time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
@@ -126,6 +127,27 @@ func (s *RouterSuite) TestContainerLock(c *check.C) {
c.Check(jresp["uuid"], check.IsNil)
}
+func (s *RouterSuite) TestFullTimestampsInResponse(c *check.C) {
+ uuid := arvadostest.CollectionReplicationDesired2Confirmed2UUID
+ token := arvadostest.ActiveTokenV2
+
+ _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections/`+uuid, nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["uuid"], check.Equals, uuid)
+ expectNS := map[string]int{
+ "created_at": 596506000, // fixture says 596506247, but truncated by postgresql
+ "modified_at": 596338000, // fixture says 596338465, but truncated by postgresql
+ }
+ for key, ns := range expectNS {
+ mt, ok := jresp[key].(string)
+ c.Logf("jresp[%q] == %q", key, mt)
+ c.Assert(ok, check.Equals, true)
+ t, err := time.Parse(time.RFC3339Nano, mt)
+ c.Check(err, check.IsNil)
+ c.Check(t.Nanosecond(), check.Equals, ns)
+ }
+}
+
func (s *RouterSuite) TestSelectParam(c *check.C) {
uuid := arvadostest.QueuedContainerUUID
token := arvadostest.ActiveTokenV2
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index 4f648e9b4..f2f457448 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -56,6 +56,8 @@ const (
FooCollectionSharingToken = "iknqgmunrhgsyfok8uzjlwun9iscwm3xacmzmg65fa1j1lpdss"
WorkflowWithDefinitionYAMLUUID = "zzzzz-7fd4e-validworkfloyml"
+
+ CollectionReplicationDesired2Confirmed2UUID = "zzzzz-4zz18-434zv1tnnf2rygp"
)
// PathologicalManifest : A valid manifest designed to test
commit 199c4797fe3b3626a7548e4d4147c4729bd72d5a
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 09:47:01 2019 -0400
14287: Fix token in container test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 30c15c28e..9b04c2672 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -108,7 +108,7 @@ func (s *RouterSuite) TestContainerList(c *check.C) {
func (s *RouterSuite) TestContainerLock(c *check.C) {
uuid := arvadostest.QueuedContainerUUID
- token := arvadostest.ActiveTokenV2
+ token := arvadostest.AdminToken
_, rw, jresp := s.doRequest(c, token, "POST", "/arvados/v1/containers/"+uuid+"/lock", nil, nil)
c.Check(rw.Code, check.Equals, http.StatusOK)
c.Check(jresp["uuid"], check.HasLen, 27)
commit 58a959a40145a6b5ecc7c29002ceb4d5f7b33904
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 09:45:58 2019 -0400
14287: Accept rpc requests without tokens.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 7c23ed170..9bb3eb33f 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -71,10 +71,15 @@ func (conn *Conn) requestAndDecode(ctx context.Context, dst interface{}, ep arva
tokens, err := conn.tokenProvider(ctx)
if err != nil {
return err
- } else if len(tokens) == 0 {
- return fmt.Errorf("bug: token provider returned no tokens and no error")
+ } else if len(tokens) > 0 {
+ ctx = context.WithValue(ctx, "Authorization", "Bearer "+tokens[0])
+ } else {
+ // Use a non-empty auth string to ensure we override
+ // any default token set on aClient -- and to avoid
+ // having the remote prompt us to send a token by
+ // responding 401.
+ ctx = context.WithValue(ctx, "Authorization", "Bearer -")
}
- ctx = context.WithValue(ctx, "Authorization", "Bearer "+tokens[0])
// Encode opts to JSON and decode from there to a
// map[string]interface{}, so we can munge the query params
commit 0704c11d4818f1eee71671d82493e1d01cc76709
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 09:40:47 2019 -0400
14287: Fix accepting nil as filter operand.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/resource_list.go b/sdk/go/arvados/resource_list.go
index 505ba51ec..d1a25c438 100644
--- a/sdk/go/arvados/resource_list.go
+++ b/sdk/go/arvados/resource_list.go
@@ -55,7 +55,7 @@ func (f *Filter) UnmarshalJSON(data []byte) error {
}
operand := elements[2]
switch operand.(type) {
- case string, float64, []interface{}:
+ case string, float64, []interface{}, nil:
default:
return fmt.Errorf("invalid filter operand %q", elements[2])
}
diff --git a/sdk/go/arvados/resource_list_test.go b/sdk/go/arvados/resource_list_test.go
index 5642599b4..4e09c5375 100644
--- a/sdk/go/arvados/resource_list_test.go
+++ b/sdk/go/arvados/resource_list_test.go
@@ -23,3 +23,14 @@ func TestMarshalFiltersWithNanoseconds(t *testing.T) {
t.Errorf("Encoded as %q, expected %q", buf, expect)
}
}
+
+func TestMarshalFiltersWithNil(t *testing.T) {
+ buf, err := json.Marshal([]Filter{
+ {Attr: "modified_at", Operator: "=", Operand: nil}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if expect := []byte(`[["modified_at","=",null]]`); 0 != bytes.Compare(buf, expect) {
+ t.Errorf("Encoded as %q, expected %q", buf, expect)
+ }
+}
commit 7c57f91650bc33a35067ba9b74123a90aadfc39f
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Tue May 7 09:39:58 2019 -0400
14287: Fix accepting boolean params via query string.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/controller/router/request.go b/lib/controller/router/request.go
index 67d4e0ffb..8ea253e6c 100644
--- a/lib/controller/router/request.go
+++ b/lib/controller/router/request.go
@@ -29,6 +29,8 @@ func (rtr *router) loadRequestParams(req *http.Request, attrsKey string) (map[st
for k, values := range req.Form {
for _, v := range values {
switch {
+ case boolParams[k]:
+ params[k] = stringToBool(v)
case v == "null" || v == "":
params[k] = nil
case strings.HasPrefix(v, "["):
@@ -110,3 +112,18 @@ func (rtr *router) transcode(src interface{}, dst interface{}) error {
}
return err
}
+
+var boolParams = map[string]bool{
+ "ensure_unique_name": true,
+ "include_trash": true,
+ "include_old_versions": true,
+}
+
+func stringToBool(s string) bool {
+ switch s {
+ case "", "false", "0":
+ return false
+ default:
+ return true
+ }
+}
diff --git a/lib/controller/router/router_test.go b/lib/controller/router/router_test.go
index 97710d265..30c15c28e 100644
--- a/lib/controller/router/router_test.go
+++ b/lib/controller/router/router_test.go
@@ -5,6 +5,7 @@
package router
import (
+ "bytes"
"encoding/json"
"io"
"net/http"
@@ -52,6 +53,26 @@ func (s *RouterSuite) doRequest(c *check.C, token, method, path string, hdrs htt
return req, rw, jresp
}
+func (s *RouterSuite) TestCollectionParams(c *check.C) {
+ token := arvadostest.ActiveTokenV2
+
+ _, rw, jresp := s.doRequest(c, token, "GET", `/arvados/v1/collections?include_trash=true`, nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["items_available"], check.FitsTypeOf, float64(0))
+
+ _, rw, jresp = s.doRequest(c, token, "GET", `/arvados/v1/collections`, nil, bytes.NewBufferString(`{"include_trash":true}`))
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["items"], check.FitsTypeOf, []interface{}{})
+
+ _, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections`, nil, bytes.NewBufferString(`ensure_unique_name=true`))
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["uuid"], check.FitsTypeOf, "")
+
+ _, rw, jresp = s.doRequest(c, token, "POST", `/arvados/v1/collections?ensure_unique_name=true`, nil, nil)
+ c.Check(rw.Code, check.Equals, http.StatusOK)
+ c.Check(jresp["uuid"], check.FitsTypeOf, "")
+}
+
func (s *RouterSuite) TestContainerList(c *check.C) {
token := arvadostest.ActiveTokenV2
commit 92a42f3f715254305e9fe0f4a47cbc9ccc03fe54
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon May 6 16:44:12 2019 -0400
14287: Dedup "UUIDs seen" list before diff in test.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/collection_test.go b/services/keep-balance/collection_test.go
index a548b1ff9..6aaf07aba 100644
--- a/services/keep-balance/collection_test.go
+++ b/services/keep-balance/collection_test.go
@@ -30,7 +30,6 @@ func (s *integrationSuite) TestIdenticalTimestamps(c *check.C) {
var lastMod time.Time
sawUUID := make(map[string]bool)
err := EachCollection(&s.config.Client, pageSize, func(c arvados.Collection) error {
- got[trial] = append(got[trial], c.UUID)
if c.ModifiedAt == nil {
return nil
}
@@ -38,6 +37,7 @@ func (s *integrationSuite) TestIdenticalTimestamps(c *check.C) {
// dup
return nil
}
+ got[trial] = append(got[trial], c.UUID)
sawUUID[c.UUID] = true
if lastMod == *c.ModifiedAt {
streak++
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list