[ARVADOS] updated: 6a86061651165186cfa49af41c6f7f856fee267e
git at public.curoverse.com
git at public.curoverse.com
Mon May 19 18:09:39 EDT 2014
Summary of changes:
.../arvados/v1/collections_controller.rb | 53 +++++--
services/api/config/application.default.yml | 6 +-
services/api/config/application.yml.example | 19 ++-
services/api/lib/locator.rb | 63 ++++++++
.../arvados/v1/collections_controller_test.rb | 173 ++++++++++++++++++++-
5 files changed, 287 insertions(+), 27 deletions(-)
create mode 100644 services/api/lib/locator.rb
via 6a86061651165186cfa49af41c6f7f856fee267e (commit)
via aad9cd74e61cff985944c400c40fe3f85907a1e7 (commit)
from b12f667daa270a4e3c656d16f30620ca763f9578 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6a86061651165186cfa49af41c6f7f856fee267e
Merge: aad9cd7 b12f667
Author: Tim Pierce <twp at curoverse.com>
Date: Mon May 19 18:09:18 2014 -0400
Merge branch '2755-api-collection-permissions' of git.curoverse.com:arvados into 2755-api-collection-permissions
Conflicts:
services/api/app/controllers/arvados/v1/collections_controller.rb
services/api/config/application.default.yml
services/api/config/application.yml.example
services/api/test/functional/arvados/v1/collections_controller_test.rb
commit aad9cd74e61cff985944c400c40fe3f85907a1e7
Author: Tim Pierce <twp at curoverse.com>
Date: Tue May 13 11:06:00 2014 -0400
2755: Verify permission signatures on create.
Phase 1 of #2755: when creating a new collection, verify any permission
signatures found in the manifest. Unsigned locators in the manifest are
implicitly permitted (to be disabled in Phase 4)
* New "Locator" class to parse, examine and manipulate Keep locators.
* Collections.create checks permission signatures in a manifest.
* Collections.show signs locators in a manifest.
* collections_controller_test.rb: new unit tests to exercise signed
manifests and related features:
- "create collection with signed manifest"
- "create collection with signed manifest and explicit TTL"
- "create fails with invalid signature"
- "create fails with uuid of signed manifest"
- "multiple locators per line"
- "multiple signed locators per line"
* application.yml.example: new configuration variables
- Rails.configuration.blob_signing_key
- Rails.configuration.blob_signing_ttl
(refs #2755)
diff --git a/services/api/app/controllers/arvados/v1/collections_controller.rb b/services/api/app/controllers/arvados/v1/collections_controller.rb
index 8db93c3..2844cb4 100644
--- a/services/api/app/controllers/arvados/v1/collections_controller.rb
+++ b/services/api/app/controllers/arvados/v1/collections_controller.rb
@@ -1,3 +1,5 @@
+require 'locator'
+
class Arvados::V1::CollectionsController < ApplicationController
def create
# Collections are owned by system_user. Creating a collection has
@@ -10,6 +12,48 @@ class Arvados::V1::CollectionsController < ApplicationController
logger.warn "User #{current_user.andand.uuid} tried to set collection owner_uuid to #{owner_uuid}"
raise ArvadosModel::PermissionDeniedError
end
+
+ # Check permissions on the collection manifest.
+ # If any signature cannot be verified, return 403 Permission denied.
+ perms_ok = true
+ api_token = current_api_client_authorization.andand.api_token
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: api_token,
+ ttl: Rails.configuration.blob_signing_ttl,
+ }
+ resource_attrs[:manifest_text].lines.each do |entry|
+ entry.split[1..-1].each do |tok|
+ # TODO(twp): fail the request if this match fails.
+ # Add in Phase 4 (see #2755)
+ loc = Locator.parse(tok)
+ if loc and loc.signature
+ if !api_token
+ logger.warn "No API token present; cannot verify signature on #{loc}"
+ perms_ok = false
+ elsif !Blob.verify_signature tok, signing_opts
+ logger.warn "Invalid signature on locator #{loc}"
+ perms_ok = false
+ end
+ end
+ end
+ end
+ unless perms_ok
+ raise ArvadosModel::PermissionDeniedError
+ end
+
+ # Remove any permission signatures from the manifest.
+ resource_attrs[:manifest_text]
+ .gsub!(/[[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) { |word|
+ loc = Locator.parse(word)
+ if loc
+ loc.without_signature.to_s
+ else
+ word
+ end
+ }
+
+ # Save the collection with the stripped manifest.
act_as_system_user do
@object = model_class.new resource_attrs.reject { |k,v| k == :owner_uuid }
begin
@@ -25,7 +69,6 @@ class Arvados::V1::CollectionsController < ApplicationController
@object = @existing_object || @object
end
end
-
if @object
link_attrs = {
owner_uuid: owner_uuid,
@@ -45,6 +88,22 @@ class Arvados::V1::CollectionsController < ApplicationController
end
def show
+ if current_api_client_authorization
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: current_api_client_authorization.api_token,
+ ttl: Rails.configuration.blob_signing_ttl,
+ }
+ @object[:manifest_text]
+ .gsub!(/[[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)/) { |word|
+ loc = Locator.parse(word)
+ if loc
+ Blob.sign_locator(word, signing_opts)
+ else
+ word
+ end
+ }
+ end
render json: @object.as_api_response(:with_data)
end
@@ -214,5 +273,4 @@ class Arvados::V1::CollectionsController < ApplicationController
end
end
end
-
end
diff --git a/services/api/config/application.default.yml b/services/api/config/application.default.yml
index 67aa401..a3ff680 100644
--- a/services/api/config/application.default.yml
+++ b/services/api/config/application.default.yml
@@ -43,6 +43,7 @@ test:
common:
secret_token: ~
+ blob_signing_key: ~
uuid_prefix: <%= Digest::MD5.hexdigest(`hostname`).to_i(16).to_s(36)[0..4] %>
# Git repositories must be readable by api server, or you won't be
@@ -122,3 +123,7 @@ common:
# configuration variable so that the primary server can give out the correct
# address of the dedicated websocket server:
#websocket_address: wss://127.0.0.1:3333/websocket
+
+ # Amount of time (in seconds) for which a blob permission signature
+ # remains valid. Default: 2 weeks (1209600 seconds)
+ blob_signing_ttl: 1209600
diff --git a/services/api/config/application.yml.example b/services/api/config/application.yml.example
index 9162fc4..ccdd6af 100644
--- a/services/api/config/application.yml.example
+++ b/services/api/config/application.yml.example
@@ -11,11 +11,23 @@
# 5. Section in application.default.yml called "common"
development:
+ # The blob_signing_key is a string of alphanumeric characters used
+ # to sign permission hints for Keep locators. It must be identical
+ # to the permission key given to Keep. If you run both apiserver
+ # and Keep in development, change this to a hardcoded string and
+ # make sure both systems use the same value.
+ blob_signing_key: ~
production:
# At minimum, you need a nice long randomly generated secret_token here.
+ # Use a long string of alphanumeric characters (at least 36).
secret_token: ~
+ # blob_signing_key is required and must be identical to the
+ # permission secret provisioned to Keep.
+ # Use a long string of alphanumeric characters (at least 36).
+ blob_signing_key: ~
+
uuid_prefix: bogus
# compute_node_domain: example.org
diff --git a/services/api/lib/locator.rb b/services/api/lib/locator.rb
new file mode 100644
index 0000000..0ec3f62
--- /dev/null
+++ b/services/api/lib/locator.rb
@@ -0,0 +1,63 @@
+class Locator
+ # This regex will match a word that appears to be a locator.
+ @@pattern = %r![[:xdigit:]]{32}(\+[[:digit:]]+)?(\+\S+)?!
+
+ def initialize(hasharg, sizearg, optarg)
+ @hash = hasharg
+ @size = sizearg
+ @options = optarg
+ end
+
+ def self.parse(tok)
+ Locator.parse!(tok) rescue nil
+ end
+
+ def self.parse!(tok)
+ m = /^([[:xdigit:]]{32})(\+([[:digit:]]+))?(\+.*)?$/.match(tok)
+ unless m
+ raise ArgumentError.new "could not parse #{tok}"
+ end
+
+ tokhash, _, toksize, trailer = m[1..4]
+ tokopts = []
+ while m = /^\+[[:upper:]][^\s+]+/.match(trailer)
+ opt = m.to_s
+ if opt =~ /^\+A[[:xdigit:]]+@[[:xdigit:]]{8}$/ or
+ opt =~ /\+K@[[:alnum:]]+$/
+ tokopts.push(opt)
+ trailer = m.post_match
+ else
+ raise ArgumentError.new "unknown option #{opt}"
+ end
+ end
+ if trailer and !trailer.empty?
+ raise ArgumentError.new "unrecognized trailing chars #{trailer}"
+ end
+
+ Locator.new(tokhash, toksize, tokopts)
+ end
+
+ def signature
+ @options.grep(/^\+A/).first
+ end
+
+ def without_signature
+ Locator.new(@hash, @size, @options.reject { |o| o.start_with?("+A") })
+ end
+
+ def hash
+ @hash
+ end
+
+ def size
+ @size
+ end
+
+ def options
+ @options
+ end
+
+ def to_s
+ [ @hash + "+", @size, *@options].join
+ end
+end
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 501c5a1..afda91c 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -220,4 +220,220 @@ EOS
assert_equal true, !!found.index('1f4b0bc7583c2a7f9102c395f4ffc5e3+45')
end
+ test "create collection with signed manifest" do
+ authorize_with :active
+ locators = %w(
+ d41d8cd98f00b204e9800998ecf8427e+0
+ acbd18db4cc2f85cedef654fccc4a4d8+3
+ ea10d51bcf88862dbcc36eb292017dfd+45)
+
+ unsigned_manifest = locators.map { |loc|
+ ". " + loc + " 0:0:foo.txt\n"
+ }.join()
+ manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
+ '+' +
+ unsigned_manifest.length.to_s
+
+ # build a manifest with both signed and unsigned locators.
+ # TODO(twp): in phase 4, all locators will need to be signed, so
+ # this test should break and will need to be rewritten. Issue #2755.
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: api_token(:active),
+ }
+ signed_manifest =
+ ". " + locators[0] + " 0:0:foo.txt\n" +
+ ". " + Blob.sign_locator(locators[1], signing_opts) + " 0:0:foo.txt\n" +
+ ". " + Blob.sign_locator(locators[2], signing_opts) + " 0:0:foo.txt\n"
+
+ post :create, {
+ collection: {
+ manifest_text: signed_manifest,
+ uuid: manifest_uuid,
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ resp = JSON.parse(@response.body)
+ assert_equal manifest_uuid, resp['uuid']
+ assert_equal 48, resp['data_size']
+ # All of the locators in the output must be signed.
+ resp['manifest_text'].lines.each do |entry|
+ m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
+ if m
+ assert Blob.verify_signature m[0], signing_opts
+ end
+ end
+ end
+
+ test "create collection with signed manifest and explicit TTL" do
+ authorize_with :active
+ locators = %w(
+ d41d8cd98f00b204e9800998ecf8427e+0
+ acbd18db4cc2f85cedef654fccc4a4d8+3
+ ea10d51bcf88862dbcc36eb292017dfd+45)
+
+ unsigned_manifest = locators.map { |loc|
+ ". " + loc + " 0:0:foo.txt\n"
+ }.join()
+ manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
+ '+' +
+ unsigned_manifest.length.to_s
+
+ # build a manifest with both signed and unsigned locators.
+ # TODO(twp): in phase 4, all locators will need to be signed, so
+ # this test should break and will need to be rewritten. Issue #2755.
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: api_token(:active),
+ ttl: 3600 # 1 hour
+ }
+ signed_manifest =
+ ". " + locators[0] + " 0:0:foo.txt\n" +
+ ". " + Blob.sign_locator(locators[1], signing_opts) + " 0:0:foo.txt\n" +
+ ". " + Blob.sign_locator(locators[2], signing_opts) + " 0:0:foo.txt\n"
+
+ post :create, {
+ collection: {
+ manifest_text: signed_manifest,
+ uuid: manifest_uuid,
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ resp = JSON.parse(@response.body)
+ assert_equal manifest_uuid, resp['uuid']
+ assert_equal 48, resp['data_size']
+ # All of the locators in the output must be signed.
+ resp['manifest_text'].lines.each do |entry|
+ m = /([[:xdigit:]]{32}\+\S+)/.match(entry)
+ if m
+ assert Blob.verify_signature m[0], signing_opts
+ end
+ end
+ end
+
+ test "create fails with invalid signature" do
+ authorize_with :active
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: api_token(:active),
+ }
+
+ # Generate a locator with a bad signature.
+ unsigned_locator = "d41d8cd98f00b204e9800998ecf8427e+0"
+ bad_locator = unsigned_locator + "+Affffffff at ffffffff"
+ assert !Blob.verify_signature(bad_locator, signing_opts)
+
+ # Creating a collection with this locator should
+ # produce 403 Permission denied.
+ unsigned_manifest = ". #{unsigned_locator} 0:0:foo.txt\n"
+ manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
+ '+' +
+ unsigned_manifest.length.to_s
+
+ bad_manifest = ". #{bad_locator} 0:0:foo.txt\n"
+ post :create, {
+ collection: {
+ manifest_text: bad_manifest,
+ uuid: manifest_uuid
+ }
+ }
+
+ assert_response 403
+ end
+
+ test "create fails with uuid of signed manifest" do
+ authorize_with :active
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: api_token(:active),
+ }
+
+ unsigned_locator = "d41d8cd98f00b204e9800998ecf8427e+0"
+ signed_locator = Blob.sign_locator(unsigned_locator, signing_opts)
+ signed_manifest = ". #{signed_locator} 0:0:foo.txt\n"
+ manifest_uuid = Digest::MD5.hexdigest(signed_manifest) +
+ '+' +
+ signed_manifest.length.to_s
+
+ post :create, {
+ collection: {
+ manifest_text: signed_manifest,
+ uuid: manifest_uuid
+ }
+ }
+
+ assert_response 422
+ end
+
+ test "multiple locators per line" do
+ authorize_with :active
+ locators = %w(
+ d41d8cd98f00b204e9800998ecf8427e+0
+ acbd18db4cc2f85cedef654fccc4a4d8+3
+ ea10d51bcf88862dbcc36eb292017dfd+45)
+
+ manifest_text = [".", *locators, "0:0:foo.txt\n"].join(" ")
+ manifest_uuid = Digest::MD5.hexdigest(manifest_text) +
+ '+' +
+ manifest_text.length.to_s
+
+ post :create, {
+ collection: {
+ manifest_text: manifest_text,
+ uuid: manifest_uuid,
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ resp = JSON.parse(@response.body)
+ assert_equal manifest_uuid, resp['uuid']
+ assert_equal 48, resp['data_size']
+ assert_equal resp['manifest_text'], manifest_text
+ end
+
+ test "multiple signed locators per line" do
+ authorize_with :active
+ locators = %w(
+ d41d8cd98f00b204e9800998ecf8427e+0
+ acbd18db4cc2f85cedef654fccc4a4d8+3
+ ea10d51bcf88862dbcc36eb292017dfd+45)
+
+ signing_opts = {
+ key: Rails.configuration.blob_signing_key,
+ api_token: api_token(:active),
+ }
+
+ unsigned_manifest = [".", *locators, "0:0:foo.txt\n"].join(" ")
+ manifest_uuid = Digest::MD5.hexdigest(unsigned_manifest) +
+ '+' +
+ unsigned_manifest.length.to_s
+
+ signed_locators = locators.map { |loc| Blob.sign_locator loc, signing_opts }
+ signed_manifest = [".", *signed_locators, "0:0:foo.txt\n"].join(" ")
+
+ post :create, {
+ collection: {
+ manifest_text: signed_manifest,
+ uuid: manifest_uuid,
+ }
+ }
+ assert_response :success
+ assert_not_nil assigns(:object)
+ resp = JSON.parse(@response.body)
+ assert_equal manifest_uuid, resp['uuid']
+ assert_equal 48, resp['data_size']
+ # All of the locators in the output must be signed.
+ # Each line is of the form "path locator locator ... 0:0:file.txt"
+ # entry.split[1..-2] will yield just the tokens in the middle of the line
+ returned_locator_count = 0
+ resp['manifest_text'].lines.each do |entry|
+ entry.split[1..-2].each do |tok|
+ returned_locator_count += 1
+ assert Blob.verify_signature tok, signing_opts
+ end
+ end
+ assert_equal locators.count, returned_locator_count
+ end
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list