[ARVADOS] updated: 08a10afd0681a04e5bae019870d636d49c3dc223
git at public.curoverse.com
git at public.curoverse.com
Thu Aug 13 09:30:12 EDT 2015
Summary of changes:
sdk/go/arvadosclient/arvadosclient.go | 158 +++++++++------------
sdk/go/arvadosclient/arvadosclient_test.go | 90 ++++++++++++
sdk/go/arvadosclient/pool.go | 52 +++++++
sdk/go/auth/auth.go | 61 ++++++++
.../go/auth}/basic_auth_go13.go | 2 +-
.../go/auth}/basic_auth_go14.go | 2 +-
.../go/auth}/basic_auth_test.go | 2 +-
.../server.go => sdk/go/httpserver/httpserver.go | 56 +++-----
sdk/go/httpserver/log.go | 24 ++++
sdk/go/httpserver/responsewriter.go | 43 ++++++
sdk/go/keepclient/hashcheck.go | 24 ++--
sdk/go/keepclient/support.go | 1 -
sdk/pam/.dockerignore | 6 +
sdk/{cwl => pam}/.gitignore | 0
sdk/pam/Dockerfile | 52 +++++++
sdk/pam/MANIFEST.in | 3 +
sdk/pam/README.rst | 21 +++
sdk/pam/arvados_pam.py | 100 -------------
sdk/pam/arvados_pam/__init__.py | 51 +++++++
sdk/pam/arvados_pam/auth_event.py | 88 ++++++++++++
sdk/pam/debian/arvados_pam | 10 --
sdk/pam/debian/shellinabox | 136 ------------------
sdk/pam/examples/etc_pam.d_arvados-pam-test | 3 +
sdk/pam/examples/etc_pam.d_shellinabox | 25 ++++
sdk/pam/fpm-info.sh | 17 +++
sdk/{cwl => pam}/gittaggers.py | 0
.../pam/integration_tests}/__init__.py | 0
sdk/pam/integration_tests/test_pam.py | 26 ++++
sdk/pam/lib/libpam_arvados.py | 1 +
sdk/pam/pam-configs/arvados | 14 ++
sdk/pam/setup.py | 45 ++++++
.../performance => sdk/pam/tests}/__init__.py | 0
sdk/pam/tests/integration_test.pl | 46 ++++++
sdk/pam/tests/mocker.py | 59 ++++++++
sdk/pam/tests/test_auth_event.py | 95 +++++++++++++
sdk/pam/tests/test_pam_sm.py | 26 ++++
services/api/script/salvage_collection.rb | 49 +++----
services/arv-git-httpd/auth_handler.go | 70 +++------
services/arv-git-httpd/server.go | 85 +----------
services/arv-git-httpd/server_test.go | 4 +
services/nodemanager/bin/arvados-node-manager | 0
41 files changed, 1003 insertions(+), 544 deletions(-)
create mode 100644 sdk/go/arvadosclient/pool.go
create mode 100644 sdk/go/auth/auth.go
rename {services/arv-git-httpd => sdk/go/auth}/basic_auth_go13.go (97%)
rename {services/arv-git-httpd => sdk/go/auth}/basic_auth_go14.go (91%)
rename {services/arv-git-httpd => sdk/go/auth}/basic_auth_test.go (98%)
copy services/arv-git-httpd/server.go => sdk/go/httpserver/httpserver.go (56%)
create mode 100644 sdk/go/httpserver/log.go
create mode 100644 sdk/go/httpserver/responsewriter.go
create mode 100644 sdk/pam/.dockerignore
copy sdk/{cwl => pam}/.gitignore (100%)
create mode 100644 sdk/pam/Dockerfile
create mode 100644 sdk/pam/MANIFEST.in
create mode 100644 sdk/pam/README.rst
delete mode 100644 sdk/pam/arvados_pam.py
create mode 100644 sdk/pam/arvados_pam/__init__.py
create mode 100644 sdk/pam/arvados_pam/auth_event.py
delete mode 100644 sdk/pam/debian/arvados_pam
delete mode 100644 sdk/pam/debian/shellinabox
create mode 100644 sdk/pam/examples/etc_pam.d_arvados-pam-test
create mode 100644 sdk/pam/examples/etc_pam.d_shellinabox
create mode 100644 sdk/pam/fpm-info.sh
copy sdk/{cwl => pam}/gittaggers.py (100%)
copy {services/fuse/tests/performance => sdk/pam/integration_tests}/__init__.py (100%)
create mode 100644 sdk/pam/integration_tests/test_pam.py
create mode 100644 sdk/pam/lib/libpam_arvados.py
create mode 100644 sdk/pam/pam-configs/arvados
create mode 100755 sdk/pam/setup.py
copy {services/fuse/tests/performance => sdk/pam/tests}/__init__.py (100%)
create mode 100755 sdk/pam/tests/integration_test.pl
create mode 100644 sdk/pam/tests/mocker.py
create mode 100644 sdk/pam/tests/test_auth_event.py
create mode 100644 sdk/pam/tests/test_pam_sm.py
mode change 100644 => 100755 services/nodemanager/bin/arvados-node-manager
via 08a10afd0681a04e5bae019870d636d49c3dc223 (commit)
via a0daf49f4231d5b4ec6a8a00bfa0aac1ce69cde8 (commit)
via f96550cc40f340c79339338d1da81394bfcb64ad (commit)
via 586ea290542efbd35f6e32fd6959d07cab00c969 (commit)
via ca35c58c19b28dcdbbff71fe3da0ee16ca760673 (commit)
via 3dc08cb9e6222486dcaadd50e61cad26d3200289 (commit)
via 7671ee342c26503984616eb6a864b44070b8f6d9 (commit)
via 98d6a7c2db7b8be7050fe7394147df5c5ce392bb (commit)
via c0f9c128aabb366435d751a3ea1a63b76c177f5b (commit)
via c249e92657d76221cf3977145a8dfbd79e8f6d9a (commit)
via bf23a991e15aa268f98de95e5ebee0564dfb1221 (commit)
via 4c3860e66b4a4f8108de793ddcfb66b8b5b182aa (commit)
via f8daac938f766732ade005aaf8ac239dce1f697f (commit)
via 493e39c8a1ab3caf10ae1c03e16d82f4b77e4a6a (commit)
via 094d247c7aaddbcd0a512887c6ce115558d14f72 (commit)
via 2566020d5e03c6934b9ae6c8b70368da1d03f526 (commit)
via 98feead9acd503a81c69b06bf07d6c1bfd3dd458 (commit)
via 8e671a545fd8abbf74afa109c0150c1d5772a207 (commit)
via bd583d21bb62894a5960b10bf81b375fe6336267 (commit)
via 1e5f8ef2a8c594eb0de874bef72d6d1485725b5a (commit)
via e736def390001e01ff6887acbb3c6f08366f6a91 (commit)
via 6423d36f00308b5c313715d8ddcc160052010a1c (commit)
via fbe23d045022aac8ff3ae691052af25968680944 (commit)
via fe4f75a44988826afa194c68b455c685d3b3fa8c (commit)
via c2f718800dbb03d336c0370631e9ea81dbb3997f (commit)
via 9f45d7dd8adfac9a2f690de6a0831498cff5512f (commit)
via e14e011f667d314e557c580de69a271534b6149f (commit)
via 2dfb886e960cf918e54b5f03477f464afb322a9b (commit)
via 173ebc1102e6a5a5c3a26c1bb231a4b035713369 (commit)
via e889ec14bbd18cf82acfabc681d0db967772692d (commit)
via d2e546749afaa1ff8fff8cb920b9a54d58154b76 (commit)
via 132bbb69970640e2906bc1285bee9709accb3fc7 (commit)
via 507110dc0aa1329ac4e5aad59c347a49e9f77364 (commit)
via 1702335792308d7c1d578c143a5c99b943f112a9 (commit)
via 4fc613797f88dbb33c234ba7cd13965b1236bfee (commit)
from 07f2ff368aa089e4f128578c49287d40d84dfcdf (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 08a10afd0681a04e5bae019870d636d49c3dc223
Author: radhika <radhika at curoverse.com>
Date: Thu Aug 13 09:29:17 2015 -0400
6859: fix issue with tempfile not actually being written and also update message logging.
diff --git a/services/api/script/salvage_collection.rb b/services/api/script/salvage_collection.rb
index 97eb6a8..bac033e 100755
--- a/services/api/script/salvage_collection.rb
+++ b/services/api/script/salvage_collection.rb
@@ -1,13 +1,13 @@
#!/usr/bin/env ruby
# Take two input parameters: a collection uuid and reason
-# Get "src" collection with the given uuid
-# Create a new collection "dest" with:
-# src.manifest_text as "invalid manifest_text.txt"
-# Locators from src.manifest_text as "salvaged_data"
-# Update src collection:
-# Set src.manifest_text to: ""
-# Append to src.name: " (reason; salvaged data at dest.uuid)"
+# Get "src_collection" with the given uuid
+# Create a new collection with:
+# src_collection.manifest_text as "invalid_manifest_text.txt"
+# Locators from src_collection.manifest_text as "salvaged_data"
+# Update src_collection:
+# Set src_collection.manifest_text to: ""
+# Append to src_collection.name: " (reason; salvaged data at new_collection.uuid)"
# Set portable_data_hash to "d41d8cd98f00b204e9800998ecf8427e+0"
require 'trollop'
@@ -38,18 +38,18 @@ require 'tempfile'
def salvage_collection uuid, reason
act_as_system_user do
- src = Collection.find_by_uuid uuid
- if !src
- puts "No collection found for #{uuid}"
+ src_collection = Collection.find_by_uuid uuid
+ if !src_collection
+ $stderr.puts "No collection found for #{uuid}. Returning."
return
end
begin
- src_manifest_text = src.manifest_text || ''
+ src_manifest = src_collection.manifest_text || ''
# Get all the locators from the original manifest
locators = []
- src_manifest_text.each_line do |line|
+ src_manifest.each_line do |line|
line.split(' ').each do |word|
if match = Keep::Locator::LOCATOR_REGEXP.match(word)
word = word.split('+')[0..1].join('+') # get rid of any hints
@@ -61,11 +61,11 @@ def salvage_collection uuid, reason
# create new collection using 'arv-put' with original manifest_text as the data
temp_file = Tempfile.new('temp')
- temp_file.write(src.manifest_text)
+ temp_file.write(src_manifest)
+ temp_file.close
created = %x(arv-put --use-filename invalid_manifest_text.txt #{temp_file.path})
- temp_file.close
temp_file.unlink
created.rstrip!
@@ -83,27 +83,28 @@ def salvage_collection uuid, reason
end
new_manifest += (". #{locators.join(' ')} 0:#{total_size}:salvaged_data\n")
- new_collection.name = "salvaged from #{src.uuid}, #{src.portable_data_hash}"
+ new_collection.name = "salvaged from #{src_collection.uuid}, #{src_collection.portable_data_hash}"
new_collection.manifest_text = new_manifest
new_collection.portable_data_hash = Digest::MD5.hexdigest(new_manifest)
new_collection.save!
- puts "Created collection for salvaged #{uuid} with uuid: #{created} #{match}"
+ $stderr.puts "Salvaged manifest and data for #{uuid} are in #{new_collection.uuid}."
+ puts "Created new collection #{created}"
rescue => error
- puts "Error creating collection for #{uuid}: #{error}"
+ $stderr.puts "Error creating collection for #{uuid}: #{error}"
return
end
begin
- # update src collection name, pdh, and manifest_text
- src.name = (src.name || '') + ' (' + (reason || '') + '; salvaged data at ' + created + ')'
- src.manifest_text = ''
- src.portable_data_hash = 'd41d8cd98f00b204e9800998ecf8427e+0'
- src.save!
- puts "Salvaged collection #{uuid}"
+ # update src_collection collection name, pdh, and manifest_text
+ src_collection.name = (src_collection.name || '') + ' (' + (reason || '') + '; salvaged data at ' + created + ')'
+ src_collection.manifest_text = ''
+ src_collection.portable_data_hash = 'd41d8cd98f00b204e9800998ecf8427e+0'
+ src_collection.save!
+ $stderr.puts "Collection #{uuid} emptied and renamed to #{src_collection.name.inspect}."
rescue => error
- puts "Error salvaging collection #{uuid}: #{error}"
+ $stderr.puts "Error salvaging collection #{uuid}: #{error}"
end
end
end
commit a0daf49f4231d5b4ec6a8a00bfa0aac1ce69cde8
Merge: ca35c58 f96550c
Author: radhika <radhika at curoverse.com>
Date: Thu Aug 13 08:50:24 2015 -0400
Merge branch 'master' into 6859-fix-invalid-manifests
commit ca35c58c19b28dcdbbff71fe3da0ee16ca760673
Merge: 07f2ff3 98d6a7c
Author: radhika <radhika at curoverse.com>
Date: Wed Aug 12 17:13:13 2015 -0400
Merge branch 'master' into 6859-fix-invalid-manifests
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list