[ARVADOS] updated: 19656a41f019488120f950b06ecf9e19074b11a3
git at public.curoverse.com
git at public.curoverse.com
Mon Mar 9 11:35:54 EDT 2015
Summary of changes:
...tion_portable_data_hash_with_hinted_manifest.rb | 111 +++++++++++++++++++++
services/api/db/structure.sql | 4 +-
services/api/test/fixtures/collections.yml | 2 +-
3 files changed, 115 insertions(+), 2 deletions(-)
create mode 100644 services/api/db/migrate/20150303210106_fix_collection_portable_data_hash_with_hinted_manifest.rb
via 19656a41f019488120f950b06ecf9e19074b11a3 (commit)
via 2a610a9d5d7e9446935436ff37267b8ada25fdbd (commit)
via cf3a1032b04dd30774e21680f5f26d75922919e3 (commit)
from 633660dcd81d4ddea5c0a55a7731af35c4586052 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 19656a41f019488120f950b06ecf9e19074b11a3
Merge: 633660d 2a610a9
Author: Brett Smith <brett at curoverse.com>
Date: Mon Mar 9 11:35:39 2015 -0400
Merge branch '5319-fix-hinted-collection-hashes-wip'
Closes #5319, #5369.
commit 2a610a9d5d7e9446935436ff37267b8ada25fdbd
Author: Brett Smith <brett at curoverse.com>
Date: Tue Mar 3 18:00:21 2015 -0500
5319: Add API migration for manifests with bad portable data hashes.
Refer to #5319 for background discussion and rationale. The migration
ensures that collections are still addressable by the bad
portable_data_hash, but the existing collection object has the correct
portable_data_hash.
diff --git a/services/api/db/migrate/20150303210106_fix_collection_portable_data_hash_with_hinted_manifest.rb b/services/api/db/migrate/20150303210106_fix_collection_portable_data_hash_with_hinted_manifest.rb
new file mode 100644
index 0000000..89d54a0
--- /dev/null
+++ b/services/api/db/migrate/20150303210106_fix_collection_portable_data_hash_with_hinted_manifest.rb
@@ -0,0 +1,111 @@
+require 'has_uuid'
+require 'kind_and_etag'
+
+class FixCollectionPortableDataHashWithHintedManifest < ActiveRecord::Migration
+ include CurrentApiClient
+
+ class ArvadosModel < ActiveRecord::Base
+ self.abstract_class = true
+ extend HasUuid::ClassMethods
+ include KindAndEtag
+ before_create do |record|
+ record.uuid ||= record.class.generate_uuid
+ record.owner_uuid ||= system_user_uuid
+ end
+ serialize :properties, Hash
+
+ def self.to_s
+ # Clean up the name of the stub model class so we generate correct UUIDs.
+ super.sub("FixCollectionPortableDataHashWithHintedManifest::", "")
+ end
+ end
+
+ class Collection < ArvadosModel
+ end
+
+ class Log < ArvadosModel
+ def self.log_for(thing, age="old")
+ { "#{age}_etag" => thing.etag,
+ "#{age}_attributes" => thing.attributes,
+ }
+ end
+
+ def self.log_create(thing)
+ new_log("create", thing, log_for(thing, "new"))
+ end
+
+ def self.log_update(thing, start_state)
+ new_log("update", thing, start_state.merge(log_for(thing, "new")))
+ end
+
+ def self.log_destroy(thing)
+ new_log("destroy", thing, log_for(thing, "old"))
+ end
+
+ private
+
+ def self.new_log(event_type, thing, properties)
+ create!(event_type: event_type,
+ event_at: Time.now,
+ object_uuid: thing.uuid,
+ object_owner_uuid: thing.owner_uuid,
+ properties: properties)
+ end
+ end
+
+ def each_bad_collection
+ Collection.find_each do |coll|
+ next unless (coll.manifest_text =~ /\+[A-Z]/)
+ stripped_manifest = coll.manifest_text.
+ gsub(/( [0-9a-f]{32}(\+\d+)?)(\+\S+)/, '\1')
+ stripped_pdh = sprintf("%s+%i",
+ Digest::MD5.hexdigest(stripped_manifest),
+ stripped_manifest.bytesize)
+ yield [coll, stripped_pdh] if (coll.portable_data_hash != stripped_pdh)
+ end
+ end
+
+ def up
+ Collection.reset_column_information
+ Log.reset_column_information
+ copied_attr_names =
+ [:owner_uuid, :created_at, :modified_by_client_uuid, :manifest_text,
+ :modified_by_user_uuid, :modified_at, :updated_at, :name,
+ :description, :portable_data_hash, :replication_desired,
+ :replication_confirmed, :replication_confirmed_at, :expires_at]
+ new_expiry = Date.new(2038, 1, 31)
+
+ each_bad_collection do |coll, stripped_pdh|
+ # Create a copy of the collection including bad portable data hash,
+ # with an expiration. This makes it possible to resolve the bad
+ # portable data hash, but the expiration can hide the Collection
+ # from more user-friendly interfaces like Workbench.
+ start_log = Log.log_for(coll)
+ attributes = Hash[copied_attr_names.map { |key| [key, coll.send(key)] }]
+ attributes[:expires_at] ||= new_expiry
+ attributes[:properties] = (coll.properties.dup rescue {})
+ attributes[:properties]["migrated_from"] ||= coll.uuid
+ coll_copy = Collection.create!(attributes)
+ Log.log_create(coll_copy)
+ coll.update_attributes(portable_data_hash: stripped_pdh)
+ Log.log_update(coll, start_log)
+ end
+ end
+
+ def down
+ Collection.reset_column_information
+ Log.reset_column_information
+ each_bad_collection do |coll, stripped_pdh|
+ if ((src_uuid = coll.properties["migrated_from"]) and
+ (src_coll = Collection.where(uuid: src_uuid).first) and
+ (src_coll.portable_data_hash == stripped_pdh))
+ start_log = Log.log_for(src_coll)
+ src_coll.portable_data_hash = coll.portable_data_hash
+ src_coll.save!
+ Log.log_update(src_coll, start_log)
+ coll.destroy or raise Exception.new("failed to destroy old collection")
+ Log.log_destroy(coll)
+ end
+ end
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index afc0351..007d05e 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -2360,4 +2360,6 @@ INSERT INTO schema_migrations (version) VALUES ('20150206210804');
INSERT INTO schema_migrations (version) VALUES ('20150206230342');
-INSERT INTO schema_migrations (version) VALUES ('20150216193428');
\ No newline at end of file
+INSERT INTO schema_migrations (version) VALUES ('20150216193428');
+
+INSERT INTO schema_migrations (version) VALUES ('20150303210106');
\ No newline at end of file
commit cf3a1032b04dd30774e21680f5f26d75922919e3
Author: Brett Smith <brett at curoverse.com>
Date: Tue Mar 3 17:36:24 2015 -0500
5319: Fix incorrect portable_data_hash in test fixture.
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 2201225..d8c4c44 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -434,7 +434,7 @@ replication_desired_2_confirmed_2:
created_at: 2015-02-07 00:19:28.596506247 Z
modified_by_user_uuid: zzzzz-tpzed-xurymjxw79nv3jz
modified_at: 2015-02-07 00:19:28.596338465 Z
- portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ portable_data_hash: ec53808e4c23e6aeebea24d998ae5346+88
replication_desired: 2
replication_confirmed_at: 2015-02-07 00:24:52.983381227 Z
replication_confirmed: 2
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list