[ARVADOS] created: bb821d03eb10ddcc7822fac51a565d1a11082ebc
Git user
git at public.curoverse.com
Thu Jul 6 17:30:54 EDT 2017
at bb821d03eb10ddcc7822fac51a565d1a11082ebc (commit)
commit bb821d03eb10ddcc7822fac51a565d1a11082ebc
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 6 16:21:55 2017 -0400
11908: Migrate collections.properties to jsonb.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index ea69735..489c4ae 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -354,13 +354,14 @@ class ArvadosModel < ActiveRecord::Base
def self.full_text_searchable_columns
self.columns.select do |col|
- col.type == :string or col.type == :text
+ [:string, :text, :jsonb].include?(col.type)
end.map(&:name)
end
def self.full_text_tsvector
parts = full_text_searchable_columns.collect do |column|
- "coalesce(#{column},'')"
+ cast = serialized_attributes[column] ? '::text' : ''
+ "coalesce(#{column}#{cast},'')"
end
"to_tsvector('english', #{parts.join(" || ' ' || ")})"
end
diff --git a/services/api/db/migrate/20161213172944_full_text_search_indexes.rb b/services/api/db/migrate/20161213172944_full_text_search_indexes.rb
index 3f484b4..0ec5212 100644
--- a/services/api/db/migrate/20161213172944_full_text_search_indexes.rb
+++ b/services/api/db/migrate/20161213172944_full_text_search_indexes.rb
@@ -15,17 +15,18 @@ class FullTextSearchIndexes < ActiveRecord::Migration
}
end
+ def replace_index(t)
+ i = fts_indexes[t]
+ t.classify.constantize.reset_column_information
+ execute "DROP INDEX IF EXISTS #{i}"
+ execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector})"
+ end
+
def up
- # remove existing fts indexes and create up to date ones with no leading space
- fts_indexes.each do |t, i|
- t.classify.constantize.reset_column_information
- ActiveRecord::Base.connection.indexes(t).each do |idx|
- if idx.name == i
- remove_index t.to_sym, :name => i
- break
- end
- end
- execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector});"
+ # remove existing fts indexes and create up to date ones with no
+ # leading space
+ fts_indexes.keys.each do |t|
+ replace_index(t)
end
end
diff --git a/services/api/db/migrate/20170706141334_json_collection_properties.rb b/services/api/db/migrate/20170706141334_json_collection_properties.rb
new file mode 100644
index 0000000..35ee7ea
--- /dev/null
+++ b/services/api/db/migrate/20170706141334_json_collection_properties.rb
@@ -0,0 +1,13 @@
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class JsonCollectionProperties < ActiveRecord::Migration
+ def up
+ ActiveRecord::Base.connection.execute 'DROP INDEX collections_full_text_search_idx'
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
+ FullTextSearchIndexes.new.replace_index('collections')
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE text'
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index a10baab..77f8528 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -170,7 +170,7 @@ CREATE TABLE collections (
manifest_text text,
name character varying(255),
description character varying(524288),
- properties text,
+ properties jsonb,
delete_at timestamp without time zone,
file_names character varying(8192),
trash_at timestamp without time zone,
@@ -1505,7 +1505,7 @@ CREATE INDEX authorized_keys_search_index ON authorized_keys USING btree (uuid,
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -; Tablespace:
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
--
@@ -2793,3 +2793,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170628185847');
INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+INSERT INTO schema_migrations (version) VALUES ('20170706141334');
+
diff --git a/services/api/lib/serializers.rb b/services/api/lib/serializers.rb
index ec6fe19..37734e0 100644
--- a/services/api/lib/serializers.rb
+++ b/services/api/lib/serializers.rb
@@ -32,7 +32,10 @@ class Serializer
end
def self.load(s)
- if s.nil?
+ if s.is_a?(object_class)
+ # Rails already deserialized for us
+ s
+ elsif s.nil?
object_class.new()
elsif s[0] == first_json_char
SafeJSON.load(s)
diff --git a/services/api/test/unit/arvados_model_test.rb b/services/api/test/unit/arvados_model_test.rb
index 1e26798..678d2e8 100644
--- a/services/api/test/unit/arvados_model_test.rb
+++ b/services/api/test/unit/arvados_model_test.rb
@@ -257,4 +257,24 @@ class ArvadosModelTest < ActiveSupport::TestCase
assert_equal true, (updated_at_2 > updated_at_1), "Expected updated time 2 to be newer than 1"
end
+
+ test 'jsonb column' do
+ set_user_from_auth :active
+
+ c = Collection.create!(properties: {})
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {'foo' => 'foo'})
+ c.reload
+ assert_equal({'foo' => 'foo'}, c.properties)
+
+ c.update_attributes(properties: nil)
+ c.reload
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {foo: 'bar'})
+ assert_equal({'foo' => 'bar'}, c.properties)
+ c.reload
+ assert_equal({'foo' => 'bar'}, c.properties)
+ end
end
commit 8e92b7b10f2a7b3b775a4a16f73700228876206d
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jul 4 22:14:23 2017 -0400
11807: Migrate data from YAML to JSON.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/api/db/migrate/20170704160233_yaml_to_json.rb b/services/api/db/migrate/20170704160233_yaml_to_json.rb
new file mode 100644
index 0000000..dfa08db
--- /dev/null
+++ b/services/api/db/migrate/20170704160233_yaml_to_json.rb
@@ -0,0 +1,37 @@
+require 'migrate_yaml_to_json'
+
+class YamlToJson < ActiveRecord::Migration
+ def up
+ [
+ ['collections', 'properties'],
+ ['containers', 'environment'],
+ ['containers', 'mounts'],
+ ['containers', 'runtime_constraints'],
+ ['containers', 'command'],
+ ['containers', 'scheduling_parameters'],
+ ['container_requests', 'properties'],
+ ['container_requests', 'environment'],
+ ['container_requests', 'mounts'],
+ ['container_requests', 'runtime_constraints'],
+ ['container_requests', 'command'],
+ ['container_requests', 'scheduling_parameters'],
+ ['humans', 'properties'],
+ ['job_tasks', 'parameters'],
+ ['links', 'properties'],
+ ['nodes', 'info'],
+ ['nodes', 'properties'],
+ ['pipeline_instances', 'components'],
+ ['pipeline_instances', 'properties'],
+ ['pipeline_instances', 'components_summary'],
+ ['pipeline_templates', 'components'],
+ ['specimens', 'properties'],
+ ['traits', 'properties'],
+ ['users', 'prefs'],
+ ].each do |table, column|
+ MigrateYAMLToJSON.migrate(table, column)
+ end
+ end
+
+ def down
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 7e6bb1d..a10baab 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -2791,3 +2791,5 @@ INSERT INTO schema_migrations (version) VALUES ('20170419175801');
INSERT INTO schema_migrations (version) VALUES ('20170628185847');
+INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list