[ARVADOS] created: 1.1.1-162-g2bd1ff1
Git user
git at public.curoverse.com
Thu Dec 7 10:26:09 EST 2017
at 2bd1ff1d4885786bf3ed838bcefb7198fcab6bc7 (commit)
commit 2bd1ff1d4885786bf3ed838bcefb7198fcab6bc7
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jul 11 11:14:01 2017 -0400
11908: Add comment about dropping/recreating FT index.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/api/db/migrate/20170706141334_json_collection_properties.rb b/services/api/db/migrate/20170706141334_json_collection_properties.rb
index 55ab0df..003e5fb 100644
--- a/services/api/db/migrate/20170706141334_json_collection_properties.rb
+++ b/services/api/db/migrate/20170706141334_json_collection_properties.rb
@@ -2,6 +2,9 @@ require './db/migrate/20161213172944_full_text_search_indexes'
class JsonCollectionProperties < ActiveRecord::Migration
def up
+ # Drop the FT index before changing column type to avoid
+ # "PG::DatatypeMismatch: ERROR: COALESCE types jsonb and text
+ # cannot be matched".
ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS collections_full_text_search_idx'
ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
FullTextSearchIndexes.new.replace_index('collections')
commit 5be901f5813fe9d026e2aa2a6a65d4ea4da4a001
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jul 11 10:25:16 2017 -0400
11908: Fix crash if index being deleted is already deleted.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/api/db/migrate/20170706141334_json_collection_properties.rb b/services/api/db/migrate/20170706141334_json_collection_properties.rb
index 35ee7ea..55ab0df 100644
--- a/services/api/db/migrate/20170706141334_json_collection_properties.rb
+++ b/services/api/db/migrate/20170706141334_json_collection_properties.rb
@@ -2,7 +2,7 @@ require './db/migrate/20161213172944_full_text_search_indexes'
class JsonCollectionProperties < ActiveRecord::Migration
def up
- ActiveRecord::Base.connection.execute 'DROP INDEX collections_full_text_search_idx'
+ ActiveRecord::Base.connection.execute 'DROP INDEX IF EXISTS collections_full_text_search_idx'
ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
FullTextSearchIndexes.new.replace_index('collections')
end
commit 267c1a22e4b9e04ed29d1360f4e7ea6519845d12
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jul 6 16:21:55 2017 -0400
11908: Migrate collections.properties to jsonb.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 0828d01..08d7e93 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -357,13 +357,14 @@ class ArvadosModel < ActiveRecord::Base
def self.full_text_searchable_columns
self.columns.select do |col|
- col.type == :string or col.type == :text
+ [:string, :text, :jsonb].include?(col.type)
end.map(&:name)
end
def self.full_text_tsvector
parts = full_text_searchable_columns.collect do |column|
- "coalesce(#{column},'')"
+ cast = serialized_attributes[column] ? '::text' : ''
+ "coalesce(#{column}#{cast},'')"
end
"to_tsvector('english', #{parts.join(" || ' ' || ")})"
end
diff --git a/services/api/db/migrate/20161213172944_full_text_search_indexes.rb b/services/api/db/migrate/20161213172944_full_text_search_indexes.rb
index 3f484b4..0ec5212 100644
--- a/services/api/db/migrate/20161213172944_full_text_search_indexes.rb
+++ b/services/api/db/migrate/20161213172944_full_text_search_indexes.rb
@@ -15,17 +15,18 @@ class FullTextSearchIndexes < ActiveRecord::Migration
}
end
+ def replace_index(t)
+ i = fts_indexes[t]
+ t.classify.constantize.reset_column_information
+ execute "DROP INDEX IF EXISTS #{i}"
+ execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector})"
+ end
+
def up
- # remove existing fts indexes and create up to date ones with no leading space
- fts_indexes.each do |t, i|
- t.classify.constantize.reset_column_information
- ActiveRecord::Base.connection.indexes(t).each do |idx|
- if idx.name == i
- remove_index t.to_sym, :name => i
- break
- end
- end
- execute "CREATE INDEX #{i} ON #{t} USING gin(#{t.classify.constantize.full_text_tsvector});"
+ # remove existing fts indexes and create up to date ones with no
+ # leading space
+ fts_indexes.keys.each do |t|
+ replace_index(t)
end
end
diff --git a/services/api/db/migrate/20170706141334_json_collection_properties.rb b/services/api/db/migrate/20170706141334_json_collection_properties.rb
new file mode 100644
index 0000000..35ee7ea
--- /dev/null
+++ b/services/api/db/migrate/20170706141334_json_collection_properties.rb
@@ -0,0 +1,13 @@
+require './db/migrate/20161213172944_full_text_search_indexes'
+
+class JsonCollectionProperties < ActiveRecord::Migration
+ def up
+ ActiveRecord::Base.connection.execute 'DROP INDEX collections_full_text_search_idx'
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE jsonb USING properties::jsonb'
+ FullTextSearchIndexes.new.replace_index('collections')
+ end
+
+ def down
+ ActiveRecord::Base.connection.execute 'ALTER TABLE collections ALTER COLUMN properties TYPE text'
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index a4b5e2e..d2da430 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -166,7 +166,7 @@ CREATE TABLE collections (
manifest_text text,
name character varying(255),
description character varying(524288),
- properties text,
+ properties jsonb,
delete_at timestamp without time zone,
file_names character varying(8192),
trash_at timestamp without time zone,
@@ -1607,7 +1607,7 @@ CREATE INDEX authorized_keys_search_index ON authorized_keys USING btree (uuid,
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
-CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE(properties, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
+CREATE INDEX collections_full_text_search_idx ON collections USING gin (to_tsvector('english'::regconfig, (((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || (COALESCE(file_names, ''::character varying))::text)));
--
@@ -3028,6 +3028,8 @@ INSERT INTO schema_migrations (version) VALUES ('20170628185847');
INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+INSERT INTO schema_migrations (version) VALUES ('20170706141334');
+
INSERT INTO schema_migrations (version) VALUES ('20170824202826');
INSERT INTO schema_migrations (version) VALUES ('20170906224040');
diff --git a/services/api/lib/serializers.rb b/services/api/lib/serializers.rb
index ec6fe19..37734e0 100644
--- a/services/api/lib/serializers.rb
+++ b/services/api/lib/serializers.rb
@@ -32,7 +32,10 @@ class Serializer
end
def self.load(s)
- if s.nil?
+ if s.is_a?(object_class)
+ # Rails already deserialized for us
+ s
+ elsif s.nil?
object_class.new()
elsif s[0] == first_json_char
SafeJSON.load(s)
diff --git a/services/api/test/unit/arvados_model_test.rb b/services/api/test/unit/arvados_model_test.rb
index c6b76c4..9230838 100644
--- a/services/api/test/unit/arvados_model_test.rb
+++ b/services/api/test/unit/arvados_model_test.rb
@@ -258,4 +258,24 @@ class ArvadosModelTest < ActiveSupport::TestCase
assert_equal true, (updated_at_2 > updated_at_1), "Expected updated time 2 to be newer than 1"
end
+
+ test 'jsonb column' do
+ set_user_from_auth :active
+
+ c = Collection.create!(properties: {})
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {'foo' => 'foo'})
+ c.reload
+ assert_equal({'foo' => 'foo'}, c.properties)
+
+ c.update_attributes(properties: nil)
+ c.reload
+ assert_equal({}, c.properties)
+
+ c.update_attributes(properties: {foo: 'bar'})
+ assert_equal({'foo' => 'bar'}, c.properties)
+ c.reload
+ assert_equal({'foo' => 'bar'}, c.properties)
+ end
end
commit e18cf2e1fb88278e4fb31af59e8d944576e31391
Author: Tom Clegg <tom at curoverse.com>
Date: Tue Jul 4 22:14:23 2017 -0400
11807: Migrate data from YAML to JSON.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curoverse.com>
diff --git a/services/api/db/migrate/20170704160233_yaml_to_json.rb b/services/api/db/migrate/20170704160233_yaml_to_json.rb
new file mode 100644
index 0000000..dfa08db
--- /dev/null
+++ b/services/api/db/migrate/20170704160233_yaml_to_json.rb
@@ -0,0 +1,37 @@
+require 'migrate_yaml_to_json'
+
+class YamlToJson < ActiveRecord::Migration
+ def up
+ [
+ ['collections', 'properties'],
+ ['containers', 'environment'],
+ ['containers', 'mounts'],
+ ['containers', 'runtime_constraints'],
+ ['containers', 'command'],
+ ['containers', 'scheduling_parameters'],
+ ['container_requests', 'properties'],
+ ['container_requests', 'environment'],
+ ['container_requests', 'mounts'],
+ ['container_requests', 'runtime_constraints'],
+ ['container_requests', 'command'],
+ ['container_requests', 'scheduling_parameters'],
+ ['humans', 'properties'],
+ ['job_tasks', 'parameters'],
+ ['links', 'properties'],
+ ['nodes', 'info'],
+ ['nodes', 'properties'],
+ ['pipeline_instances', 'components'],
+ ['pipeline_instances', 'properties'],
+ ['pipeline_instances', 'components_summary'],
+ ['pipeline_templates', 'components'],
+ ['specimens', 'properties'],
+ ['traits', 'properties'],
+ ['users', 'prefs'],
+ ].each do |table, column|
+ MigrateYAMLToJSON.migrate(table, column)
+ end
+ end
+
+ def down
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index e062d43..a4b5e2e 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -3026,9 +3026,10 @@ INSERT INTO schema_migrations (version) VALUES ('20170419175801');
INSERT INTO schema_migrations (version) VALUES ('20170628185847');
+INSERT INTO schema_migrations (version) VALUES ('20170704160233');
+
INSERT INTO schema_migrations (version) VALUES ('20170824202826');
INSERT INTO schema_migrations (version) VALUES ('20170906224040');
INSERT INTO schema_migrations (version) VALUES ('20171027183824');
-
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list