[ARVADOS] updated: 1.1.1-216-g79a2d81
Git user
git at public.curoverse.com
Tue Dec 12 10:41:15 EST 2017
Summary of changes:
doc/api/methods.html.textile.liquid | 27 ++++++++++++++----
services/api/app/models/arvados_model.rb | 2 +-
...53352_add_gin_index_to_collection_properties.rb | 8 ++++++
services/api/db/structure.sql | 9 ++++++
services/api/lib/record_filters.rb | 32 ++++++++++++++++------
services/api/test/fixtures/collections.yml | 14 ++++++++++
.../api/test/functional/arvados/v1/filters_test.rb | 24 +++++++++++++---
services/api/test/unit/arvados_model_test.rb | 2 +-
8 files changed, 97 insertions(+), 21 deletions(-)
create mode 100644 services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb
via 79a2d819d596e610f26c08beee53f5432bfbb360 (commit)
via 9ec15072e6631578584bd9c08d26025e807a8d48 (commit)
via f9a3dea1aed09b7b1d885c3116c63310e0440007 (commit)
from 40d5d40955a88dbd5cd7c25292268f1ab4536bda (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 79a2d819d596e610f26c08beee53f5432bfbb360
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Dec 12 10:40:40 2017 -0500
4019: Add index for efficient jsonb query on collection properties
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb b/services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb
new file mode 100644
index 0000000..ce2403e
--- /dev/null
+++ b/services/api/db/migrate/20171212153352_add_gin_index_to_collection_properties.rb
@@ -0,0 +1,8 @@
+class AddGinIndexToCollectionProperties < ActiveRecord::Migration
+ def up
+ ActiveRecord::Base.connection.execute("CREATE INDEX collection_index_on_properties ON collections USING gin (properties);")
+ end
+ def down
+ ActiveRecord::Base.connection.execute("DROP INDEX collection_index_on_properties")
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 8f405c0..14729d3 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -1606,6 +1606,13 @@ CREATE INDEX authorized_keys_search_index ON authorized_keys USING btree (uuid,
--
+-- Name: collection_index_on_properties; Type: INDEX; Schema: public; Owner: -
+--
+
+CREATE INDEX collection_index_on_properties ON collections USING gin (properties);
+
+
+--
-- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
--
@@ -3040,3 +3047,5 @@ INSERT INTO schema_migrations (version) VALUES ('20171027183824');
INSERT INTO schema_migrations (version) VALUES ('20171208203841');
+INSERT INTO schema_migrations (version) VALUES ('20171212153352');
+
commit 9ec15072e6631578584bd9c08d26025e807a8d48
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Tue Dec 12 10:32:15 2017 -0500
4019: Add URI quoting option for keys. Update docs. Fix tests.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index 6f7eca7..00c120d 100644
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -100,7 +100,7 @@ table(table table-bordered table-condensed).
h4. Filtering on subproperties
-Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.<user_property>@.
+Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.user_property at . You may also use JSON-LD / RDF style URIs for property keys by enclosing them in @<...>@ for example @properties.<http://example.com/user_property>@. Alternately you may also provide a JSON-LD "@context" field, however at this time JSON-LD contexts are not interpreted by Arvados.
table(table table-bordered table-condensed).
|_. Operator|_. Operand type|_. Description|_. Example|
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 72db306..08d7e93 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -132,7 +132,7 @@ class ArvadosModel < ActiveRecord::Base
textonly_operator = !operator.match(/[<=>]/)
self.columns.select do |col|
case col.type
- when :string, :text, :jsonb
+ when :string, :text
true
when :datetime, :integer, :boolean
!textonly_operator
diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 56e1649..900b784 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -63,11 +63,17 @@ module RecordFilters
attrs.each do |attr|
subproperty = attr.split(".", 2)
- if !model_class.searchable_columns(operator).index subproperty[0]
- raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' in filter")
- end
+ col = model_class.columns.select { |c| c.name == subproperty[0] }.first
if subproperty.length == 2
+ if col.type != :jsonb
+ raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' for operator '#{operator}' in filter")
+ end
+
+ if subproperty[1][0] == "<" and subproperty[1][-1] == ">"
+ subproperty[1] = subproperty[1][1..-2]
+ end
+
# jsonb search
case operator.downcase
when '=', '!='
@@ -112,7 +118,18 @@ module RecordFilters
else
raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
end
+ elsif operator.downcase == "exists"
+ if col.type != :jsonb
+ raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' for operator '#{operator}' in filter")
+ end
+
+ cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+ param_out << operand
else
+ if !model_class.searchable_columns(operator).index subproperty[0]
+ raise ArgumentError.new("Invalid attribute '#{subproperty[0]}' in filter")
+ end
+
case operator.downcase
when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
attr_type = model_class.attribute_column(attr).type
@@ -185,9 +202,6 @@ module RecordFilters
end
end
cond_out << cond.join(' OR ')
- when 'exists'
- cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
- param_out << operand
else
raise ArgumentError.new("Invalid operator '#{operator}'")
end
diff --git a/services/api/test/fixtures/collections.yml b/services/api/test/fixtures/collections.yml
index 871a357..ea87cca 100644
--- a/services/api/test/fixtures/collections.yml
+++ b/services/api/test/fixtures/collections.yml
@@ -799,6 +799,20 @@ collection_with_prop2_5:
properties:
prop2: 5
+collection_with_uri_prop:
+ uuid: zzzzz-4zz18-withuripropval1
+ portable_data_hash: fa7aeb5140e2848d39b416daeef4ffc5+45
+ owner_uuid: zzzzz-tpzed-xurymjxw79nv3jz
+ created_at: 2015-02-13T17:22:54Z
+ modified_by_client_uuid: zzzzz-ozdt8-brczlopd8u8d0jr
+ modified_by_user_uuid: zzzzz-tpzed-d9tiejq69daie8f
+ modified_at: 2015-02-13T17:22:54Z
+ updated_at: 2015-02-13T17:22:54Z
+ manifest_text: ". 37b51d194a7513e45b56f6524f2d51f2+3 0:3:bar\n"
+ name: collection with RDF-style URI property key
+ properties:
+ "http://schema.org/example": "value1"
+
# Test Helper trims the rest of the file
# Do not add your fixtures below this line as the rest of this file will be trimmed by test_helper
diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index 18f3c6b..677a5f1 100644
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -170,7 +170,8 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
['prop2', '>', 1, [:collection_with_prop2_5], [:collection_with_prop2_1]],
['prop2', '<', 5, [:collection_with_prop2_1], [:collection_with_prop2_5]],
['prop2', '<=', 5, [:collection_with_prop2_1, :collection_with_prop2_5], []],
- ['prop2', '>=', 1, [:collection_with_prop2_1, :collection_with_prop2_5], []]
+ ['prop2', '>=', 1, [:collection_with_prop2_1, :collection_with_prop2_5], []],
+ ['<http://schema.org/example>', '=', "value1", [:collection_with_uri_prop], []],
].each do |prop, op, opr, inc, ex|
test "jsonb filter properties.#{prop} #{op} #{opr})" do
@controller = Arvados::V1::CollectionsController.new
diff --git a/services/api/test/unit/arvados_model_test.rb b/services/api/test/unit/arvados_model_test.rb
index 9230838..d74cbc5 100644
--- a/services/api/test/unit/arvados_model_test.rb
+++ b/services/api/test/unit/arvados_model_test.rb
@@ -147,7 +147,7 @@ class ArvadosModelTest < ActiveSupport::TestCase
search_index_columns = table_class.searchable_columns('ilike')
# Disappointing, but text columns aren't indexed yet.
search_index_columns -= table_class.columns.select { |c|
- c.type == :text or c.name == 'description' or c.name == 'file_names'
+ c.type == :text or c.name == 'description' or c.name == 'file_names' or c.name == 'properties'
}.collect(&:name)
indexes = ActiveRecord::Base.connection.indexes(table)
commit f9a3dea1aed09b7b1d885c3116c63310e0440007
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Mon Dec 11 17:50:00 2017 -0500
4019: Update docs. Tweak syntax of 'exists' and add alternate form.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index a24f34c..6f7eca7 100644
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -90,12 +90,27 @@ table(table table-bordered table-condensed).
The following operators are available.
table(table table-bordered table-condensed).
-|_. Operator|_. Operand type|_. Example|
-|@<@, @<=@, @>=@, @>@, @like@, @ilike@|string|@["script_version","like","d00220fb%"]@|
-|@=@, @!=@|string or null|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@
-@["tail_uuid","!=",null]@|
-|@in@, @not in@|array of strings|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
-|@is_a@|string|@["head_uuid","is_a","arvados#pipelineInstance"]@|
+|_. Operator|_. Operand type|_. Description|_. Example|
+|@=@, @!=@|string, number, timestamp, or null|Equality comparison|@["tail_uuid","=","xyzzy-j7d0g-fffffffffffffff"]@ @["tail_uuid","!=",null]@|
+|@<@, @<=@, @>=@, @>@|string, number, or timestamp|Ordering comparison|@["script_version",">","123"]@|
+|@like@, @ilike@|string|SQL pattern match. Single character match is @_@ and wildcard is @%@. The @ilike@ operator is case-insensitive|@["script_version","like","d00220fb%"]@|
+|@in@, @not in@|array of strings|Set membership|@["script_version","in",["master","d00220fb38d4b85ca8fc28a8151702a2b9d1dec5"]]@|
+|@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
+|@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
+
+h4. Filtering on subproperties
+
+Some record type have an additional @properties@ attribute that allows recording and filtering on additional key-value pairs. To filter on a subproperty, the value in the @attribute@ position has the form @properties.<user_property>@.
+
+table(table table-bordered table-condensed).
+|_. Operator|_. Operand type|_. Description|_. Example|
+|@=@, @!=@|string, number or boolean|Equality comparison|@["properties.my_subproperty", "=", "fizzy whizy sparkle pop"]@|
+|@<@, @<=@, @>=@, @>@|string or number|Ordering comparison|@["properties.my_subproperty", "<", 3]@|
+|@like@, @ilike@|string|SQL pattern match, single character match is @_@ and wildcard is @%@, ilike is case-insensitive|@["properties.my_subproperty", "like", "d00220fb%"]@|
+|@in@, @not in@|array of strings|Set membership|@["properties.my_subproperty", "in", ["fizz", "buzz"]]@|
+|@exists@|boolean|Test if a subproperty is present or not (determined by operand).|@["properties.my_subproperty", "exists", true]@|
+
+Note that exclusion filters @!=@ and @not in@ will return records for which the property is not defined at all. To restrict filtering to records on which the subproperty is defined, combine with an @exists@ filter.
h3. Results of list method
diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 9ff21d5..56e1649 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -102,14 +102,11 @@ module RecordFilters
raise ArgumentError.new("Invalid operand type '#{operand.class}' "\
"for '#{operator}' operator in filters")
end
- when 'exists', 'not exists'
+ when 'exists'
if operand
- raise ArgumentError.new("Invalid operand for subproperty existence filter, should be empty or null")
- end
- if operator.downcase[0..2] == "not" then
- cond_out << "(NOT jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)) OR #{ar_table_name}.#{subproperty[0]} is NULL"
- else
cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+ else
+ cond_out << "(NOT jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)) OR #{ar_table_name}.#{subproperty[0]} is NULL"
end
param_out << subproperty[1]
else
@@ -188,6 +185,9 @@ module RecordFilters
end
end
cond_out << cond.join(' OR ')
+ when 'exists'
+ cond_out << "jsonb_exists(#{ar_table_name}.#{subproperty[0]}, ?)"
+ param_out << operand
else
raise ArgumentError.new("Invalid operator '#{operator}'")
end
diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index 6b4ea2a..18f3c6b 100644
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -154,8 +154,8 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
[['prop1', '=', 'value1', [:collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop2_1]],
['prop1', '!=', 'value1', [:collection_with_prop1_value2, :collection_with_prop2_1], [:collection_with_prop1_value1]],
- ['prop1', 'exists', nil, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
- ['prop1', 'not exists', nil, [:collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1]],
+ ['prop1', 'exists', true, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
+ ['prop1', 'exists', false, [:collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1]],
['prop1', 'in', ['value1', 'value2'], [:collection_with_prop1_value1, :collection_with_prop1_value2], [:collection_with_prop1_value3, :collection_with_prop2_1]],
['prop1', 'in', ['value1', 'valueX'], [:collection_with_prop1_value1], [:collection_with_prop1_value3, :collection_with_prop2_1]],
['prop1', 'not in', ['value1', 'value2'], [:collection_with_prop1_value3, :collection_with_prop1_other1, :collection_with_prop2_1], [:collection_with_prop1_value1, :collection_with_prop1_value2]],
@@ -196,7 +196,22 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
@controller = Arvados::V1::CollectionsController.new
authorize_with :admin
get :index, {
- filters: [ ['properties.prop1', 'exists', nil], ['properties.prop1', '!=', 'value1'] ]
+ filters: [ ['properties.prop1', 'exists', true], ['properties.prop1', '!=', 'value1'] ]
+ }
+ assert_response :success
+ found = assigns(:objects).collect(&:uuid)
+ assert_equal found.length, 3
+ assert_not_includes(found, collections(:collection_with_prop1_value1).uuid)
+ assert_includes(found, collections(:collection_with_prop1_value2).uuid)
+ assert_includes(found, collections(:collection_with_prop1_value3).uuid)
+ assert_includes(found, collections(:collection_with_prop1_other1).uuid)
+ end
+
+ test "jsonb alternate form 'exists' and '!=' filter" do
+ @controller = Arvados::V1::CollectionsController.new
+ authorize_with :admin
+ get :index, {
+ filters: [ ['properties', 'exists', 'prop1'], ['properties.prop1', '!=', 'value1'] ]
}
assert_response :success
found = assigns(:objects).collect(&:uuid)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list