[ARVADOS] updated: 2.1.0-1170-g6f06db5ea
Git user
git at public.arvados.org
Thu Aug 12 15:03:58 UTC 2021
Summary of changes:
services/api/lib/record_filters.rb | 21 ++----
.../api/test/functional/arvados/v1/filters_test.rb | 76 +---------------------
.../api/test/integration/collections_api_test.rb | 69 --------------------
services/api/test/integration/groups_test.rb | 40 ------------
4 files changed, 6 insertions(+), 200 deletions(-)
via 6f06db5ead38db82e7ab1ca50cb7a12c9befe54c (commit)
from e7f55069386e86136f058ae35f2de4505fd1458b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6f06db5ead38db82e7ab1ca50cb7a12c9befe54c
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 12 12:03:30 2021 -0300
17948: Removes support for the '@@' full text search operator.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 5688ca614..301524a6c 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -31,7 +31,10 @@ module RecordFilters
model_table_name = model_class.table_name
filters.each do |filter|
attrs_in, operator, operand = filter
- if attrs_in == 'any' && operator != '@@'
+ if operator == '@@'
+ raise ArgumentError.new("Full text search is no longer supported")
+ end
+ if attrs_in == 'any'
attrs = model_class.searchable_columns(operator)
elsif attrs_in.is_a? Array
attrs = attrs_in
@@ -54,22 +57,6 @@ module RecordFilters
attrs = []
end
- if operator == '@@'
- # Full-text search
- if attrs_in != 'any'
- raise ArgumentError.new("Full text search on individual columns is not supported")
- end
- if operand.is_a? Array
- raise ArgumentError.new("Full text search not supported for array operands")
- end
-
- # Skip the generic per-column operator loop below
- attrs = []
- # Use to_tsquery since plainto_tsquery does not support prefix
- # search. And, split operand and join the words with ' & '
- cond_out << model_class.full_text_tsvector+" @@ to_tsquery(?)"
- param_out << operand.split.join(' & ')
- end
attrs.each do |attr|
subproperty = attr.split(".", 2)
diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index 26270b1c3..bcb180786 100644
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -29,34 +29,14 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
json_response['errors'].join(' '))
end
- test 'error message for full text search on a specific column' do
+ test 'error message for unsupported full text search' do
@controller = Arvados::V1::CollectionsController.new
authorize_with :active
get :index, params: {
filters: [['uuid', '@@', 'abcdef']],
}
assert_response 422
- assert_match(/not supported/, json_response['errors'].join(' '))
- end
-
- test 'difficult characters in full text search' do
- @controller = Arvados::V1::CollectionsController.new
- authorize_with :active
- get :index, params: {
- filters: [['any', '@@', 'a|b"c']],
- }
- assert_response :success
- # (Doesn't matter so much which results are returned.)
- end
-
- test 'array operand in full text search' do
- @controller = Arvados::V1::CollectionsController.new
- authorize_with :active
- get :index, params: {
- filters: [['any', '@@', ['abc', 'def']]],
- }
- assert_response 422
- assert_match(/not supported/, json_response['errors'].join(' '))
+ assert_match(/no longer supported/, json_response['errors'].join(' '))
end
test 'api responses provide timestamps with nanoseconds' do
@@ -100,58 +80,6 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
end
end
- test "full text search with count='none'" do
- @controller = Arvados::V1::GroupsController.new
- authorize_with :admin
-
- get :contents, params: {
- format: :json,
- count: 'none',
- limit: 1000,
- filters: [['any', '@@', Rails.configuration.ClusterID]],
- }
-
- assert_response :success
-
- all_objects = Hash.new(0)
- json_response['items'].map{|o| o['kind']}.each{|t| all_objects[t] += 1}
-
- assert_equal true, all_objects['arvados#group']>0
- assert_equal true, all_objects['arvados#job']>0
- assert_equal true, all_objects['arvados#pipelineInstance']>0
- assert_equal true, all_objects['arvados#pipelineTemplate']>0
-
- # Perform test again mimicking a second page request with:
- # last_object_class = PipelineInstance
- # and hence groups and jobs should not be included in the response
- # offset = 5, which means first 5 pipeline instances were already received in page 1
- # and hence the remaining pipeline instances and all other object types should be included in the response
-
- @test_counter = 0 # Reset executed action counter
-
- @controller = Arvados::V1::GroupsController.new
-
- get :contents, params: {
- format: :json,
- count: 'none',
- limit: 1000,
- offset: '5',
- last_object_class: 'PipelineInstance',
- filters: [['any', '@@', Rails.configuration.ClusterID]],
- }
-
- assert_response :success
-
- second_page = Hash.new(0)
- json_response['items'].map{|o| o['kind']}.each{|t| second_page[t] += 1}
-
- assert_equal false, second_page.include?('arvados#group')
- assert_equal false, second_page.include?('arvados#job')
- assert_equal true, second_page['arvados#pipelineInstance']>0
- assert_equal all_objects['arvados#pipelineInstance'], second_page['arvados#pipelineInstance']+5
- assert_equal true, second_page['arvados#pipelineTemplate']>0
- end
-
[['prop1', '=', 'value1', [:collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop2_1]],
['prop1', '!=', 'value1', [:collection_with_prop1_value2, :collection_with_prop2_1], [:collection_with_prop1_value1]],
['prop1', 'exists', true, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
diff --git a/services/api/test/integration/collections_api_test.rb b/services/api/test/integration/collections_api_test.rb
index 73cbad643..070e964e5 100644
--- a/services/api/test/integration/collections_api_test.rb
+++ b/services/api/test/integration/collections_api_test.rb
@@ -373,75 +373,6 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
end
end
- test "search collection using full text search" do
- # create collection to be searched for
- signed_manifest = Collection.sign_manifest(". 85877ca2d7e05498dd3d109baf2df106+95+A3a4e26a366ee7e4ed3e476ccf05354761be2e4ae at 545a9920 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64 at 545a9920 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64 at 545a9920 0:32:file3_in_subdir4.txt 32:32:file4_in_subdir4.txt\n", api_token(:active))
- post "/arvados/v1/collections",
- params: {
- format: :json,
- collection: {description: 'specific collection description', manifest_text: signed_manifest}.to_json,
- },
- headers: auth(:active)
- assert_response :success
- assert_equal true, json_response['manifest_text'].include?('file4_in_subdir4.txt')
-
- # search using the filename
- search_using_full_text_search 'subdir2', 0
- search_using_full_text_search 'subdir2:*', 1
- search_using_full_text_search 'subdir2/subdir3/subdir4', 1
- search_using_full_text_search 'file4:*', 1
- search_using_full_text_search 'file4_in_subdir4.txt', 1
- search_using_full_text_search 'subdir2 file4:*', 0 # first word is incomplete
- search_using_full_text_search 'subdir2/subdir3/subdir4 file4:*', 1
- search_using_full_text_search 'subdir2/subdir3/subdir4 file4_in_subdir4.txt', 1
- search_using_full_text_search 'ile4', 0 # not a prefix match
- end
-
- def search_using_full_text_search search_filter, expected_items
- get '/arvados/v1/collections',
- params: {:filters => [['any', '@@', search_filter]].to_json},
- headers: auth(:active)
- assert_response :success
- response_items = json_response['items']
- assert_not_nil response_items
- if expected_items == 0
- assert_empty response_items
- else
- refute_empty response_items
- first_item = response_items.first
- assert_not_nil first_item
- end
- end
-
- # search for the filename in the file_names column and expect error
- test "full text search not supported for individual columns" do
- get '/arvados/v1/collections',
- params: {:filters => [['name', '@@', 'General']].to_json},
- headers: auth(:active)
- assert_response 422
- end
-
- [
- 'quick fox',
- 'quick_brown fox',
- 'brown_ fox',
- 'fox dogs',
- ].each do |search_filter|
- test "full text search ignores special characters and finds with filter #{search_filter}" do
- # description: The quick_brown_fox jumps over the lazy_dog
- # full text search treats '_' as space apparently
- get '/arvados/v1/collections',
- params: {:filters => [['any', '@@', search_filter]].to_json},
- headers: auth(:active)
- assert_response 200
- response_items = json_response['items']
- assert_not_nil response_items
- first_item = response_items.first
- refute_empty first_item
- assert_equal first_item['description'], 'The quick_brown_fox jumps over the lazy_dog'
- end
- end
-
test "create and get collection with properties" do
# create collection to be searched for
signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
diff --git a/services/api/test/integration/groups_test.rb b/services/api/test/integration/groups_test.rb
index aa67166f7..e76f2b540 100644
--- a/services/api/test/integration/groups_test.rb
+++ b/services/api/test/integration/groups_test.rb
@@ -64,46 +64,6 @@ class GroupsTest < ActionDispatch::IntegrationTest
end
end
- [
- ['Collection_', true], # collections and pipelines templates
- ['hash', true], # pipeline templates
- ['fa7aeb5140e2848d39b', false], # script_parameter of pipeline instances
- ['fa7aeb5140e2848d39b:*', true], # script_parameter of pipeline instances
- ['project pipeline', true], # finds "Completed pipeline in A Project"
- ['project pipeli:*', true], # finds "Completed pipeline in A Project"
- ['proje pipeli:*', false], # first word is incomplete, so no prefix match
- ['no-such-thing', false], # script_parameter of pipeline instances
- ].each do |search_filter, expect_results|
- test "full text search of group-owned objects for #{search_filter}" do
- get "/arvados/v1/groups/contents",
- params: {
- id: groups(:aproject).uuid,
- limit: 5,
- :filters => [['any', '@@', search_filter]].to_json
- },
- headers: auth(:active)
- assert_response :success
- if expect_results
- refute_empty json_response['items']
- json_response['items'].each do |item|
- assert item['uuid']
- assert_equal groups(:aproject).uuid, item['owner_uuid']
- end
- else
- assert_empty json_response['items']
- end
- end
- end
-
- test "full text search is not supported for individual columns" do
- get "/arvados/v1/groups/contents",
- params: {
- :filters => [['name', '@@', 'Private']].to_json
- },
- headers: auth(:active)
- assert_response 422
- end
-
test "group contents with include trash collections" do
get "/arvados/v1/groups/contents",
params: {
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list