[ARVADOS] created: 2.1.0-1188-g7e46e3c5d
Git user
git at public.arvados.org
Thu Aug 12 22:36:17 UTC 2021
at 7e46e3c5dd416bb3e4fc59a79e7b846e70c87e2f (commit)
commit 7e46e3c5dd416bb3e4fc59a79e7b846e70c87e2f
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 12 19:35:54 2021 -0300
15430: WIP
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/apps/workbench/test/controllers/work_units_controller_test.rb b/apps/workbench/test/controllers/work_units_controller_test.rb
index f9648c503..f471dddea 100644
--- a/apps/workbench/test/controllers/work_units_controller_test.rb
+++ b/apps/workbench/test/controllers/work_units_controller_test.rb
@@ -13,7 +13,7 @@ class WorkUnitsControllerTest < ActionController::TestCase
[
['foo', 10, 25,
['/pipeline_instances/zzzzz-d1hrv-1xfj6xkicf2muk2',
- # '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
+ '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
'/jobs/zzzzz-8i9sb-grx15v5mjnsyxk7'],
['/pipeline_instances/zzzzz-d1hrv-1yfj61234abcdk3',
'/jobs/zzzzz-8i9sb-n7omg50bvt0m1nf',
@@ -21,20 +21,20 @@ class WorkUnitsControllerTest < ActionController::TestCase
['pipeline_with_tagged_collection_input', 1, 1,
['/pipeline_instances/zzzzz-d1hrv-1yfj61234abcdk3'],
[
- # '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
+ '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
'/jobs/zzzzz-8i9sb-pshmckwoma9plh7',
'/jobs/zzzzz-8i9sb-n7omg50bvt0m1nf',
'/container_requests/zzzzz-xvhdp-cr4completedcr2']],
['no_such_match', 0, 0,
[],
[
- # '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
+ '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
'/jobs/zzzzz-8i9sb-pshmckwoma9plh7',
'/jobs/zzzzz-8i9sb-n7omg50bvt0m1nf',
'/container_requests/zzzzz-xvhdp-cr4completedcr2']],
].each do |search_filter, expected_min, expected_max, expected, not_expected|
test "all_processes page for search filter '#{search_filter}'" do
- work_units_index(filters: [['any','ilike', "%#{search_filter}%"]], show_children: true)
+ work_units_index(filters: [['any','ilike', search_filter]], show_children: true)
assert_response :success
# Verify that expected number of processes are found
commit 255ff09028b10538159b09bf2a3072a7e7897505
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 12 14:18:26 2021 -0300
15430: Testing ilike operator test updates (WIP)
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/apps/workbench/test/controllers/work_units_controller_test.rb b/apps/workbench/test/controllers/work_units_controller_test.rb
index 375c65c9b..f9648c503 100644
--- a/apps/workbench/test/controllers/work_units_controller_test.rb
+++ b/apps/workbench/test/controllers/work_units_controller_test.rb
@@ -13,20 +13,22 @@ class WorkUnitsControllerTest < ActionController::TestCase
[
['foo', 10, 25,
['/pipeline_instances/zzzzz-d1hrv-1xfj6xkicf2muk2',
- '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
+ # '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
'/jobs/zzzzz-8i9sb-grx15v5mjnsyxk7'],
['/pipeline_instances/zzzzz-d1hrv-1yfj61234abcdk3',
'/jobs/zzzzz-8i9sb-n7omg50bvt0m1nf',
'/container_requests/zzzzz-xvhdp-cr4completedcr2']],
['pipeline_with_tagged_collection_input', 1, 1,
['/pipeline_instances/zzzzz-d1hrv-1yfj61234abcdk3'],
- ['/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
+ [
+ # '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
'/jobs/zzzzz-8i9sb-pshmckwoma9plh7',
'/jobs/zzzzz-8i9sb-n7omg50bvt0m1nf',
'/container_requests/zzzzz-xvhdp-cr4completedcr2']],
['no_such_match', 0, 0,
[],
- ['/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
+ [
+ # '/pipeline_instances/zzzzz-d1hrv-jobspeccomponts',
'/jobs/zzzzz-8i9sb-pshmckwoma9plh7',
'/jobs/zzzzz-8i9sb-n7omg50bvt0m1nf',
'/container_requests/zzzzz-xvhdp-cr4completedcr2']],
commit 1bc042d23e2679f0b9060750b7e9280fce7159be
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 12 12:50:55 2021 -0300
15430: Updates wb1 test so that it exercises the ilike operator instead of @@
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/apps/workbench/test/controllers/work_units_controller_test.rb b/apps/workbench/test/controllers/work_units_controller_test.rb
index 6f74955cd..375c65c9b 100644
--- a/apps/workbench/test/controllers/work_units_controller_test.rb
+++ b/apps/workbench/test/controllers/work_units_controller_test.rb
@@ -32,7 +32,7 @@ class WorkUnitsControllerTest < ActionController::TestCase
'/container_requests/zzzzz-xvhdp-cr4completedcr2']],
].each do |search_filter, expected_min, expected_max, expected, not_expected|
test "all_processes page for search filter '#{search_filter}'" do
- work_units_index(filters: [['any','@@', search_filter]], show_children: true)
+ work_units_index(filters: [['any','ilike', "%#{search_filter}%"]], show_children: true)
assert_response :success
# Verify that expected number of processes are found
commit 55b4935bfd524013e36b1f5178b393eac53085de
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Thu Aug 12 12:03:30 2021 -0300
15430: Removes support for the '@@' full text search operator.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 5688ca614..301524a6c 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -31,7 +31,10 @@ module RecordFilters
model_table_name = model_class.table_name
filters.each do |filter|
attrs_in, operator, operand = filter
- if attrs_in == 'any' && operator != '@@'
+ if operator == '@@'
+ raise ArgumentError.new("Full text search is no longer supported")
+ end
+ if attrs_in == 'any'
attrs = model_class.searchable_columns(operator)
elsif attrs_in.is_a? Array
attrs = attrs_in
@@ -54,22 +57,6 @@ module RecordFilters
attrs = []
end
- if operator == '@@'
- # Full-text search
- if attrs_in != 'any'
- raise ArgumentError.new("Full text search on individual columns is not supported")
- end
- if operand.is_a? Array
- raise ArgumentError.new("Full text search not supported for array operands")
- end
-
- # Skip the generic per-column operator loop below
- attrs = []
- # Use to_tsquery since plainto_tsquery does not support prefix
- # search. And, split operand and join the words with ' & '
- cond_out << model_class.full_text_tsvector+" @@ to_tsquery(?)"
- param_out << operand.split.join(' & ')
- end
attrs.each do |attr|
subproperty = attr.split(".", 2)
diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index 26270b1c3..bcb180786 100644
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -29,34 +29,14 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
json_response['errors'].join(' '))
end
- test 'error message for full text search on a specific column' do
+ test 'error message for unsupported full text search' do
@controller = Arvados::V1::CollectionsController.new
authorize_with :active
get :index, params: {
filters: [['uuid', '@@', 'abcdef']],
}
assert_response 422
- assert_match(/not supported/, json_response['errors'].join(' '))
- end
-
- test 'difficult characters in full text search' do
- @controller = Arvados::V1::CollectionsController.new
- authorize_with :active
- get :index, params: {
- filters: [['any', '@@', 'a|b"c']],
- }
- assert_response :success
- # (Doesn't matter so much which results are returned.)
- end
-
- test 'array operand in full text search' do
- @controller = Arvados::V1::CollectionsController.new
- authorize_with :active
- get :index, params: {
- filters: [['any', '@@', ['abc', 'def']]],
- }
- assert_response 422
- assert_match(/not supported/, json_response['errors'].join(' '))
+ assert_match(/no longer supported/, json_response['errors'].join(' '))
end
test 'api responses provide timestamps with nanoseconds' do
@@ -100,58 +80,6 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
end
end
- test "full text search with count='none'" do
- @controller = Arvados::V1::GroupsController.new
- authorize_with :admin
-
- get :contents, params: {
- format: :json,
- count: 'none',
- limit: 1000,
- filters: [['any', '@@', Rails.configuration.ClusterID]],
- }
-
- assert_response :success
-
- all_objects = Hash.new(0)
- json_response['items'].map{|o| o['kind']}.each{|t| all_objects[t] += 1}
-
- assert_equal true, all_objects['arvados#group']>0
- assert_equal true, all_objects['arvados#job']>0
- assert_equal true, all_objects['arvados#pipelineInstance']>0
- assert_equal true, all_objects['arvados#pipelineTemplate']>0
-
- # Perform test again mimicking a second page request with:
- # last_object_class = PipelineInstance
- # and hence groups and jobs should not be included in the response
- # offset = 5, which means first 5 pipeline instances were already received in page 1
- # and hence the remaining pipeline instances and all other object types should be included in the response
-
- @test_counter = 0 # Reset executed action counter
-
- @controller = Arvados::V1::GroupsController.new
-
- get :contents, params: {
- format: :json,
- count: 'none',
- limit: 1000,
- offset: '5',
- last_object_class: 'PipelineInstance',
- filters: [['any', '@@', Rails.configuration.ClusterID]],
- }
-
- assert_response :success
-
- second_page = Hash.new(0)
- json_response['items'].map{|o| o['kind']}.each{|t| second_page[t] += 1}
-
- assert_equal false, second_page.include?('arvados#group')
- assert_equal false, second_page.include?('arvados#job')
- assert_equal true, second_page['arvados#pipelineInstance']>0
- assert_equal all_objects['arvados#pipelineInstance'], second_page['arvados#pipelineInstance']+5
- assert_equal true, second_page['arvados#pipelineTemplate']>0
- end
-
[['prop1', '=', 'value1', [:collection_with_prop1_value1], [:collection_with_prop1_value2, :collection_with_prop2_1]],
['prop1', '!=', 'value1', [:collection_with_prop1_value2, :collection_with_prop2_1], [:collection_with_prop1_value1]],
['prop1', 'exists', true, [:collection_with_prop1_value1, :collection_with_prop1_value2, :collection_with_prop1_value3, :collection_with_prop1_other1], [:collection_with_prop2_1]],
diff --git a/services/api/test/integration/collections_api_test.rb b/services/api/test/integration/collections_api_test.rb
index 73cbad643..070e964e5 100644
--- a/services/api/test/integration/collections_api_test.rb
+++ b/services/api/test/integration/collections_api_test.rb
@@ -373,75 +373,6 @@ class CollectionsApiTest < ActionDispatch::IntegrationTest
end
end
- test "search collection using full text search" do
- # create collection to be searched for
- signed_manifest = Collection.sign_manifest(". 85877ca2d7e05498dd3d109baf2df106+95+A3a4e26a366ee7e4ed3e476ccf05354761be2e4ae at 545a9920 0:95:file_in_subdir1\n./subdir2/subdir3 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64 at 545a9920 0:32:file1_in_subdir3.txt 32:32:file2_in_subdir3.txt\n./subdir2/subdir3/subdir4 2bbc341c702df4d8f42ec31f16c10120+64+A315d7e7bad2ce937e711fc454fae2d1194d14d64 at 545a9920 0:32:file3_in_subdir4.txt 32:32:file4_in_subdir4.txt\n", api_token(:active))
- post "/arvados/v1/collections",
- params: {
- format: :json,
- collection: {description: 'specific collection description', manifest_text: signed_manifest}.to_json,
- },
- headers: auth(:active)
- assert_response :success
- assert_equal true, json_response['manifest_text'].include?('file4_in_subdir4.txt')
-
- # search using the filename
- search_using_full_text_search 'subdir2', 0
- search_using_full_text_search 'subdir2:*', 1
- search_using_full_text_search 'subdir2/subdir3/subdir4', 1
- search_using_full_text_search 'file4:*', 1
- search_using_full_text_search 'file4_in_subdir4.txt', 1
- search_using_full_text_search 'subdir2 file4:*', 0 # first word is incomplete
- search_using_full_text_search 'subdir2/subdir3/subdir4 file4:*', 1
- search_using_full_text_search 'subdir2/subdir3/subdir4 file4_in_subdir4.txt', 1
- search_using_full_text_search 'ile4', 0 # not a prefix match
- end
-
- def search_using_full_text_search search_filter, expected_items
- get '/arvados/v1/collections',
- params: {:filters => [['any', '@@', search_filter]].to_json},
- headers: auth(:active)
- assert_response :success
- response_items = json_response['items']
- assert_not_nil response_items
- if expected_items == 0
- assert_empty response_items
- else
- refute_empty response_items
- first_item = response_items.first
- assert_not_nil first_item
- end
- end
-
- # search for the filename in the file_names column and expect error
- test "full text search not supported for individual columns" do
- get '/arvados/v1/collections',
- params: {:filters => [['name', '@@', 'General']].to_json},
- headers: auth(:active)
- assert_response 422
- end
-
- [
- 'quick fox',
- 'quick_brown fox',
- 'brown_ fox',
- 'fox dogs',
- ].each do |search_filter|
- test "full text search ignores special characters and finds with filter #{search_filter}" do
- # description: The quick_brown_fox jumps over the lazy_dog
- # full text search treats '_' as space apparently
- get '/arvados/v1/collections',
- params: {:filters => [['any', '@@', search_filter]].to_json},
- headers: auth(:active)
- assert_response 200
- response_items = json_response['items']
- assert_not_nil response_items
- first_item = response_items.first
- refute_empty first_item
- assert_equal first_item['description'], 'The quick_brown_fox jumps over the lazy_dog'
- end
- end
-
test "create and get collection with properties" do
# create collection to be searched for
signed_manifest = Collection.sign_manifest(". bad42fa702ae3ea7d888fef11b46f450+44 0:44:my_test_file.txt\n", api_token(:active))
diff --git a/services/api/test/integration/groups_test.rb b/services/api/test/integration/groups_test.rb
index aa67166f7..e76f2b540 100644
--- a/services/api/test/integration/groups_test.rb
+++ b/services/api/test/integration/groups_test.rb
@@ -64,46 +64,6 @@ class GroupsTest < ActionDispatch::IntegrationTest
end
end
- [
- ['Collection_', true], # collections and pipelines templates
- ['hash', true], # pipeline templates
- ['fa7aeb5140e2848d39b', false], # script_parameter of pipeline instances
- ['fa7aeb5140e2848d39b:*', true], # script_parameter of pipeline instances
- ['project pipeline', true], # finds "Completed pipeline in A Project"
- ['project pipeli:*', true], # finds "Completed pipeline in A Project"
- ['proje pipeli:*', false], # first word is incomplete, so no prefix match
- ['no-such-thing', false], # script_parameter of pipeline instances
- ].each do |search_filter, expect_results|
- test "full text search of group-owned objects for #{search_filter}" do
- get "/arvados/v1/groups/contents",
- params: {
- id: groups(:aproject).uuid,
- limit: 5,
- :filters => [['any', '@@', search_filter]].to_json
- },
- headers: auth(:active)
- assert_response :success
- if expect_results
- refute_empty json_response['items']
- json_response['items'].each do |item|
- assert item['uuid']
- assert_equal groups(:aproject).uuid, item['owner_uuid']
- end
- else
- assert_empty json_response['items']
- end
- end
- end
-
- test "full text search is not supported for individual columns" do
- get "/arvados/v1/groups/contents",
- params: {
- :filters => [['name', '@@', 'Private']].to_json
- },
- headers: auth(:active)
- assert_response 422
- end
-
test "group contents with include trash collections" do
get "/arvados/v1/groups/contents",
params: {
commit 6f3cb25564a0c543ab3d85d45da94afcdacf45d6
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Fri Aug 6 11:31:40 2021 -0300
15430: Removes FTS index from collections to allow many files.
While doing tests with a script that creates mock collections with more than
36k files, the FTS index was failing.
We're not using the collection's fts index, so we can drop it.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/services/api/db/migrate/20210806135109_drop_fts_index_on_collections.rb b/services/api/db/migrate/20210806135109_drop_fts_index_on_collections.rb
new file mode 100644
index 000000000..ef1567731
--- /dev/null
+++ b/services/api/db/migrate/20210806135109_drop_fts_index_on_collections.rb
@@ -0,0 +1,13 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class DropFtsIndexOnCollections < ActiveRecord::Migration[5.2]
+ def up
+ execute "DROP INDEX IF EXISTS collections_full_text_search_idx"
+ end
+
+ def down
+ execute "CREATE INDEX collections_full_text_search_idx ON collections USING gin(#{Collection.full_text_tsvector})"
+ end
+end
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index 2bca88721..c759d4476 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -238,29 +238,6 @@ SET default_tablespace = '';
SET default_with_oids = false;
---
--- Name: groups; Type: TABLE; Schema: public; Owner: -
---
-
-CREATE TABLE public.groups (
- id integer NOT NULL,
- uuid character varying(255),
- owner_uuid character varying(255),
- created_at timestamp without time zone NOT NULL,
- modified_by_client_uuid character varying(255),
- modified_by_user_uuid character varying(255),
- modified_at timestamp without time zone,
- name character varying(255) NOT NULL,
- description character varying(524288),
- updated_at timestamp without time zone NOT NULL,
- group_class character varying(255),
- trash_at timestamp without time zone,
- is_trashed boolean DEFAULT false NOT NULL,
- delete_at timestamp without time zone,
- properties jsonb DEFAULT '{}'::jsonb
-);
-
-
--
-- Name: api_client_authorizations; Type: TABLE; Schema: public; Owner: -
--
@@ -571,6 +548,29 @@ CREATE SEQUENCE public.containers_id_seq
ALTER SEQUENCE public.containers_id_seq OWNED BY public.containers.id;
+--
+-- Name: groups; Type: TABLE; Schema: public; Owner: -
+--
+
+CREATE TABLE public.groups (
+ id integer NOT NULL,
+ uuid character varying(255),
+ owner_uuid character varying(255),
+ created_at timestamp without time zone NOT NULL,
+ modified_by_client_uuid character varying(255),
+ modified_by_user_uuid character varying(255),
+ modified_at timestamp without time zone,
+ name character varying(255) NOT NULL,
+ description character varying(524288),
+ updated_at timestamp without time zone NOT NULL,
+ group_class character varying(255),
+ trash_at timestamp without time zone,
+ is_trashed boolean DEFAULT false NOT NULL,
+ delete_at timestamp without time zone,
+ properties jsonb DEFAULT '{}'::jsonb
+);
+
+
--
-- Name: groups_id_seq; Type: SEQUENCE; Schema: public; Owner: -
--
@@ -1722,13 +1722,6 @@ CREATE INDEX authorized_keys_search_index ON public.authorized_keys USING btree
CREATE INDEX collection_index_on_properties ON public.collections USING gin (properties);
---
--- Name: collections_full_text_search_idx; Type: INDEX; Schema: public; Owner: -
---
-
-CREATE INDEX collections_full_text_search_idx ON public.collections USING gin (to_tsvector('english'::regconfig, substr((((((((((((((((((COALESCE(owner_uuid, ''::character varying))::text || ' '::text) || (COALESCE(modified_by_client_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(modified_by_user_uuid, ''::character varying))::text) || ' '::text) || (COALESCE(portable_data_hash, ''::character varying))::text) || ' '::text) || (COALESCE(uuid, ''::character varying))::text) || ' '::text) || (COALESCE(name, ''::character varying))::text) || ' '::text) || (COALESCE(description, ''::character varying))::text) || ' '::text) || COALESCE((properties)::text, ''::text)) || ' '::text) || COALESCE(file_names, ''::text)), 0, 1000000)));
-
-
--
-- Name: collections_search_index; Type: INDEX; Schema: public; Owner: -
--
@@ -3194,6 +3187,7 @@ INSERT INTO "schema_migrations" (version) VALUES
('20201202174753'),
('20210108033940'),
('20210126183521'),
-('20210621204455');
+('20210621204455'),
+('20210806135109');
diff --git a/services/api/test/unit/arvados_model_test.rb b/services/api/test/unit/arvados_model_test.rb
index 64f780713..3d5377091 100644
--- a/services/api/test/unit/arvados_model_test.rb
+++ b/services/api/test/unit/arvados_model_test.rb
@@ -183,7 +183,7 @@ class ArvadosModelTest < ActiveSupport::TestCase
indexes[idx['table']] << idx
end
end
- fts_tables = ["collections", "container_requests", "groups", "jobs",
+ fts_tables = ["container_requests", "groups", "jobs",
"pipeline_instances", "pipeline_templates", "workflows"]
fts_tables.each do |table|
table_class = table.classify.constantize
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list