[ARVADOS] created: 2.1.0-1225-g2c5fda23e

Git user git at public.arvados.org
Fri Aug 27 14:21:57 UTC 2021


        at  2c5fda23eb64f4efe0d8f791b0a9cabf46d5866b (commit)


commit 2c5fda23eb64f4efe0d8f791b0a9cabf46d5866b
Author: Tom Clegg <tom at curii.com>
Date:   Fri Aug 27 10:19:17 2021 -0400

    17994: Document filtering by comparing two attributes.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/doc/api/methods.html.textile.liquid b/doc/api/methods.html.textile.liquid
index c6e4ba00a..befcd4651 100644
--- a/doc/api/methods.html.textile.liquid
+++ b/doc/api/methods.html.textile.liquid
@@ -107,6 +107,16 @@ table(table table-bordered table-condensed).
 |@is_a@|string|Arvados object type|@["head_uuid","is_a","arvados#collection"]@|
 |@exists@|string|Test if a subproperty is present.|@["properties","exists","my_subproperty"]@|
 
+h4(#filterexpression). Filtering by comparing two attributes
+
+If @a@ and @b@ are attributes that have numeric values, they can be compared in a filter condition using the form @["(a < b)", "=", true]@. Supported operators are @=@, @<@, @<=@, @>@, and @>=@. Examples:
+* @["(replication_desired < replication_confirmed)", "=", true]@
+* @["(replication_desired = replication_confirmed)", "=", true]@
+
+Note that only a very specific subset of boolean expressions is supported. For example:
+* @["replication_desired < replication_confirmed", "=", true]@ is not accepted. Outer parentheses are mandatory.
+* @["(replication_desired < 1)", "=", true]@ is not accepted. Operands must be attribute names.
+
 h4(#substringsearchfilter). Filtering using substring search
 
 Resources can also be filtered by searching for a substring in attributes of type @string@, @array of strings@, @text@, and @hash@, which are indexed in the database specifically for search. To use substring search, the filter must:

commit ced698936d8c6dcd75477de2ef184112567362fe
Author: Tom Clegg <tom at curii.com>
Date:   Thu Aug 26 16:34:42 2021 -0400

    17994: Accept a few very specific expressions as filters.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 8f4244f5b..635e9c1aa 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -155,6 +155,23 @@ module RecordFilters
 
           cond_out << "jsonb_exists(#{attr_table_name}.#{attr}, ?)"
           param_out << operand
+        elsif expr = /^ *\( *(\w+) *(<=?|>=?|=) *(\w+) *\) *$/.match(attr)
+          if operator != '=' || ![true,"true"].index(operand)
+            raise ArgumentError.new("Invalid expression filter '#{attr}': subsequent elements must be [\"=\", true]")
+          end
+          operator = expr[2]
+          attr1, attr2 = expr[1], expr[3]
+          allowed = attr_model_class.searchable_columns(operator)
+          [attr1, attr2].each do |tok|
+            if !allowed.index(tok)
+              raise ArgumentError.new("Invalid attribute in expression: '#{tok}'")
+            end
+            col = attr_model_class.columns.select { |c| c.name == tok }.first
+            if col.type != :integer
+              raise ArgumentError.new("Non-numeric attribute in expression: '#{tok}'")
+            end
+          end
+          cond_out << "#{attr1} #{operator} #{attr2}"
         else
           if !attr_model_class.searchable_columns(operator).index(attr) &&
              !(col.andand.type == :jsonb && ['contains', '=', '<>', '!='].index(operator))
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 5b77a544e..fbee80d5a 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -1469,4 +1469,48 @@ EOS
       end
     end
   end
+
+  [['<', :<],
+   ['<=', :<=],
+   ['>', :>],
+   ['>=', :>=],
+   ['=', :==]].each do |op, rubyop|
+    test "filter collections by replication_desired #{op} replication_confirmed" do
+      authorize_with(:active)
+      get :index, params: {
+            filters: [["(replication_desired #{op} replication_confirmed)", "=", true]],
+          }
+      assert_response :success
+      json_response["items"].each do |c|
+        assert_operator(c["replication_desired"], rubyop, c["replication_confirmed"])
+      end
+    end
+  end
+
+  ["(replication_desired < bogus)",
+   "replication_desired < replication_confirmed",
+   "(replication_desired < replication_confirmed",
+   "(replication_desired ! replication_confirmed)",
+   "(replication_desired <)",
+   "(replication_desired < manifest_text)",
+   "(manifest_text < manifest_text)", # currently only numeric attrs are supported
+   "(replication_desired < 2)", # currently only attrs are supported, not literals
+   "(1 < 2)",
+  ].each do |expr|
+    test "invalid filter expression #{expr}" do
+      authorize_with(:active)
+      get :index, params: {
+            filters: [[expr, "=", true]],
+          }
+      assert_response 422
+    end
+  end
+
+  test "invalid op/arg with filter expression" do
+    authorize_with(:active)
+    get :index, params: {
+          filters: [["replication_desired < replication_confirmed", "!=", false]],
+        }
+    assert_response 422
+  end
 end

commit 402e69f6e55dce4e11d354c3ca708b8e536c124b
Author: Tom Clegg <tom at curii.com>
Date:   Tue Aug 24 09:57:10 2021 -0400

    17994: Update comment.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 337197c2c..8f4244f5b 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -255,9 +255,12 @@ module RecordFilters
                 raise ArgumentError.new("Invalid element #{operand.inspect} in operand for #{operator.inspect} operator (operand must be a string or array of strings)")
               end
             end
-            q = operand.map { |s| ActiveRecord::Base.connection.quote(s) }.join(',')
             # We use jsonb_exists_all(a,b) instead of "a ?& b" because
-            # the pg gem thinks "?" is a bind var.
+            # the pg gem thinks "?" is a bind var. And we use string
+            # interpolation instead of param_out because the pg gem
+            # flattens param_out / doesn't support passing arrays as
+            # bind vars.
+            q = operand.map { |s| ActiveRecord::Base.connection.quote(s) }.join(',')
             cond_out << "jsonb_exists_all(#{attr_table_name}.#{attr}, array[#{q}])"
           else
             raise ArgumentError.new("Invalid operator '#{operator}'")

commit 388713bd42fb20bae6f628c50f4ad1e3ba067b5b
Author: Tom Clegg <tom at curii.com>
Date:   Tue Aug 24 09:47:04 2021 -0400

    17994: Support filtering by element/key presence in jsonb columns.
    
    Remove storage_classes_* from searchable_columns so that filters with
    the magic attr "any" don't try to search them.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index f4c717999..1bbe8cc66 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -613,7 +613,7 @@ class Collection < ArvadosModel
   end
 
   def self.searchable_columns operator
-    super - ["manifest_text"] + ["storage_classes_desired", "storage_classes_confirmed"]
+    super - ["manifest_text"]
   end
 
   def self.full_text_searchable_columns
diff --git a/services/api/lib/record_filters.rb b/services/api/lib/record_filters.rb
index 5688ca614..337197c2c 100644
--- a/services/api/lib/record_filters.rb
+++ b/services/api/lib/record_filters.rb
@@ -44,9 +44,10 @@ module RecordFilters
         raise ArgumentError.new("Invalid operator '#{operator}' (#{operator.class}) in filter")
       end
 
+      operator = operator.downcase
       cond_out = []
 
-      if attrs_in == 'any' && (operator.casecmp('ilike').zero? || operator.casecmp('like').zero?) && (operand.is_a? String) && operand.match('^[%].*[%]$')
+      if attrs_in == 'any' && (operator == 'ilike' || operator == 'like') && (operand.is_a? String) && operand.match('^[%].*[%]$')
         # Trigram index search
         cond_out << model_class.full_text_trgm + " #{operator} ?"
         param_out << operand
@@ -98,9 +99,9 @@ module RecordFilters
           end
 
           # jsonb search
-          case operator.downcase
+          case operator
           when '=', '!='
-            not_in = if operator.downcase == "!=" then "NOT " else "" end
+            not_in = if operator == "!=" then "NOT " else "" end
             cond_out << "#{not_in}(#{attr_table_name}.#{attr} @> ?::jsonb)"
             param_out << SafeJSON.dump({proppath => operand})
           when 'in'
@@ -147,7 +148,7 @@ module RecordFilters
           else
             raise ArgumentError.new("Invalid operator for subproperty search '#{operator}'")
           end
-        elsif operator.downcase == "exists"
+        elsif operator == "exists"
           if col.type != :jsonb
             raise ArgumentError.new("Invalid attribute '#{attr}' for operator '#{operator}' in filter")
           end
@@ -155,11 +156,12 @@ module RecordFilters
           cond_out << "jsonb_exists(#{attr_table_name}.#{attr}, ?)"
           param_out << operand
         else
-          if !attr_model_class.searchable_columns(operator).index attr
+          if !attr_model_class.searchable_columns(operator).index(attr) &&
+             !(col.andand.type == :jsonb && ['contains', '=', '<>', '!='].index(operator))
             raise ArgumentError.new("Invalid attribute '#{attr}' in filter")
           end
 
-          case operator.downcase
+          case operator
           when '=', '<', '<=', '>', '>=', '!=', 'like', 'ilike'
             attr_type = attr_model_class.attribute_column(attr).type
             operator = '<>' if operator == '!='
@@ -240,6 +242,23 @@ module RecordFilters
               end
             end
             cond_out << cond.join(' OR ')
+          when 'contains'
+            if col.andand.type != :jsonb
+              raise ArgumentError.new("Invalid attribute '#{attr}' for '#{operator}' operator")
+            end
+            if operand == []
+              raise ArgumentError.new("Invalid operand '#{operand.inspect}' for '#{operator}' operator")
+            end
+            operand = [operand] unless operand.is_a? Array
+            operand.each do |op|
+              if !op.is_a?(String)
+                raise ArgumentError.new("Invalid element #{operand.inspect} in operand for #{operator.inspect} operator (operand must be a string or array of strings)")
+              end
+            end
+            q = operand.map { |s| ActiveRecord::Base.connection.quote(s) }.join(',')
+            # We use jsonb_exists_all(a,b) instead of "a ?& b" because
+            # the pg gem thinks "?" is a bind var.
+            cond_out << "jsonb_exists_all(#{attr_table_name}.#{attr}, array[#{q}])"
           else
             raise ArgumentError.new("Invalid operator '#{operator}'")
           end
diff --git a/services/api/test/functional/arvados/v1/filters_test.rb b/services/api/test/functional/arvados/v1/filters_test.rb
index 26270b1c3..04d0329b2 100644
--- a/services/api/test/functional/arvados/v1/filters_test.rb
+++ b/services/api/test/functional/arvados/v1/filters_test.rb
@@ -319,4 +319,51 @@ class Arvados::V1::FiltersTest < ActionController::TestCase
     assert_includes(found, collections(:replication_desired_2_unconfirmed).uuid)
     assert_includes(found, collections(:replication_desired_2_confirmed_2).uuid)
   end
+
+  [
+    [1, "foo"],
+    [1, ["foo"]],
+    [1, ["bar"]],
+    [1, ["bar", "foo"]],
+    [0, ["foo", "qux"]],
+    [0, ["qux"]],
+    [nil, []],
+    [nil, [[]]],
+    [nil, [["bogus"]]],
+    [nil, [{"foo" => "bar"}]],
+    [nil, {"foo" => "bar"}],
+  ].each do |results, operand|
+    test "storage_classes_desired contains #{operand.inspect}" do
+      @controller = Arvados::V1::CollectionsController.new
+      authorize_with(:active)
+      c = Collection.create!(
+        manifest_text: "",
+        storage_classes_desired: ["foo", "bar", "baz"])
+      get :index, params: {
+            filters: [["storage_classes_desired", "contains", operand]],
+          }
+      if results.nil?
+        assert_response 422
+        next
+      end
+      assert_response :success
+      assert_equal results, json_response["items"].length
+      if results > 0
+        assert_equal c.uuid, json_response["items"][0]["uuid"]
+      end
+    end
+  end
+
+  test "collections properties contains top level key" do
+    @controller = Arvados::V1::CollectionsController.new
+    authorize_with(:active)
+    get :index, params: {
+          filters: [["properties", "contains", "prop1"]],
+        }
+    assert_response :success
+    assert_not_empty json_response["items"]
+    json_response["items"].each do |c|
+      assert c["properties"].has_key?("prop1")
+    end
+  end
 end

commit 902f8cd258a8dfec749a7f94d478a4027e319750
Author: Tom Clegg <tom at curii.com>
Date:   Mon Aug 23 14:21:36 2021 -0400

    17994: Allow filtering on storage_classes_desired/confirmed fields.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 1bbe8cc66..f4c717999 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -613,7 +613,7 @@ class Collection < ArvadosModel
   end
 
   def self.searchable_columns operator
-    super - ["manifest_text"]
+    super - ["manifest_text"] + ["storage_classes_desired", "storage_classes_confirmed"]
   end
 
   def self.full_text_searchable_columns
diff --git a/services/api/test/functional/arvados/v1/collections_controller_test.rb b/services/api/test/functional/arvados/v1/collections_controller_test.rb
index 1ca2dd1dc..5b77a544e 100644
--- a/services/api/test/functional/arvados/v1/collections_controller_test.rb
+++ b/services/api/test/functional/arvados/v1/collections_controller_test.rb
@@ -1455,4 +1455,18 @@ EOS
     assert_response :success
     assert_equal col.version, json_response['version'], 'Trashing a collection should not create a new version'
   end
+
+  ["storage_classes_desired", "storage_classes_confirmed"].each do |attr|
+    test "filter collections by #{attr}" do
+      authorize_with(:active)
+      get :index, params: {
+            filters: [[attr, "=", '["default"]']]
+          }
+      assert_response :success
+      assert_not_equal 0, json_response["items"].length
+      json_response["items"].each do |c|
+        assert_equal ["default"], c[attr]
+      end
+    end
+  end
 end

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list