[ARVADOS] updated: 1.3.0-1020-g157881e36

[Git user]

Summary of changes:
 doc/admin/upgrading.html.textile.liquid |  8 ++++
 services/api/lib/tasks/symbols.rake     | 71 ++++++++++++++++++++++++++++++---
 2 files changed, 74 insertions(+), 5 deletions(-)

       via  157881e369576619e19d3fb50a76c6135ff35e01 (commit)
      from  0ecff72955e194a1060e4b1dd847ef9900b0fbbb (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 157881e369576619e19d3fb50a76c6135ff35e01
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date:   Thu Jun 6 17:17:28 2019 -0400

    15311: Add release note and symbols:stringify task.
    
    Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>

diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 053acb220..9a8d2dac8 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -30,6 +30,14 @@ Note to developers: Add new items at the top. Include the date, issue number, co
 TODO: extract this information based on git commit messages and generate changelogs / release notes automatically.
 {% endcomment %}
 
+h3. master
+
+h4. No longer stripping ':' from strings in serialized database columns
+
+ (bug #15311) Strings read from serialized columns in the database with a leading ':' would have the ':' stripped after loading the record.  This behavior existed due to legacy serialization behavior which stored Ruby symbols with a leading ':'.  Unfortunately this corrupted fields where the leading ":" was intentional.  This behavior has been removed.
+
+You can test if any records in your database are affected by going to the API server directory and running @bundle exec rake symbols:check at .  This will report which records contain fields with a leading ':' that would previously have been stripped.  If there are records to be updated, you can update the database using @bundle exec rake symbols:stringify at .
+
 h3. v1.4.0 (2019-05-31)
 
 h4. Populating the new file_count and file_size_total columns on the collections table
diff --git a/services/api/lib/tasks/symbols.rake b/services/api/lib/tasks/symbols.rake
index 5b4eab47e..a2e6df8b5 100644
--- a/services/api/lib/tasks/symbols.rake
+++ b/services/api/lib/tasks/symbols.rake
@@ -1,3 +1,11 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+require 'current_api_client'
+
+include CurrentApiClient
+
 def has_symbols? x
   if x.is_a? Hash
     x.each do |k,v|
@@ -16,6 +24,34 @@ def has_symbols? x
 end
 
 def check_for_serialized_symbols rec
+  jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
+  (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
+    if has_symbols? rec.attributes[colname]
+      st = recursive_stringify rec.attributes[colname]
+      puts "Found value potentially containing Ruby symbols in #{colname} attribute of #{rec.uuid}, current value is\n#{rec.attributes[colname].to_s[0..1024]}\nrake symbols:stringify will update it to:\n#{st.to_s[0..1024]}\n\n"
+    end
+  end
+end
+
+def recursive_stringify x
+  if x.is_a? Hash
+    Hash[x.collect do |k,v|
+           [recursive_stringify(k), recursive_stringify(v)]
+         end]
+  elsif x.is_a? Array
+    x.collect do |k|
+      recursive_stringify k
+    end
+  elsif x.is_a? Symbol
+    x.to_s
+  elsif x.is_a? String and x.start_with?(':') and !x.start_with?('::')
+    x[1..-1]
+  else
+    x
+  end
+end
+
+def stringify_serialized_symbols rec
   # ensure_serialized_attribute_type should prevent symbols from
   # getting into the database in the first place. If someone managed
   # to get them into the database (perhaps using an older version)
@@ -25,14 +61,37 @@ def check_for_serialized_symbols rec
   jsonb_cols = rec.class.columns.select{|c| c.type == :jsonb}.collect{|j| j.name}
   (jsonb_cols + rec.class.serialized_attributes.keys).uniq.each do |colname|
     if has_symbols? rec.attributes[colname]
-      puts "Found string with leading ':' in attribute '#{colname}' of #{rec.uuid}:\n#{rec.attributes[colname].to_s[0..1024]}\n\n"
+      begin
+        st = recursive_stringify rec.attributes[colname]
+        puts "Updating #{colname} attribute of #{rec.uuid} from\n#{rec.attributes[colname].to_s[0..1024]}\nto\n#{st.to_s[0..1024]}\n\n"
+        rec.write_attribute(colname, st)
+        rec.save!
+      rescue => e
+        puts "Failed to update #{rec.uuid}: #{e}"
+      end
     end
   end
 end
 
-namespace :legacy do
+namespace :symbols do
   desc 'Warn about serialized values starting with ":" that may be symbols'
-  task symbols: :environment do
+  task check: :environment do
+    [ApiClientAuthorization, ApiClient,
+     AuthorizedKey, Collection,
+     Container, ContainerRequest, Group,
+     Human, Job, JobTask, KeepDisk, KeepService, Link,
+     Node, PipelineInstance, PipelineTemplate,
+     Repository, Specimen, Trait, User, VirtualMachine,
+     Workflow].each do |klass|
+      act_as_system_user do
+        klass.all.each do |c|
+          check_for_serialized_symbols c
+        end
+      end
+    end
+  end
+
+  task stringify: :environment do
     [ApiClientAuthorization, ApiClient,
      AuthorizedKey, Collection,
      Container, ContainerRequest, Group,
@@ -40,8 +99,10 @@ namespace :legacy do
      Node, PipelineInstance, PipelineTemplate,
      Repository, Specimen, Trait, User, VirtualMachine,
      Workflow].each do |klass|
-      klass.all.each do |c|
-        check_for_serialized_symbols c
+      act_as_system_user do
+        klass.all.each do |c|
+          stringify_serialized_symbols c
+        end
       end
     end
   end

-----------------------------------------------------------------------


hooks/post-receive
--