[arvados] updated: 2.5.0-115-g4b2b6ca09

[git repository hosting]

Summary of changes:
 sdk/go/arvados/fs_collection.go      | 40 +++++++++++++++++++++++++++---------
 sdk/go/arvados/fs_collection_test.go |  2 +-
 2 files changed, 31 insertions(+), 11 deletions(-)

       via  4b2b6ca09aec9dfb9f3871bd202484b1ae4063b9 (commit)
      from  c4c57180c9ee9f79a1d272710aa7b8747d4d0c38 (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 4b2b6ca09aec9dfb9f3871bd202484b1ae4063b9
Author: Tom Clegg <tom at curii.com>
Date:   Fri Feb 10 02:34:34 2023 -0500

    20083: Track both loadedPDH and savedPDH for checking local changes.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>

diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 354658a25..8e0537730 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -44,9 +44,17 @@ type CollectionFileSystem interface {
 type collectionFileSystem struct {
 	fileSystem
 	uuid           string
-	savedPDH       atomic.Value
 	replicas       int
 	storageClasses []string
+
+	// PDH returned by the server as of last sync/load.
+	loadedPDH atomic.Value
+	// PDH of the locally generated manifest as of last
+	// sync/load. This can differ from loadedPDH after loading a
+	// version that was generated with different code and sorts
+	// filenames differently than we do, for example.
+	savedPDH atomic.Value
+
 	// guessSignatureTTL tracks a lower bound for the server's
 	// configured BlobSigningTTL. The guess is initially zero, and
 	// increases when we come across a signature with an expiry
@@ -74,7 +82,7 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
 			thr:       newThrottle(concurrentWriters),
 		},
 	}
-	fs.savedPDH.Store(c.PortableDataHash)
+	fs.loadedPDH.Store(c.PortableDataHash)
 	if r := c.ReplicationDesired; r != nil {
 		fs.replicas = *r
 	}
@@ -94,6 +102,13 @@ func (c *Collection) FileSystem(client apiClient, kc keepClient) (CollectionFile
 	if err := root.loadManifest(c.ManifestText); err != nil {
 		return nil, err
 	}
+
+	txt, err := root.marshalManifest(context.Background(), ".")
+	if err != nil {
+		return nil, err
+	}
+	fs.savedPDH.Store(PortableDataHash(txt))
+
 	backdateTree(root, modTime)
 	fs.root = root
 	return fs, nil
@@ -296,7 +311,7 @@ func (fs *collectionFileSystem) Truncate(int64) error {
 // Return value is true if new content was loaded from upstream and
 // any unsaved local changes have been discarded.
 func (fs *collectionFileSystem) checkChangesOnServer(force bool) (bool, error) {
-	if fs.uuid == "" && fs.savedPDH.Load() == "" {
+	if fs.uuid == "" && fs.loadedPDH.Load() == "" {
 		return false, nil
 	}
 
@@ -316,6 +331,7 @@ func (fs *collectionFileSystem) checkChangesOnServer(force bool) (bool, error) {
 		return false, nil
 	}
 
+	loadedPDH, _ := fs.loadedPDH.Load().(string)
 	getparams := map[string]interface{}{"select": []string{"portable_data_hash", "manifest_text"}}
 	if fs.uuid != "" {
 		var coll Collection
@@ -323,7 +339,7 @@ func (fs *collectionFileSystem) checkChangesOnServer(force bool) (bool, error) {
 		if err != nil {
 			return false, err
 		}
-		if coll.PortableDataHash != fs.savedPDH.Load().(string) {
+		if coll.PortableDataHash != loadedPDH {
 			// collection has changed upstream since we
 			// last loaded or saved. Refresh local data,
 			// losing any unsaved local changes.
@@ -339,15 +355,16 @@ func (fs *collectionFileSystem) checkChangesOnServer(force bool) (bool, error) {
 			if err != nil {
 				return false, err
 			}
-			fs.savedPDH.Store(coll.PortableDataHash)
+			fs.loadedPDH.Store(coll.PortableDataHash)
+			fs.savedPDH.Store(newfs.(*collectionFileSystem).savedPDH.Load())
 			return true, nil
 		}
 		fs.updateSignatures(coll.ManifestText)
 		return false, nil
 	}
-	if pdh := fs.savedPDH.Load().(string); pdh != "" {
+	if loadedPDH != "" {
 		var coll Collection
-		err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+pdh, nil, getparams)
+		err := fs.RequestAndDecode(&coll, "GET", "arvados/v1/collections/"+loadedPDH, nil, getparams)
 		if err != nil {
 			return false, err
 		}
@@ -368,8 +385,9 @@ func (fs *collectionFileSystem) refreshSignature(locator string) string {
 		go fs.checkChangesOnServer(false)
 		return locator
 	}
+	loadedPDH, _ := fs.loadedPDH.Load().(string)
 	var manifests string
-	for _, id := range []string{fs.uuid, fs.savedPDH.Load().(string)} {
+	for _, id := range []string{fs.uuid, loadedPDH} {
 		if id == "" {
 			continue
 		}
@@ -405,7 +423,8 @@ func (fs *collectionFileSystem) Sync() error {
 	if err != nil {
 		return fmt.Errorf("sync failed: %s", err)
 	}
-	if PortableDataHash(txt) == fs.savedPDH.Load() {
+	savingPDH := PortableDataHash(txt)
+	if savingPDH == fs.savedPDH.Load() {
 		// No local changes since last save or initial load.
 		return nil
 	}
@@ -432,7 +451,8 @@ func (fs *collectionFileSystem) Sync() error {
 		return fmt.Errorf("sync failed: update %s: %w", fs.uuid, err)
 	}
 	fs.updateSignatures(coll.ManifestText)
-	fs.savedPDH.Store(coll.PortableDataHash)
+	fs.loadedPDH.Store(coll.PortableDataHash)
+	fs.savedPDH.Store(savingPDH)
 	return nil
 }
 
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 8b84ddf3e..e91853c16 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -1661,7 +1661,7 @@ func (s *CollectionFSUnitSuite) TestLargeManifest(c *check.C) {
 	runtime.ReadMemStats(&memstats)
 	c.Logf("%s Alloc=%d Sys=%d", time.Now(), memstats.Alloc, memstats.Sys)
 
-	f, err := coll.FileSystem(nil, nil)
+	f, err := coll.FileSystem(NewClientFromEnv(), &keepClientStub{})
 	c.Check(err, check.IsNil)
 	c.Logf("%s loaded", time.Now())
 	c.Check(f.Size(), check.Equals, int64(42*dirCount*fileCount))

-----------------------------------------------------------------------


hooks/post-receive
--