[ARVADOS] updated: 1.3.0-2641-g570c793f1

Git user git at public.arvados.org
Fri Jun 5 20:23:36 UTC 2020


Summary of changes:
 doc/_config.yml                                    |  1 +
 .../undeleting-collections.html.textile.liquid     | 37 +++++++++++++
 lib/undelete/cmd.go                                | 60 ++++++++++++++++------
 3 files changed, 81 insertions(+), 17 deletions(-)
 create mode 100644 doc/admin/undeleting-collections.html.textile.liquid

       via  570c793f1aa43fd9763a0368554dd395dacdd238 (commit)
       via  9e38275d42407c52397e079d741d50b278cdc3c6 (commit)
       via  bf277d86fe874fa701c117365ed4c88060b8a984 (commit)
      from  2be95ce7f069d9eb131b0d2d922a5e556f75810c (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 570c793f1aa43fd9763a0368554dd395dacdd238
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jun 5 16:23:01 2020 -0400

    16427: Add "undeleting collections" doc page in admin section.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/doc/_config.yml b/doc/_config.yml
index 48fe1b53d..9f016fcc8 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -174,6 +174,7 @@ navbar:
       - admin/logs-table-management.html.textile.liquid
       - admin/workbench2-vocabulary.html.textile.liquid
       - admin/storage-classes.html.textile.liquid
+      - admin/undeleting-collections.html.textile.liquid
     - Cloud:
       - admin/spot-instances.html.textile.liquid
       - admin/cloudtest.html.textile.liquid
diff --git a/doc/admin/undeleting-collections.html.textile.liquid b/doc/admin/undeleting-collections.html.textile.liquid
new file mode 100644
index 000000000..461671b53
--- /dev/null
+++ b/doc/admin/undeleting-collections.html.textile.liquid
@@ -0,0 +1,37 @@
+---
+layout: default
+navsection: admin
+title: Undeleting collections
+...
+
+{% comment %}
+Copyright (C) The Arvados Authors. All rights reserved.
+
+SPDX-License-Identifier: CC-BY-SA-3.0
+{% endcomment %}
+
+In some cases, it is possible to recover files that have been lost by modifying or deleting a collection.
+
+Possibility of recovery depends on many factors, including:
+* Whether the collection manifest is still available, e.g., in an audit log entry
+* Whether the data blocks are also referenced by other collections
+* Whether the data blocks have been unreferenced long enough to be marked for deletion/trash by keep-balance
+* Blob signature TTL, trash lifetime, trash check interval, and other config settings
+
+To attempt recovery of a previous version of a deleted/modified collection, use the @arvados-server undelete@ command. It should be run on one of your server nodes where the @arvados-server@ package is installed and the @/etc/arvados/config.yml@ file is up to date.
+
+Specify the collection you want to recover by passing either the UUID of an audit log entry, or a file containing the manifest.
+
+If recovery is successful, the undelete program saves the recovered data a new collection belonging to the system user, and print the new collection's UUID on stdout.
+
+<pre>
+# arvados-server undelete 9tee4-57u5n-nb5awmk1pahac2t
+INFO[2020-06-05T19:52:29.557761245Z] loaded log entry                              logged_event_time="2020-06-05 16:48:01.438791 +0000 UTC" logged_event_type=update old_collection_uuid=9tee4-4zz18-1ex26g95epmgw5w src=9tee4-57u5n-nb5awmk1pahac2t
+INFO[2020-06-05T19:52:29.642145127Z] recovery succeeded                            UUID=9tee4-4zz18-5trfp4k4xxg97f1 src=9tee4-57u5n-nb5awmk1pahac2t
+9tee4-4zz18-5trfp4k4xxg97f1
+INFO[2020-06-05T19:52:29.644699436Z] exiting
+</pre>
+
+In this example, the original data has been restored and saved in a new collection with UUID @9tee4-4zz18-5trfp4k4xxg97f1 at .
+
+For more options, run @arvados-server undelete -help at .

commit 9e38275d42407c52397e079d741d50b278cdc3c6
Author: Tom Clegg <tom at tomclegg.ca>
Date:   Fri Jun 5 13:10:22 2020 -0400

    16427: Support looking up old manifest for given log entry UUID.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>

diff --git a/lib/undelete/cmd.go b/lib/undelete/cmd.go
index 09adfae3c..9d4bc84ea 100644
--- a/lib/undelete/cmd.go
+++ b/lib/undelete/cmd.go
@@ -42,7 +42,7 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
 	flags.SetOutput(stderr)
 	flags.Usage = func() {
 		fmt.Fprintf(flags.Output(), `Usage:
-	%s [options ...] /path/to/manifest.txt [...]
+	%s [options ...] { /path/to/manifest.txt | log-entry-uuid } [...]
 
 	This program recovers deleted collections. Recovery is
 	possible when the collection's manifest is still available and
@@ -52,10 +52,17 @@ func (command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
 	collections, or the blocks have been trashed but not yet
 	deleted).
 
+	Collections can be specified either by filename (a local file
+	containing a manifest with the desired data) or by log UUID
+	(an Arvados log entry, typically a "delete" or "update" event,
+	whose "old attributes" have a manifest with the desired data).
+
 	For each provided collection manifest, once all data blocks
 	are recovered/protected from garbage collection, a new
 	collection is saved and its UUID is printed on stdout.
 
+	Restored collections will belong to the system (root) user.
+
 	Exit status will be zero if recovery is successful, i.e., a
 	collection is saved for each provided manifest.
 Options:
@@ -105,26 +112,47 @@ Options:
 	exitcode := 0
 	for _, src := range flags.Args() {
 		logger := logger.WithField("src", src)
+		var mtxt string
 		if len(src) == 27 && src[5:12] == "-57u5n-" {
-			logger.Error("log entry lookup not implemented")
-			exitcode = 1
-			continue
-		} else {
-			mtxt, err := ioutil.ReadFile(src)
+			var logent struct {
+				EventType  string    `json:"event_type"`
+				EventAt    time.Time `json:"event_at"`
+				ObjectUUID string    `json:"object_uuid"`
+				Properties struct {
+					OldAttributes struct {
+						ManifestText string `json:"manifest_text"`
+					} `json:"old_attributes"`
+				} `json:"properties"`
+			}
+			err = client.RequestAndDecode(&logent, "GET", "arvados/v1/logs/"+src, nil, nil)
 			if err != nil {
-				logger.WithError(err).Error("error loading manifest data")
+				logger.WithError(err).Error("failed to load log entry")
 				exitcode = 1
 				continue
 			}
-			uuid, err := und.RecoverManifest(string(mtxt))
+			logger.WithFields(logrus.Fields{
+				"old_collection_uuid": logent.ObjectUUID,
+				"logged_event_type":   logent.EventType,
+				"logged_event_time":   logent.EventAt,
+			}).Info("loaded log entry")
+			mtxt = logent.Properties.OldAttributes.ManifestText
+		} else {
+			buf, err := ioutil.ReadFile(src)
 			if err != nil {
-				logger.WithError(err).Error("recovery failed")
+				logger.WithError(err).Error("failed to load manifest data from file")
 				exitcode = 1
 				continue
 			}
-			logger.WithField("UUID", uuid).Info("recovery succeeded")
-			fmt.Fprintln(stdout, uuid)
+			mtxt = string(buf)
+		}
+		uuid, err := und.RecoverManifest(string(mtxt))
+		if err != nil {
+			logger.WithError(err).Error("recovery failed")
+			exitcode = 1
+			continue
 		}
+		logger.WithField("UUID", uuid).Info("recovery succeeded")
+		fmt.Fprintln(stdout, uuid)
 	}
 	return exitcode
 }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list