[ARVADOS] updated: 1.3.0-2643-g6e674eaf1
Git user
git at public.arvados.org
Wed Jun 10 19:32:45 UTC 2020
Summary of changes:
cmd/arvados-server/cmd.go | 24 ++---
doc/_config.yml | 2 +-
...vering-deleted-collections.html.textile.liquid} | 10 +-
lib/{undelete => recovercollection}/cmd.go | 101 +++++++++++++--------
lib/{undelete => recovercollection}/cmd_test.go | 24 ++++-
5 files changed, 102 insertions(+), 59 deletions(-)
rename doc/admin/{undeleting-collections.html.textile.liquid => recovering-deleted-collections.html.textile.liquid} (72%)
rename lib/{undelete => recovercollection}/cmd.go (72%)
rename lib/{undelete => recovercollection}/cmd_test.go (77%)
via 6e674eaf19242d400b0992551e106eac8de6d08f (commit)
via 5fd885a3037f1bc98344c17a68fcdeff75ab974b (commit)
from 570c793f1aa43fd9763a0368554dd395dacdd238 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 6e674eaf19242d400b0992551e106eac8de6d08f
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Jun 10 15:31:23 2020 -0400
16427: Option to recover from given collection's last log entry.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/lib/recovercollection/cmd.go b/lib/recovercollection/cmd.go
index d19bf3116..f962ac812 100644
--- a/lib/recovercollection/cmd.go
+++ b/lib/recovercollection/cmd.go
@@ -113,29 +113,54 @@ Options:
for _, src := range flags.Args() {
logger := logger.WithField("src", src)
var mtxt string
- if len(src) == 27 && src[5:12] == "-57u5n-" {
- var logent struct {
- EventType string `json:"event_type"`
- EventAt time.Time `json:"event_at"`
- ObjectUUID string `json:"object_uuid"`
- Properties struct {
- OldAttributes struct {
- ManifestText string `json:"manifest_text"`
- } `json:"old_attributes"`
- } `json:"properties"`
+ if len(src) == 27 {
+ var filters []arvados.Filter
+ if src[5:12] == "-57u5n-" {
+ filters = []arvados.Filter{{"uuid", "=", src}}
+ } else if src[5:12] == "-4zz18-" {
+ filters = []arvados.Filter{{"object_uuid", "=", src}}
}
- err = client.RequestAndDecode(&logent, "GET", "arvados/v1/logs/"+src, nil, nil)
+ var resp struct {
+ Items []struct {
+ UUID string `json:"uuid"`
+ EventType string `json:"event_type"`
+ EventAt time.Time `json:"event_at"`
+ ObjectUUID string `json:"object_uuid"`
+ Properties struct {
+ OldAttributes struct {
+ ManifestText string `json:"manifest_text"`
+ } `json:"old_attributes"`
+ } `json:"properties"`
+ }
+ }
+ err = client.RequestAndDecode(&resp, "GET", "arvados/v1/logs", nil, arvados.ListOptions{
+ Limit: 1,
+ Order: []string{"event_at desc"},
+ Filters: filters,
+ })
if err != nil {
- logger.WithError(err).Error("failed to load log entry")
+ logger.WithError(err).Error("error looking up log entry")
+ exitcode = 1
+ continue
+ } else if len(resp.Items) == 0 {
+ logger.Error("log entry not found")
exitcode = 1
continue
}
+ logent := resp.Items[0]
logger.WithFields(logrus.Fields{
+ "uuid": logent.UUID,
"old_collection_uuid": logent.ObjectUUID,
"logged_event_type": logent.EventType,
"logged_event_time": logent.EventAt,
+ "logged_object_uuid": logent.ObjectUUID,
}).Info("loaded log entry")
mtxt = logent.Properties.OldAttributes.ManifestText
+ if mtxt == "" {
+ logger.Error("log entry properties.old_attributes.manifest_text missing or empty")
+ exitcode = 1
+ continue
+ }
} else {
buf, err := ioutil.ReadFile(src)
if err != nil {
diff --git a/lib/recovercollection/cmd_test.go b/lib/recovercollection/cmd_test.go
index a6bf19de2..f8ec61f53 100644
--- a/lib/recovercollection/cmd_test.go
+++ b/lib/recovercollection/cmd_test.go
@@ -36,7 +36,7 @@ func (*Suite) TestUnrecoverableBlock(c *check.C) {
mfile := tmp + "/manifest"
ioutil.WriteFile(mfile, []byte(". aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa+410 0:410:Gone\n"), 0777)
var stdout, stderr bytes.Buffer
- exitcode := Command.RunCommand("undelete.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 1)
c.Check(stdout.String(), check.Equals, "")
c.Log(stderr.String())
@@ -93,7 +93,7 @@ func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
}
var stdout, stderr bytes.Buffer
- exitcode := Command.RunCommand("undelete.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
+ exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", mfile}, &bytes.Buffer{}, &stdout, &stderr)
c.Check(exitcode, check.Equals, 0)
c.Check(stdout.String(), check.Matches, `zzzzz-4zz18-.{15}\n`)
c.Log(stderr.String())
@@ -115,3 +115,21 @@ func (*Suite) TestUntrashAndTouchBlock(c *check.C) {
}
c.Check(found, check.Equals, true)
}
+
+func (*Suite) TestUnusableManifestSourceArg(c *check.C) {
+ for _, trial := range []struct {
+ srcArg string
+ errRegexp string
+ }{
+ {"zzzzz-4zz18-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found".*`},
+ {"zzzzz-57u5n-aaaaaaaaaaaaaaa", `(?ms).*msg="log entry not found.*`},
+ {"zzzzz-57u5n-containerlog006", `(?ms).*msg="log entry properties\.old_attributes\.manifest_text missing or empty".*`},
+ } {
+ var stdout, stderr bytes.Buffer
+ exitcode := Command.RunCommand("recovercollection.test", []string{"-log-level=debug", trial.srcArg}, &bytes.Buffer{}, &stdout, &stderr)
+ c.Check(exitcode, check.Equals, 1)
+ c.Check(stdout.String(), check.Equals, "")
+ c.Log(stderr.String())
+ c.Check(stderr.String(), check.Matches, trial.errRegexp)
+ }
+}
commit 5fd885a3037f1bc98344c17a68fcdeff75ab974b
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Jun 10 10:26:18 2020 -0400
16427: Rename undelete -> recover-collection.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/cmd/arvados-server/cmd.go b/cmd/arvados-server/cmd.go
index 1b2de11ac..ff99de75c 100644
--- a/cmd/arvados-server/cmd.go
+++ b/cmd/arvados-server/cmd.go
@@ -15,7 +15,7 @@ import (
"git.arvados.org/arvados.git/lib/crunchrun"
"git.arvados.org/arvados.git/lib/dispatchcloud"
"git.arvados.org/arvados.git/lib/install"
- "git.arvados.org/arvados.git/lib/undelete"
+ "git.arvados.org/arvados.git/lib/recovercollection"
"git.arvados.org/arvados.git/services/ws"
)
@@ -25,17 +25,17 @@ var (
"-version": cmd.Version,
"--version": cmd.Version,
- "boot": boot.Command,
- "cloudtest": cloudtest.Command,
- "config-check": config.CheckCommand,
- "config-defaults": config.DumpDefaultsCommand,
- "config-dump": config.DumpCommand,
- "controller": controller.Command,
- "crunch-run": crunchrun.Command,
- "dispatch-cloud": dispatchcloud.Command,
- "install": install.Command,
- "undelete": undelete.Command,
- "ws": ws.Command,
+ "boot": boot.Command,
+ "cloudtest": cloudtest.Command,
+ "config-check": config.CheckCommand,
+ "config-defaults": config.DumpDefaultsCommand,
+ "config-dump": config.DumpCommand,
+ "controller": controller.Command,
+ "crunch-run": crunchrun.Command,
+ "dispatch-cloud": dispatchcloud.Command,
+ "install": install.Command,
+ "recover-collection": recovercollection.Command,
+ "ws": ws.Command,
})
)
diff --git a/doc/_config.yml b/doc/_config.yml
index 9f016fcc8..3b59cbca4 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -174,7 +174,7 @@ navbar:
- admin/logs-table-management.html.textile.liquid
- admin/workbench2-vocabulary.html.textile.liquid
- admin/storage-classes.html.textile.liquid
- - admin/undeleting-collections.html.textile.liquid
+ - admin/recovering-deleted-collections.html.textile.liquid
- Cloud:
- admin/spot-instances.html.textile.liquid
- admin/cloudtest.html.textile.liquid
diff --git a/doc/admin/undeleting-collections.html.textile.liquid b/doc/admin/recovering-deleted-collections.html.textile.liquid
similarity index 72%
rename from doc/admin/undeleting-collections.html.textile.liquid
rename to doc/admin/recovering-deleted-collections.html.textile.liquid
index 461671b53..59c576ce0 100644
--- a/doc/admin/undeleting-collections.html.textile.liquid
+++ b/doc/admin/recovering-deleted-collections.html.textile.liquid
@@ -1,7 +1,7 @@
---
layout: default
navsection: admin
-title: Undeleting collections
+title: Recovering deleted collections
...
{% comment %}
@@ -18,14 +18,14 @@ Possibility of recovery depends on many factors, including:
* Whether the data blocks have been unreferenced long enough to be marked for deletion/trash by keep-balance
* Blob signature TTL, trash lifetime, trash check interval, and other config settings
-To attempt recovery of a previous version of a deleted/modified collection, use the @arvados-server undelete@ command. It should be run on one of your server nodes where the @arvados-server@ package is installed and the @/etc/arvados/config.yml@ file is up to date.
+To attempt recovery of a previous version of a deleted/modified collection, use the @arvados-server recover-collection@ command. It should be run on one of your server nodes where the @arvados-server@ package is installed and the @/etc/arvados/config.yml@ file is up to date.
Specify the collection you want to recover by passing either the UUID of an audit log entry, or a file containing the manifest.
-If recovery is successful, the undelete program saves the recovered data a new collection belonging to the system user, and print the new collection's UUID on stdout.
+If recovery is successful, the @recover-collection@ program saves the recovered data a new collection belonging to the system user, and prints the new collection's UUID on stdout.
<pre>
-# arvados-server undelete 9tee4-57u5n-nb5awmk1pahac2t
+# arvados-server recover-collection 9tee4-57u5n-nb5awmk1pahac2t
INFO[2020-06-05T19:52:29.557761245Z] loaded log entry logged_event_time="2020-06-05 16:48:01.438791 +0000 UTC" logged_event_type=update old_collection_uuid=9tee4-4zz18-1ex26g95epmgw5w src=9tee4-57u5n-nb5awmk1pahac2t
INFO[2020-06-05T19:52:29.642145127Z] recovery succeeded UUID=9tee4-4zz18-5trfp4k4xxg97f1 src=9tee4-57u5n-nb5awmk1pahac2t
9tee4-4zz18-5trfp4k4xxg97f1
@@ -34,4 +34,4 @@ INFO[2020-06-05T19:52:29.644699436Z] exiting
In this example, the original data has been restored and saved in a new collection with UUID @9tee4-4zz18-5trfp4k4xxg97f1 at .
-For more options, run @arvados-server undelete -help at .
+For more options, run @arvados-server recover-collection -help at .
diff --git a/lib/undelete/cmd.go b/lib/recovercollection/cmd.go
similarity index 82%
rename from lib/undelete/cmd.go
rename to lib/recovercollection/cmd.go
index 9d4bc84ea..d19bf3116 100644
--- a/lib/undelete/cmd.go
+++ b/lib/recovercollection/cmd.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: AGPL-3.0
-package undelete
+package recovercollection
import (
"context"
@@ -103,7 +103,7 @@ Options:
return 1
}
client.AuthToken = cluster.SystemRootToken
- und := undeleter{
+ rcvr := recoverer{
client: client,
cluster: cluster,
logger: logger,
@@ -145,7 +145,7 @@ Options:
}
mtxt = string(buf)
}
- uuid, err := und.RecoverManifest(string(mtxt))
+ uuid, err := rcvr.RecoverManifest(string(mtxt))
if err != nil {
logger.WithError(err).Error("recovery failed")
exitcode = 1
@@ -157,7 +157,7 @@ Options:
return exitcode
}
-type undeleter struct {
+type recoverer struct {
client *arvados.Client
cluster *arvados.Cluster
logger logrus.FieldLogger
@@ -167,8 +167,8 @@ var errNotFound = errors.New("not found")
// Finds the timestamp of the newest copy of blk on svc. Returns
// errNotFound if blk is not on svc at all.
-func (und undeleter) newestMtime(logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
- found, err := svc.Index(und.client, blk)
+func (rcvr recoverer) newestMtime(logger logrus.FieldLogger, blk string, svc arvados.KeepService) (time.Time, error) {
+ found, err := svc.Index(rcvr.client, blk)
if err != nil {
logger.WithError(err).Warn("error getting index")
return time.Time{}, err
@@ -198,17 +198,17 @@ var errTouchIneffective = errors.New("(BUG?) touch succeeded but had no effect -
// decide to trash it, all before our recovered collection gets
// saved. But if the block's timestamp is more recent than blobsigttl,
// keepstore will refuse to trash it even if told to by keep-balance.
-func (und undeleter) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
- if latest, err := und.newestMtime(logger, blk, svc); err != nil {
+func (rcvr recoverer) ensureSafe(ctx context.Context, logger logrus.FieldLogger, blk string, svc arvados.KeepService, blobsigttl time.Duration, blobsigexp time.Time) error {
+ if latest, err := rcvr.newestMtime(logger, blk, svc); err != nil {
return err
} else if latest.Add(blobsigttl).After(blobsigexp) {
return nil
}
- if err := svc.Touch(ctx, und.client, blk); err != nil {
+ if err := svc.Touch(ctx, rcvr.client, blk); err != nil {
return fmt.Errorf("error updating timestamp: %s", err)
}
logger.Debug("updated timestamp")
- if latest, err := und.newestMtime(logger, blk, svc); err == errNotFound {
+ if latest, err := rcvr.newestMtime(logger, blk, svc); err == errNotFound {
return fmt.Errorf("(BUG?) touch succeeded, but then block did not appear in index")
} else if err != nil {
return err
@@ -222,7 +222,7 @@ func (und undeleter) ensureSafe(ctx context.Context, logger logrus.FieldLogger,
// Untrash and update GC timestamps (as needed) on blocks referenced
// by the given manifest, save a new collection and return the new
// collection's UUID.
-func (und undeleter) RecoverManifest(mtxt string) (string, error) {
+func (rcvr recoverer) RecoverManifest(mtxt string) (string, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -238,9 +238,9 @@ func (und undeleter) RecoverManifest(mtxt string) (string, error) {
go close(todo)
var services []arvados.KeepService
- err = und.client.EachKeepService(func(svc arvados.KeepService) error {
+ err = rcvr.client.EachKeepService(func(svc arvados.KeepService) error {
if svc.ServiceType == "proxy" {
- und.logger.WithField("service", svc).Debug("ignore proxy service")
+ rcvr.logger.WithField("service", svc).Debug("ignore proxy service")
} else {
services = append(services, svc)
}
@@ -249,7 +249,7 @@ func (und undeleter) RecoverManifest(mtxt string) (string, error) {
if err != nil {
return "", fmt.Errorf("error getting list of keep services: %s", err)
}
- und.logger.WithField("services", services).Debug("got list of services")
+ rcvr.logger.WithField("services", services).Debug("got list of services")
// blobsigexp is our deadline for saving the rescued
// collection. This must be less than BlobSigningTTL
@@ -263,9 +263,9 @@ func (und undeleter) RecoverManifest(mtxt string) (string, error) {
// would have lived long enough anyway if left alone.
// BlobSigningTTL/2 (typically around 1 week) is much longer
// than than we need to recover even a very large collection.
- blobsigttl := und.cluster.Collections.BlobSigningTTL.Duration()
+ blobsigttl := rcvr.cluster.Collections.BlobSigningTTL.Duration()
blobsigexp := time.Now().Add(blobsigttl / 2)
- und.logger.WithField("blobsigexp", blobsigexp).Debug("chose save deadline")
+ rcvr.logger.WithField("blobsigexp", blobsigexp).Debug("chose save deadline")
// We'll start a number of threads, each working on
// checking/recovering one block at a time. The threads
@@ -283,18 +283,18 @@ func (und undeleter) RecoverManifest(mtxt string) (string, error) {
nextblk:
for idx := range todo {
blk := strings.SplitN(string(blks[idx]), "+", 2)[0]
- logger := und.logger.WithField("block", blk)
+ logger := rcvr.logger.WithField("block", blk)
for _, untrashing := range []bool{false, true} {
for _, svc := range services {
logger := logger.WithField("service", fmt.Sprintf("%s:%d", svc.ServiceHost, svc.ServicePort))
if untrashing {
- if err := svc.Untrash(ctx, und.client, blk); err != nil {
+ if err := svc.Untrash(ctx, rcvr.client, blk); err != nil {
logger.WithError(err).Debug("untrash failed")
continue
}
logger.Info("untrashed")
}
- err := und.ensureSafe(ctx, logger, blk, svc, blobsigttl, blobsigexp)
+ err := rcvr.ensureSafe(ctx, logger, blk, svc, blobsigttl, blobsigexp)
if err == errNotFound {
logger.Debug(err)
} else if err != nil {
@@ -321,17 +321,17 @@ func (und undeleter) RecoverManifest(mtxt string) (string, error) {
}
if havenot > 0 {
if have > 0 {
- und.logger.Warn("partial recovery is not implemented")
+ rcvr.logger.Warn("partial recovery is not implemented")
}
return "", fmt.Errorf("unable to recover %d of %d blocks", havenot, have+havenot)
}
- if und.cluster.Collections.BlobSigning {
- key := []byte(und.cluster.Collections.BlobSigningKey)
- coll.ManifestText = arvados.SignManifest(coll.ManifestText, und.client.AuthToken, blobsigexp, blobsigttl, key)
+ if rcvr.cluster.Collections.BlobSigning {
+ key := []byte(rcvr.cluster.Collections.BlobSigningKey)
+ coll.ManifestText = arvados.SignManifest(coll.ManifestText, rcvr.client.AuthToken, blobsigexp, blobsigttl, key)
}
- und.logger.WithField("manifest", coll.ManifestText).Debug("updated blob signatures in manifest")
- err = und.client.RequestAndDecodeContext(ctx, &coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ rcvr.logger.WithField("manifest", coll.ManifestText).Debug("updated blob signatures in manifest")
+ err = rcvr.client.RequestAndDecodeContext(ctx, &coll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
"collection": map[string]interface{}{
"manifest_text": coll.ManifestText,
},
@@ -339,6 +339,6 @@ func (und undeleter) RecoverManifest(mtxt string) (string, error) {
if err != nil {
return "", fmt.Errorf("error saving new collection: %s", err)
}
- und.logger.WithField("UUID", coll.UUID).Debug("created new collection")
+ rcvr.logger.WithField("UUID", coll.UUID).Debug("created new collection")
return coll.UUID, nil
}
diff --git a/lib/undelete/cmd_test.go b/lib/recovercollection/cmd_test.go
similarity index 99%
rename from lib/undelete/cmd_test.go
rename to lib/recovercollection/cmd_test.go
index a5edaf90b..a6bf19de2 100644
--- a/lib/undelete/cmd_test.go
+++ b/lib/recovercollection/cmd_test.go
@@ -2,7 +2,7 @@
//
// SPDX-License-Identifier: AGPL-3.0
-package undelete
+package recovercollection
import (
"bytes"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list