[ARVADOS] updated: a74c81b035c67d299e2a7298f8db3d368a578510
Git user
git at public.curoverse.com
Fri Jul 1 13:56:17 EDT 2016
Summary of changes:
doc/install/install-keepstore.html.textile.liquid | 78 ++++++++++++++++------
sdk/go/arvados/client_test.go | 2 +-
sdk/go/arvados/keep_service.go | 9 +++
sdk/go/crunchrunner/crunchrunner_test.go | 40 +++++------
sdk/go/keepclient/collectionreader.go | 2 +-
.../crunch-dispatch-local/crunch-dispatch-local.go | 2 +-
.../crunch-dispatch-local_test.go | 2 +-
.../crunch-dispatch-slurm_test.go | 2 +-
services/crunch-run/crunchrun.go | 2 +-
services/crunchstat/crunchstat_test.go | 2 +-
services/datamanager/collection/collection.go | 12 ++--
services/datamanager/collection/collection_test.go | 14 ++--
services/datamanager/keep/keep.go | 20 ++++--
services/datamanager/keep/keep_test.go | 4 +-
services/datamanager/summary/pull_list_test.go | 50 +++++++-------
services/datamanager/summary/summary_test.go | 24 +++----
services/datamanager/summary/trash_list.go | 2 +-
services/datamanager/summary/trash_list_test.go | 30 ++++-----
services/keep-balance/balance.go | 4 +-
services/keep-balance/balance_test.go | 4 +-
services/keepstore/azure_blob_volume.go | 2 +-
services/keepstore/keepstore.go | 4 +-
services/keepstore/s3_volume.go | 4 +-
services/keepstore/trash_worker.go | 6 +-
services/keepstore/trash_worker_test.go | 2 +-
services/keepstore/volume_generic_test.go | 37 +++++++---
services/keepstore/volume_unix.go | 7 +-
services/keepstore/work_queue_test.go | 2 +-
28 files changed, 222 insertions(+), 147 deletions(-)
via a74c81b035c67d299e2a7298f8db3d368a578510 (commit)
via 3546fd8b7c50ba7a1f4c088f4ad3ab5392cdf548 (commit)
via b7569de781d06b65f2daf2b269ff185eb2ed13c1 (commit)
via f9e3d32c92920a806d50548dbb9b6c0aab7d8c54 (commit)
via 15c918a67ceda31bde38cab75736d89bd015476a (commit)
from 984e8475628cc18eaf600bea2bbdc7ce228706e8 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit a74c81b035c67d299e2a7298f8db3d368a578510
Merge: 984e847 3546fd8
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jul 1 13:53:21 2016 -0400
Merge branch '9437-keep-index-ns'
refs #9437
See https://dev.arvados.org/issues/9437#note-3 and #note-4
commit 3546fd8b7c50ba7a1f4c088f4ad3ab5392cdf548
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jul 1 10:10:28 2016 -0400
9437: Clarify units for -blob-signature-ttl. Update keepstore -h message in install docs.
diff --git a/doc/install/install-keepstore.html.textile.liquid b/doc/install/install-keepstore.html.textile.liquid
index 6548422..102a3f4 100644
--- a/doc/install/install-keepstore.html.textile.liquid
+++ b/doc/install/install-keepstore.html.textile.liquid
@@ -35,28 +35,66 @@ Verify that Keepstore is functional:
<notextile>
<pre><code>~$ <span class="userinput">keepstore -h</span>
-2015/05/08 13:41:16 keepstore starting, pid 2565
+2016/07/01 14:06:21 keepstore starting, pid 32339
Usage of ./keepstore:
- -azure-storage-account-key-file="": File containing the account key used for subsequent --azure-storage-container-volume arguments.
- -azure-storage-account-name="": Azure storage account name used for subsequent --azure-storage-container-volume arguments.
- -azure-storage-container-volume=[]: Use the given container as a storage volume. Can be given multiple times.
- -azure-storage-replication=3: Replication level to report to clients when data is stored in an Azure container.
- -blob-signature-ttl=1209600: Lifetime of blob permission signatures. Modifying the ttl will invalidate all existing signatures. See services/api/config/application.default.yml.
- -blob-signing-key-file="": File containing the secret key for generating and verifying blob permission signatures.
- -data-manager-token-file="": File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
- -enforce-permissions=false: Enforce permission signatures on requests.
- -listen=":25107": Listening address, in the form "host:port". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces.
- -max-buffers=128: Maximum RAM to use for data buffers, given in multiples of block size (64 MiB). When this limit is reached, HTTP requests requiring buffers (like GET and PUT) will wait for buffer space to be released.
+ -azure-max-get-bytes int
+ Maximum bytes to request in a single GET request. If smaller than 67108864, use multiple concurrent range requests to retrieve a block. (default 67108864)
+ -azure-storage-account-key-file string
+ File containing the account key used for subsequent --azure-storage-container-volume arguments.
+ -azure-storage-account-name string
+ Azure storage account name used for subsequent --azure-storage-container-volume arguments.
+ -azure-storage-container-volume value
+ Use the given container as a storage volume. Can be given multiple times. (default [])
+ -azure-storage-replication int
+ Replication level to report to clients when data is stored in an Azure container. (default 3)
+ -blob-signature-ttl int
+ Lifetime of blob permission signatures in seconds. Modifying the ttl will invalidate all existing signatures. See services/api/config/application.default.yml. (default 1209600)
+ -blob-signing-key-file string
+ File containing the secret key for generating and verifying blob permission signatures.
+ -data-manager-token-file string
+ File with the API token used by the Data Manager. All DELETE requests or GET /index requests must carry this token.
+ -enforce-permissions
+ Enforce permission signatures on requests.
+ -listen string
+ Listening address, in the form "host:port". e.g., 10.0.1.24:8000. Omit the host part to listen on all interfaces. (default ":25107")
+ -max-buffers int
+ Maximum RAM to use for data buffers, given in multiples of block size (64 MiB). When this limit is reached, HTTP requests requiring buffers (like GET and PUT) will wait for buffer space to be released. (default 128)
-max-requests int
- Maximum concurrent requests. When this limit is reached, new requests will receive 503 responses. Note: this limit does not include idle connections from clients using HTTP keepalive, so it does not strictly limit the number of concurrent connections. (default 2 * max-buffers)
- -never-delete=false: If set, nothing will be deleted. HTTP 405 will be returned for valid DELETE requests.
- -permission-key-file="": Synonym for -blob-signing-key-file.
- -permission-ttl=0: Synonym for -blob-signature-ttl.
- -pid="": Path to write pid file during startup. This file is kept open and locked with LOCK_EX until keepstore exits, so `fuser -k pidfile` is one way to shut down. Exit immediately if there is an error opening, locking, or writing the pid file.
- -readonly=false: Do not write, delete, or touch anything on the following volumes.
- -serialize=false: Serialize read and write operations on the following volumes.
- -volume=[]: Local storage directory. Can be given more than once to add multiple directories. If none are supplied, the default is to use all directories named "keep" that exist in the top level directory of a mount point at startup time. Can be a comma-separated list, but this is deprecated: use multiple -volume arguments instead.
- -volumes=[]: Deprecated synonym for -volume.
+ Maximum concurrent requests. When this limit is reached, new requests will receive 503 responses. Note: this limit does not include idle connections from clients using HTTP keepalive, so it does not strictly limit the number of concurrent connections. (default 2 * max-buffers)
+ -never-delete
+ If true, nothing will be deleted. Warning: the relevant features in keepstore and data manager have not been extensively tested. You should leave this option alone unless you can afford to lose data. (default true)
+ -permission-key-file string
+ Synonym for -blob-signing-key-file.
+ -permission-ttl int
+ Synonym for -blob-signature-ttl.
+ -pid fuser -k pidfile
+ Path to write pid file during startup. This file is kept open and locked with LOCK_EX until keepstore exits, so fuser -k pidfile is one way to shut down. Exit immediately if there is an error opening, locking, or writing the pid file.
+ -readonly
+ Do not write, delete, or touch anything on the following volumes.
+ -s3-access-key-file string
+ File containing the access key used for subsequent -s3-bucket-volume arguments.
+ -s3-bucket-volume value
+ Use the given bucket as a storage volume. Can be given multiple times. (default [])
+ -s3-endpoint string
+ Endpoint URL used for subsequent -s3-bucket-volume arguments. If blank, use the AWS endpoint corresponding to the -s3-region argument. For Google Storage, use "https://storage.googleapis.com".
+ -s3-region string
+ AWS region used for subsequent -s3-bucket-volume arguments. Allowed values are ["ap-southeast-1" "eu-west-1" "us-gov-west-1" "sa-east-1" "cn-north-1" "ap-northeast-1" "ap-southeast-2" "eu-central-1" "us-east-1" "us-west-1" "us-west-2"].
+ -s3-replication int
+ Replication level reported to clients for subsequent -s3-bucket-volume arguments. (default 2)
+ -s3-secret-key-file string
+ File containing the secret key used for subsequent -s3-bucket-volume arguments.
+ -s3-unsafe-delete
+ EXPERIMENTAL. Enable deletion (garbage collection), even though there are known race conditions that can cause data loss.
+ -serialize
+ Serialize read and write operations on the following volumes.
+ -trash-check-interval duration
+ Time duration at which the emptyTrash goroutine will check and delete expired trashed blocks. Default is one day. (default 24h0m0s)
+ -trash-lifetime duration
+ Time duration after a block is trashed during which it can be recovered using an /untrash request
+ -volume value
+ Local storage directory. Can be given more than once to add multiple directories. If none are supplied, the default is to use all directories named "keep" that exist in the top level directory of a mount point at startup time. Can be a comma-separated list, but this is deprecated: use multiple -volume arguments instead. (default [])
+ -volumes value
+ Deprecated synonym for -volume. (default [])
</code></pre>
</notextile>
diff --git a/services/keepstore/keepstore.go b/services/keepstore/keepstore.go
index 819d52f..00f7b3c 100644
--- a/services/keepstore/keepstore.go
+++ b/services/keepstore/keepstore.go
@@ -197,8 +197,8 @@ func main() {
flag.IntVar(
&permissionTTLSec,
"blob-signature-ttl",
- int(time.Duration(2*7*24*time.Hour).Seconds()),
- "Lifetime of blob permission signatures. Modifying the ttl will invalidate all existing signatures. "+
+ 2*7*24*3600,
+ "Lifetime of blob permission signatures in seconds. Modifying the ttl will invalidate all existing signatures. "+
"See services/api/config/application.default.yml.")
flag.BoolVar(
&flagSerializeIO,
commit b7569de781d06b65f2daf2b269ff185eb2ed13c1
Author: Tom Clegg <tom at curoverse.com>
Date: Fri Jun 24 10:08:45 2016 -0400
9437: Accept 1-second timestamps from old keepstore servers.
Running a new keep-balance/datamanager with old keepstore servers will
interpret timestamps correctly.
Running an old keep-balance/datamanager with new keepstore servers
will not perform any garbage collection: interpreting the new
nanosecond timestamps as 1-second timestamps will result in all blocks
appearing to come from the future, making them ineligible for
trash/delete.
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 87dbd2a..b29748a 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -109,6 +109,14 @@ func (s *KeepService) Index(c *Client, prefix string) ([]KeepServiceIndexEntry,
if err != nil {
return nil, fmt.Errorf("Malformed index line %q: mtime: %v", line, err)
}
+ if mtime < 1e12 {
+ // An old version of keepstore is giving us
+ // timestamps in seconds instead of
+ // nanoseconds. (This threshold correctly
+ // handles all times between 1970-01-02 and
+ // 33658-09-27.)
+ mtime = mtime * 1e9
+ }
entries = append(entries, KeepServiceIndexEntry{
SizedDigest: SizedDigest(fields[0]),
Mtime: mtime,
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index e7843ea..651c869 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -430,13 +430,23 @@ func parseBlockInfoFromIndexLine(indexLine string) (blockInfo BlockInfo, err err
return
}
- blockInfo.Mtime, err = strconv.ParseInt(tokens[1], 10, 64)
+ var ns int64
+ ns, err = strconv.ParseInt(tokens[1], 10, 64)
if err != nil {
return
}
- blockInfo.Digest =
- blockdigest.DigestWithSize{Digest: locator.Digest,
- Size: uint32(locator.Size)}
+ if ns < 1e12 {
+ // An old version of keepstore is giving us timestamps
+ // in seconds instead of nanoseconds. (This threshold
+ // correctly handles all times between 1970-01-02 and
+ // 33658-09-27.)
+ ns = ns * 1e9
+ }
+ blockInfo.Mtime = ns
+ blockInfo.Digest = blockdigest.DigestWithSize{
+ Digest: locator.Digest,
+ Size: uint32(locator.Size),
+ }
return
}
commit f9e3d32c92920a806d50548dbb9b6c0aab7d8c54
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jun 23 16:22:56 2016 -0400
9437: gofmt
diff --git a/sdk/go/arvados/client_test.go b/sdk/go/arvados/client_test.go
index 422ad90..5011aa8 100644
--- a/sdk/go/arvados/client_test.go
+++ b/sdk/go/arvados/client_test.go
@@ -117,7 +117,7 @@ func TestAnythingToValues(t *testing.T) {
},
},
{
- in: map[string]interface{}{"foo": map[string]interface{}{"bar":1.234}},
+ in: map[string]interface{}{"foo": map[string]interface{}{"bar": 1.234}},
ok: func(out url.Values) bool {
return out.Get("foo") == `{"bar":1.234}`
},
diff --git a/sdk/go/crunchrunner/crunchrunner_test.go b/sdk/go/crunchrunner/crunchrunner_test.go
index 52d5c1a..9805412 100644
--- a/sdk/go/crunchrunner/crunchrunner_test.go
+++ b/sdk/go/crunchrunner/crunchrunner_test.go
@@ -53,7 +53,7 @@ func (s *TestSuite) TestSimpleRun(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"}}}}},
Task{Sequence: 0})
c.Check(err, IsNil)
@@ -89,8 +89,8 @@ func (s *TestSuite) TestSimpleRunSubtask(c *C) {
tmpdir,
"",
Job{Script_parameters: Tasks{[]TaskDef{
- TaskDef{Command: []string{"echo", "bar"}},
- TaskDef{Command: []string{"echo", "foo"}}}}},
+ {Command: []string{"echo", "bar"}},
+ {Command: []string{"echo", "foo"}}}}},
Task{Parameters: TaskDef{
Command: []string{"echo", "foo"},
Stdout: "output.txt"},
@@ -118,7 +118,7 @@ func (s *TestSuite) TestRedirect(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"cat"},
Stdout: "output.txt",
Stdin: tmpfile.Name()}}}},
@@ -140,7 +140,7 @@ func (s *TestSuite) TestEnv(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $BAR"},
Stdout: "output.txt",
Env: map[string]string{"BAR": "foo"}}}}},
@@ -161,7 +161,7 @@ func (s *TestSuite) TestEnvSubstitute(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"foo\n",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $BAR"},
Stdout: "output.txt",
Env: map[string]string{"BAR": "$(task.keep)"}}}}},
@@ -182,7 +182,7 @@ func (s *TestSuite) TestEnvReplace(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "echo $PATH"},
Stdout: "output.txt",
Env: map[string]string{"PATH": "foo"}}}}},
@@ -211,12 +211,12 @@ func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters a
func (s *TestSuite) TestScheduleSubtask(c *C) {
api := SubtaskTestClient{c, []Task{
- Task{Job_uuid: "zzzz-8i9sb-111111111111111",
+ {Job_uuid: "zzzz-8i9sb-111111111111111",
Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
Sequence: 1,
Parameters: TaskDef{
Command: []string{"echo", "bar"}}},
- Task{Job_uuid: "zzzz-8i9sb-111111111111111",
+ {Job_uuid: "zzzz-8i9sb-111111111111111",
Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
Sequence: 1,
Parameters: TaskDef{
@@ -234,8 +234,8 @@ func (s *TestSuite) TestScheduleSubtask(c *C) {
tmpdir,
"",
Job{Script_parameters: Tasks{[]TaskDef{
- TaskDef{Command: []string{"echo", "bar"}},
- TaskDef{Command: []string{"echo", "foo"}}}}},
+ {Command: []string{"echo", "bar"}},
+ {Command: []string{"echo", "foo"}}}}},
Task{Sequence: 0})
c.Check(err, IsNil)
@@ -252,7 +252,7 @@ func (s *TestSuite) TestRunFail(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
@@ -269,7 +269,7 @@ func (s *TestSuite) TestRunSuccessCode(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"},
SuccessCodes: []int{0, 1}}}}},
Task{Sequence: 0})
@@ -287,7 +287,7 @@ func (s *TestSuite) TestRunFailCode(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 0"},
PermanentFailCodes: []int{0, 1}}}}},
Task{Sequence: 0})
@@ -305,7 +305,7 @@ func (s *TestSuite) TestRunTempFailCode(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"/bin/sh", "-c", "exit 1"},
TemporaryFailCodes: []int{1}}}}},
Task{Sequence: 0})
@@ -329,7 +329,7 @@ func (s *TestSuite) TestVwd(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"ls", "output.txt"},
Vwd: map[string]string{
"output.txt": tmpfile.Name()}}}}},
@@ -361,7 +361,7 @@ func (s *TestSuite) TestSubstitutionStdin(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"cat"},
Stdout: "output.txt",
Stdin: "$(task.keep)/file1.txt"}}}},
@@ -389,7 +389,7 @@ func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"cat", "$(task.keep)/file1.txt"},
Stdout: "output.txt"}}}},
Task{Sequence: 0})
@@ -417,7 +417,7 @@ func (s *TestSuite) TestSignal(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"sleep", "4"}}}}},
Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
@@ -437,7 +437,7 @@ func (s *TestSuite) TestQuoting(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Job{Script_parameters: Tasks{[]TaskDef{{
Command: []string{"echo", "foo"},
Stdout: "s ub:dir/:e vi\nl"}}}},
Task{Sequence: 0})
diff --git a/sdk/go/keepclient/collectionreader.go b/sdk/go/keepclient/collectionreader.go
index bed60f4..33bb587 100644
--- a/sdk/go/keepclient/collectionreader.go
+++ b/sdk/go/keepclient/collectionreader.go
@@ -243,7 +243,7 @@ GET:
// In case we exited the above loop early: before returning,
// drain the toGet channel so its sender doesn't sit around
// blocking forever.
- for _ = range r.toGet {
+ for range r.toGet {
}
}
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local.go b/services/crunch-dispatch-local/crunch-dispatch-local.go
index 936a908..0ca7651 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local.go
@@ -168,7 +168,7 @@ func run(dispatcher *dispatch.Dispatcher,
}
// drain any subsequent status changes
- for _ = range status {
+ for range status {
}
log.Printf("Finalized container %v", uuid)
diff --git a/services/crunch-dispatch-local/crunch-dispatch-local_test.go b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
index 9628bf2..17f9d67 100644
--- a/services/crunch-dispatch-local/crunch-dispatch-local_test.go
+++ b/services/crunch-dispatch-local/crunch-dispatch-local_test.go
@@ -88,7 +88,7 @@ func (s *TestSuite) TestIntegration(c *C) {
// There should be no queued containers now
params := arvadosclient.Dict{
- "filters": [][]string{[]string{"state", "=", "Queued"}},
+ "filters": [][]string{{"state", "=", "Queued"}},
}
var containers arvados.ContainerList
err = arv.List("containers", params, &containers)
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index b72ad9f..d8b94f9 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -136,7 +136,7 @@ func (s *TestSuite) integrationTest(c *C,
// There should be no queued containers now
params := arvadosclient.Dict{
- "filters": [][]string{[]string{"state", "=", "Queued"}},
+ "filters": [][]string{{"state", "=", "Queued"}},
}
var containers arvados.ContainerList
err = arv.List("containers", params, &containers)
diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 7da1beb..2fd0029 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -102,7 +102,7 @@ func (runner *ContainerRunner) SetupSignals() {
signal.Notify(runner.SigChan, syscall.SIGQUIT)
go func(sig <-chan os.Signal) {
- for _ = range sig {
+ for range sig {
if !runner.Cancelled {
runner.CancelLock.Lock()
runner.Cancelled = true
diff --git a/services/crunchstat/crunchstat_test.go b/services/crunchstat/crunchstat_test.go
index 69f31af..fe95f03 100644
--- a/services/crunchstat/crunchstat_test.go
+++ b/services/crunchstat/crunchstat_test.go
@@ -89,7 +89,7 @@ func TestCopyPipeToChildLogLongLines(t *testing.T) {
}
line = line[5:]
}
- if len(line) >= 6 && string(line[len(line)-6:len(line)]) == "[...]\n" {
+ if len(line) >= 6 && string(line[len(line)-6:]) == "[...]\n" {
line = line[:len(line)-6]
} else {
done = true
diff --git a/services/datamanager/collection/collection.go b/services/datamanager/collection/collection.go
index 55b3f61..5fcacff 100644
--- a/services/datamanager/collection/collection.go
+++ b/services/datamanager/collection/collection.go
@@ -49,11 +49,11 @@ type GetCollectionsParams struct {
// SdkCollectionInfo holds collection info from api
type SdkCollectionInfo struct {
- UUID string `json:"uuid"`
- OwnerUUID string `json:"owner_uuid"`
- ReplicationDesired int `json:"replication_desired"`
- ModifiedAt time.Time `json:"modified_at"`
- ManifestText string `json:"manifest_text"`
+ UUID string `json:"uuid"`
+ OwnerUUID string `json:"owner_uuid"`
+ ReplicationDesired int `json:"replication_desired"`
+ ModifiedAt time.Time `json:"modified_at"`
+ ManifestText string `json:"manifest_text"`
}
// SdkCollectionList lists collections from api
@@ -131,7 +131,7 @@ func GetCollections(params GetCollectionsParams) (results ReadCollections, err e
sdkParams := arvadosclient.Dict{
"select": fieldsWanted,
"order": []string{"modified_at ASC", "uuid ASC"},
- "filters": [][]string{[]string{"modified_at", ">=", "1900-01-01T00:00:00Z"}},
+ "filters": [][]string{{"modified_at", ">=", "1900-01-01T00:00:00Z"}},
"offset": 0}
if params.BatchSize > 0 {
diff --git a/services/datamanager/collection/collection_test.go b/services/datamanager/collection/collection_test.go
index 47ab5fa..b23ef2c 100644
--- a/services/datamanager/collection/collection_test.go
+++ b/services/datamanager/collection/collection_test.go
@@ -64,7 +64,7 @@ func CompareSummarizedReadCollections(c *C,
}
func (s *MySuite) TestSummarizeSimple(checker *C) {
- rc := MakeTestReadCollections([]TestCollectionSpec{TestCollectionSpec{
+ rc := MakeTestReadCollections([]TestCollectionSpec{{
ReplicationLevel: 5,
Blocks: []int{1, 2},
}})
@@ -79,7 +79,7 @@ func (s *MySuite) TestSummarizeSimple(checker *C) {
expected := ExpectedSummary{
OwnerToCollectionSize: map[string]int{c.OwnerUUID: c.TotalSize},
BlockToDesiredReplication: map[blockdigest.DigestWithSize]int{blockDigest1: 5, blockDigest2: 5},
- BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: []string{c.UUID}, blockDigest2: []string{c.UUID}},
+ BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{blockDigest1: {c.UUID}, blockDigest2: {c.UUID}},
}
CompareSummarizedReadCollections(checker, rc, expected)
@@ -87,11 +87,11 @@ func (s *MySuite) TestSummarizeSimple(checker *C) {
func (s *MySuite) TestSummarizeOverlapping(checker *C) {
rc := MakeTestReadCollections([]TestCollectionSpec{
- TestCollectionSpec{
+ {
ReplicationLevel: 5,
Blocks: []int{1, 2},
},
- TestCollectionSpec{
+ {
ReplicationLevel: 8,
Blocks: []int{2, 3},
},
@@ -117,9 +117,9 @@ func (s *MySuite) TestSummarizeOverlapping(checker *C) {
blockDigest3: 8,
},
BlockToCollectionUuids: map[blockdigest.DigestWithSize][]string{
- blockDigest1: []string{c0.UUID},
- blockDigest2: []string{c0.UUID, c1.UUID},
- blockDigest3: []string{c1.UUID},
+ blockDigest1: {c0.UUID},
+ blockDigest2: {c0.UUID, c1.UUID},
+ blockDigest3: {c1.UUID},
},
}
diff --git a/services/datamanager/keep/keep.go b/services/datamanager/keep/keep.go
index 206a9c4..e7843ea 100644
--- a/services/datamanager/keep/keep.go
+++ b/services/datamanager/keep/keep.go
@@ -118,7 +118,7 @@ func GetKeepServersAndSummarize(params GetKeepServersParams) (results ReadServer
// GetKeepServers from api server
func GetKeepServers(params GetKeepServersParams) (results ReadServers, err error) {
sdkParams := arvadosclient.Dict{
- "filters": [][]string{[]string{"service_type", "!=", "proxy"}},
+ "filters": [][]string{{"service_type", "!=", "proxy"}},
}
if params.Limit > 0 {
sdkParams["limit"] = params.Limit
diff --git a/services/datamanager/keep/keep_test.go b/services/datamanager/keep/keep_test.go
index 79ff3f8..6698849 100644
--- a/services/datamanager/keep/keep_test.go
+++ b/services/datamanager/keep/keep_test.go
@@ -43,7 +43,7 @@ func (s *KeepSuite) TestSendTrashLists(c *C) {
defer server.Close()
tl := map[string]TrashList{
- server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}}
+ server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}}
arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
@@ -70,7 +70,7 @@ func (tse *TestHandlerError) ServeHTTP(writer http.ResponseWriter, req *http.Req
func sendTrashListError(c *C, server *httptest.Server) {
tl := map[string]TrashList{
- server.URL: TrashList{TrashRequest{"000000000000000000000000deadbeef", 99}}}
+ server.URL: {TrashRequest{"000000000000000000000000deadbeef", 99}}}
arv := arvadosclient.ArvadosClient{ApiToken: "abc123"}
kc := keepclient.KeepClient{Arvados: &arv, Client: &http.Client{}}
diff --git a/services/datamanager/summary/pull_list_test.go b/services/datamanager/summary/pull_list_test.go
index e2050c2..60b495c 100644
--- a/services/datamanager/summary/pull_list_test.go
+++ b/services/datamanager/summary/pull_list_test.go
@@ -164,69 +164,69 @@ func (s *PullSuite) TestBuildPullLists(c *C) {
locator1 := Locator{Digest: blockdigest.MakeTestBlockDigest(0xBadBeef)}
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{}, From: []string{}}}),
+ locator1: {To: []string{}, From: []string{}}}),
PullListMapEquals,
map[string]PullList{})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{}, From: []string{"f1", "f2"}}}),
+ locator1: {To: []string{}, From: []string{"f1", "f2"}}}),
PullListMapEquals,
map[string]PullList{})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{"t1"}, From: []string{"f1", "f2"}}}),
+ locator1: {To: []string{"t1"}, From: []string{"f1", "f2"}}}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{PullRequest{locator1, []string{"f1", "f2"}}}})
+ "t1": {PullRequest{locator1, []string{"f1", "f2"}}}})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{"t1"}, From: []string{}}}),
+ locator1: {To: []string{"t1"}, From: []string{}}}),
PullListMapEquals,
- map[string]PullList{"t1": PullList{
+ map[string]PullList{"t1": {
PullRequest{locator1, []string{}}}})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{
+ locator1: {
To: []string{"t1", "t2"},
From: []string{"f1", "f2"},
}}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{PullRequest{locator1, []string{"f1", "f2"}}},
- "t2": PullList{PullRequest{locator1, []string{"f1", "f2"}}},
+ "t1": {PullRequest{locator1, []string{"f1", "f2"}}},
+ "t2": {PullRequest{locator1, []string{"f1", "f2"}}},
})
locator2 := Locator{Digest: blockdigest.MakeTestBlockDigest(0xCabbed)}
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{To: []string{"t1"}, From: []string{"f1", "f2"}},
- locator2: PullServers{To: []string{"t2"}, From: []string{"f3", "f4"}}}),
+ locator1: {To: []string{"t1"}, From: []string{"f1", "f2"}},
+ locator2: {To: []string{"t2"}, From: []string{"f3", "f4"}}}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{PullRequest{locator1, []string{"f1", "f2"}}},
- "t2": PullList{PullRequest{locator2, []string{"f3", "f4"}}},
+ "t1": {PullRequest{locator1, []string{"f1", "f2"}}},
+ "t2": {PullRequest{locator2, []string{"f3", "f4"}}},
})
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{
+ locator1: {
To: []string{"t1"},
From: []string{"f1", "f2"}},
- locator2: PullServers{
+ locator2: {
To: []string{"t2", "t1"},
From: []string{"f3", "f4"}},
}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{
+ "t1": {
PullRequest{locator1, []string{"f1", "f2"}},
PullRequest{locator2, []string{"f3", "f4"}},
},
- "t2": PullList{
+ "t2": {
PullRequest{locator2, []string{"f3", "f4"}},
},
})
@@ -235,37 +235,37 @@ func (s *PullSuite) TestBuildPullLists(c *C) {
locator4 := Locator{Digest: blockdigest.MakeTestBlockDigest(0xFedBeef)}
c.Check(
BuildPullLists(map[Locator]PullServers{
- locator1: PullServers{
+ locator1: {
To: []string{"t1"},
From: []string{"f1", "f2"}},
- locator2: PullServers{
+ locator2: {
To: []string{"t2", "t1"},
From: []string{"f3", "f4"}},
- locator3: PullServers{
+ locator3: {
To: []string{"t3", "t2", "t1"},
From: []string{"f4", "f5"}},
- locator4: PullServers{
+ locator4: {
To: []string{"t4", "t3", "t2", "t1"},
From: []string{"f1", "f5"}},
}),
PullListMapEquals,
map[string]PullList{
- "t1": PullList{
+ "t1": {
PullRequest{locator1, []string{"f1", "f2"}},
PullRequest{locator2, []string{"f3", "f4"}},
PullRequest{locator3, []string{"f4", "f5"}},
PullRequest{locator4, []string{"f1", "f5"}},
},
- "t2": PullList{
+ "t2": {
PullRequest{locator2, []string{"f3", "f4"}},
PullRequest{locator3, []string{"f4", "f5"}},
PullRequest{locator4, []string{"f1", "f5"}},
},
- "t3": PullList{
+ "t3": {
PullRequest{locator3, []string{"f4", "f5"}},
PullRequest{locator4, []string{"f1", "f5"}},
},
- "t4": PullList{
+ "t4": {
PullRequest{locator4, []string{"f1", "f5"}},
},
})
diff --git a/services/datamanager/summary/summary_test.go b/services/datamanager/summary/summary_test.go
index cc4eb92..8268404 100644
--- a/services/datamanager/summary/summary_test.go
+++ b/services/datamanager/summary/summary_test.go
@@ -85,21 +85,21 @@ func VerifyToCollectionIndexSet(
}
func TestToCollectionIndexSet(t *testing.T) {
- VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: []int{0}}, []int{0})
- VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1}}, []int{1})
- VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: []int{1, 9}}, []int{1, 9})
+ VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{6: {0}}, []int{0})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1}}, []int{1})
+ VerifyToCollectionIndexSet(t, []int{4}, map[int][]int{4: {1, 9}}, []int{1, 9})
VerifyToCollectionIndexSet(t, []int{5, 6},
- map[int][]int{5: []int{2, 3}, 6: []int{3, 4}},
+ map[int][]int{5: {2, 3}, 6: {3, 4}},
[]int{2, 3, 4})
VerifyToCollectionIndexSet(t, []int{5, 6},
- map[int][]int{5: []int{8}, 6: []int{4}},
+ map[int][]int{5: {8}, 6: {4}},
[]int{4, 8})
- VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: []int{0}}, []int{})
+ VerifyToCollectionIndexSet(t, []int{6}, map[int][]int{5: {0}}, []int{})
}
func TestSimpleSummary(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
@@ -128,7 +128,7 @@ func TestSimpleSummary(t *testing.T) {
func TestMissingBlock(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
@@ -159,7 +159,7 @@ func TestMissingBlock(t *testing.T) {
func TestUnderAndOverReplicatedBlocks(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{1, 2}},
+ {ReplicationLevel: 2, Blocks: []int{1, 2}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
@@ -190,9 +190,9 @@ func TestUnderAndOverReplicatedBlocks(t *testing.T) {
func TestMixedReplication(t *testing.T) {
rc := collection.MakeTestReadCollections([]collection.TestCollectionSpec{
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{1, 2}},
- collection.TestCollectionSpec{ReplicationLevel: 1, Blocks: []int{3, 4}},
- collection.TestCollectionSpec{ReplicationLevel: 2, Blocks: []int{5, 6}},
+ {ReplicationLevel: 1, Blocks: []int{1, 2}},
+ {ReplicationLevel: 1, Blocks: []int{3, 4}},
+ {ReplicationLevel: 2, Blocks: []int{5, 6}},
})
rc.Summarize(nil)
cIndex := rc.CollectionIndicesForTesting()
diff --git a/services/datamanager/summary/trash_list_test.go b/services/datamanager/summary/trash_list_test.go
index 555211f..3626904 100644
--- a/services/datamanager/summary/trash_list_test.go
+++ b/services/datamanager/summary/trash_list_test.go
@@ -26,12 +26,12 @@ func (s *TrashSuite) TestBuildTrashLists(c *C) {
var keepServerInfo = keep.ReadServers{
KeepServerIndexToAddress: []keep.ServerAddress{sv0, sv1},
BlockToServers: map[blockdigest.DigestWithSize][]keep.BlockServerInfo{
- block0: []keep.BlockServerInfo{
- keep.BlockServerInfo{0, 99},
- keep.BlockServerInfo{1, 101}},
- block1: []keep.BlockServerInfo{
- keep.BlockServerInfo{0, 99},
- keep.BlockServerInfo{1, 101}}}}
+ block0: {
+ {0, 99},
+ {1, 101}},
+ block1: {
+ {0, 99},
+ {1, 101}}}}
// only block0 is in delete set
var bs = make(BlockSet)
@@ -40,37 +40,37 @@ func (s *TrashSuite) TestBuildTrashLists(c *C) {
// Test trash list where only sv0 is on writable list.
c.Check(buildTrashListsInternal(
map[string]struct{}{
- sv0.URL(): struct{}{}},
+ sv0.URL(): {}},
&keepServerInfo,
110,
bs),
DeepEquals,
map[string]keep.TrashList{
- "http://keep0.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
+ "http://keep0.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
// Test trash list where both sv0 and sv1 are on writable list.
c.Check(buildTrashListsInternal(
map[string]struct{}{
- sv0.URL(): struct{}{},
- sv1.URL(): struct{}{}},
+ sv0.URL(): {},
+ sv1.URL(): {}},
&keepServerInfo,
110,
bs),
DeepEquals,
map[string]keep.TrashList{
- "http://keep0.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 99}},
- "http://keep1.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 101}}})
+ "http://keep0.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 99}},
+ "http://keep1.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 101}}})
// Test trash list where only block on sv0 is expired
c.Check(buildTrashListsInternal(
map[string]struct{}{
- sv0.URL(): struct{}{},
- sv1.URL(): struct{}{}},
+ sv0.URL(): {},
+ sv1.URL(): {}},
&keepServerInfo,
100,
bs),
DeepEquals,
map[string]keep.TrashList{
- "http://keep0.example.com:80": keep.TrashList{keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
+ "http://keep0.example.com:80": {keep.TrashRequest{"000000000000000000000000deadbeef", 99}}})
}
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index fa585d3..d22074e 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -619,7 +619,7 @@ func (bal *Balancer) commitAsync(c *arvados.Client, label string, f func(srv *Ke
}(srv)
}
var lastErr error
- for _ = range bal.KeepServices {
+ for range bal.KeepServices {
if err := <-errs; err != nil {
bal.logf("%v", err)
lastErr = err
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index b1b198d..98e1203 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -86,7 +86,7 @@ func (s *s3VolumeAdder) Set(bucketName string) error {
}
func s3regions() (okList []string) {
- for r, _ := range aws.Regions {
+ for r := range aws.Regions {
okList = append(okList, r)
}
return
diff --git a/services/keepstore/work_queue_test.go b/services/keepstore/work_queue_test.go
index 74c67f2..6b31795 100644
--- a/services/keepstore/work_queue_test.go
+++ b/services/keepstore/work_queue_test.go
@@ -98,7 +98,7 @@ func TestWorkQueueDoneness(t *testing.T) {
gate := make(chan struct{})
go func() {
<-gate
- for _ = range b.NextItem {
+ for range b.NextItem {
<-gate
time.Sleep(time.Millisecond)
b.DoneItem <- struct{}{}
commit 15c918a67ceda31bde38cab75736d89bd015476a
Author: Tom Clegg <tom at curoverse.com>
Date: Thu Jun 23 16:19:30 2016 -0400
9437: Report timestamps as nanoseconds since epoch in keepstore index.
diff --git a/sdk/go/arvados/keep_service.go b/sdk/go/arvados/keep_service.go
index 4af1b79..87dbd2a 100644
--- a/sdk/go/arvados/keep_service.go
+++ b/sdk/go/arvados/keep_service.go
@@ -30,6 +30,7 @@ type KeepServiceList struct {
// us about a stored block.
type KeepServiceIndexEntry struct {
SizedDigest
+ // Time of last write, in nanoseconds since Unix epoch
Mtime int64
}
diff --git a/services/datamanager/summary/trash_list.go b/services/datamanager/summary/trash_list.go
index b6ceace..3e4d387 100644
--- a/services/datamanager/summary/trash_list.go
+++ b/services/datamanager/summary/trash_list.go
@@ -29,7 +29,7 @@ func BuildTrashLists(kc *keepclient.KeepClient,
ttl := int64(_ttl.(float64))
// expire unreferenced blocks more than "ttl" seconds old.
- expiry := time.Now().UTC().Unix() - ttl
+ expiry := time.Now().UTC().UnixNano() - ttl*1e9
return buildTrashListsInternal(writableServers, keepServerInfo, expiry, keepBlocksNotInCollections), nil
}
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index 2d1a59e..fa585d3 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -199,7 +199,7 @@ func (bal *Balancer) GetCurrentState(c *arvados.Client, pageSize, bufs int) erro
return err
}
bal.DefaultReplication = dd.DefaultCollectionReplication
- bal.MinMtime = time.Now().Unix() - dd.BlobSignatureTTL
+ bal.MinMtime = time.Now().UnixNano() - dd.BlobSignatureTTL*1e9
errs := make(chan error, 2+len(bal.KeepServices))
wg := sync.WaitGroup{}
diff --git a/services/keep-balance/balance_test.go b/services/keep-balance/balance_test.go
index 682a5fb..b93939c 100644
--- a/services/keep-balance/balance_test.go
+++ b/services/keep-balance/balance_test.go
@@ -76,7 +76,7 @@ func (bal *balancerSuite) SetUpTest(c *check.C) {
bal.KeepServices[srv.UUID] = srv
}
- bal.MinMtime = time.Now().Unix() - bal.signatureTTL
+ bal.MinMtime = time.Now().UnixNano() - bal.signatureTTL*1e9
}
func (bal *balancerSuite) TestPerfect(c *check.C) {
@@ -240,7 +240,7 @@ func (bal *balancerSuite) srvList(knownBlockID int, order slots) (srvs []*KeepSe
// replList is like srvList but returns an "existing replicas" slice,
// suitable for a BlockState test fixture.
func (bal *balancerSuite) replList(knownBlockID int, order slots) (repls []Replica) {
- mtime := time.Now().Unix() - bal.signatureTTL - 86400
+ mtime := time.Now().UnixNano() - (bal.signatureTTL+86400)*1e9
for _, srv := range bal.srvList(knownBlockID, order) {
repls = append(repls, Replica{srv, mtime})
mtime++
diff --git a/services/keepstore/azure_blob_volume.go b/services/keepstore/azure_blob_volume.go
index 99da2a3..48cb026 100644
--- a/services/keepstore/azure_blob_volume.go
+++ b/services/keepstore/azure_blob_volume.go
@@ -350,7 +350,7 @@ func (v *AzureBlobVolume) IndexTo(prefix string, writer io.Writer) error {
// Trashed blob; exclude it from response
continue
}
- fmt.Fprintf(writer, "%s+%d %d\n", b.Name, b.Properties.ContentLength, t.Unix())
+ fmt.Fprintf(writer, "%s+%d %d\n", b.Name, b.Properties.ContentLength, t.UnixNano())
}
if resp.NextMarker == "" {
return nil
diff --git a/services/keepstore/s3_volume.go b/services/keepstore/s3_volume.go
index 80a7c89..b1b198d 100644
--- a/services/keepstore/s3_volume.go
+++ b/services/keepstore/s3_volume.go
@@ -249,7 +249,7 @@ func (v *S3Volume) IndexTo(prefix string, writer io.Writer) error {
if !v.isKeepBlock(key.Key) {
continue
}
- fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.Unix())
+ fmt.Fprintf(writer, "%s+%d %d\n", key.Key, key.Size, t.UnixNano())
}
if !listResp.IsTruncated {
break
diff --git a/services/keepstore/trash_worker.go b/services/keepstore/trash_worker.go
index 62f63d5..d11bc05 100644
--- a/services/keepstore/trash_worker.go
+++ b/services/keepstore/trash_worker.go
@@ -22,7 +22,7 @@ func RunTrashWorker(trashq *WorkQueue) {
// TrashItem deletes the indicated block from every writable volume.
func TrashItem(trashRequest TrashRequest) {
- reqMtime := time.Unix(trashRequest.BlockMtime, 0)
+ reqMtime := time.Unix(0, trashRequest.BlockMtime)
if time.Since(reqMtime) < blobSignatureTTL {
log.Printf("WARNING: data manager asked to delete a %v old block %v (BlockMtime %d = %v), but my blobSignatureTTL is %v! Skipping.",
time.Since(reqMtime),
@@ -39,8 +39,8 @@ func TrashItem(trashRequest TrashRequest) {
log.Printf("%v Delete(%v): %v", volume, trashRequest.Locator, err)
continue
}
- if trashRequest.BlockMtime != mtime.Unix() {
- log.Printf("%v Delete(%v): mtime on volume is %v does not match trash list value %v", volume, trashRequest.Locator, mtime.Unix(), trashRequest.BlockMtime)
+ if trashRequest.BlockMtime != mtime.UnixNano() {
+ log.Printf("%v Delete(%v): stored mtime %v does not match trash list value %v", volume, trashRequest.Locator, mtime.UnixNano(), trashRequest.BlockMtime)
continue
}
diff --git a/services/keepstore/trash_worker_test.go b/services/keepstore/trash_worker_test.go
index d111cae..94798d9 100644
--- a/services/keepstore/trash_worker_test.go
+++ b/services/keepstore/trash_worker_test.go
@@ -236,7 +236,7 @@ func performTrashWorkerTest(testData TrashWorkerTestData, t *testing.T) {
// Create TrashRequest for the test
trashRequest := TrashRequest{
Locator: testData.DeleteLocator,
- BlockMtime: oldBlockTime.Unix(),
+ BlockMtime: oldBlockTime.UnixNano(),
}
// Run trash worker and put the trashRequest on trashq
diff --git a/services/keepstore/volume_generic_test.go b/services/keepstore/volume_generic_test.go
index f8fe0d0..4291c6c 100644
--- a/services/keepstore/volume_generic_test.go
+++ b/services/keepstore/volume_generic_test.go
@@ -7,6 +7,7 @@ import (
"os"
"regexp"
"sort"
+ "strconv"
"strings"
"time"
@@ -355,10 +356,22 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
v := factory(t)
defer v.Teardown()
+ // minMtime and maxMtime are the minimum and maximum
+ // acceptable values the index can report for our test
+ // blocks. 1-second precision is acceptable.
+ minMtime := time.Now().UTC().UnixNano()
+ minMtime -= minMtime % 1e9
+
v.PutRaw(TestHash, TestBlock)
v.PutRaw(TestHash2, TestBlock2)
v.PutRaw(TestHash3, TestBlock3)
+ maxMtime := time.Now().UTC().UnixNano()
+ if maxMtime%1e9 > 0 {
+ maxMtime -= maxMtime % 1e9
+ maxMtime += 1e9
+ }
+
// Blocks whose names aren't Keep hashes should be omitted from
// index
v.PutRaw("fffffffffnotreallyahashfffffffff", nil)
@@ -371,15 +384,21 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
indexRows := strings.Split(string(buf.Bytes()), "\n")
sort.Strings(indexRows)
sortedIndex := strings.Join(indexRows, "\n")
- m, err := regexp.MatchString(
- `^\n`+TestHash+`\+\d+ \d+\n`+
- TestHash3+`\+\d+ \d+\n`+
- TestHash2+`\+\d+ \d+$`,
- sortedIndex)
- if err != nil {
- t.Error(err)
- } else if !m {
+ m := regexp.MustCompile(
+ `^\n` + TestHash + `\+\d+ (\d+)\n` +
+ TestHash3 + `\+\d+ \d+\n` +
+ TestHash2 + `\+\d+ \d+$`,
+ ).FindStringSubmatch(sortedIndex)
+ if m == nil {
t.Errorf("Got index %q for empty prefix", sortedIndex)
+ } else {
+ mtime, err := strconv.ParseInt(m[1], 10, 64)
+ if err != nil {
+ t.Error(err)
+ } else if mtime < minMtime || mtime > maxMtime {
+ t.Errorf("got %d for TestHash timestamp, expected %d <= t <= %d",
+ mtime, minMtime, maxMtime)
+ }
}
for _, prefix := range []string{"f", "f15", "f15ac"} {
@@ -396,7 +415,7 @@ func testIndexTo(t TB, factory TestableVolumeFactory) {
for _, prefix := range []string{"zero", "zip", "zilch"} {
buf = new(bytes.Buffer)
- v.IndexTo(prefix, buf)
+ err := v.IndexTo(prefix, buf)
if err != nil {
t.Errorf("Got error on IndexTo with no such prefix %v", err.Error())
} else if buf.Len() != 0 {
diff --git a/services/keepstore/volume_unix.go b/services/keepstore/volume_unix.go
index 7aff85e..90189dc 100644
--- a/services/keepstore/volume_unix.go
+++ b/services/keepstore/volume_unix.go
@@ -138,9 +138,8 @@ func (v *UnixVolume) Touch(loc string) error {
return e
}
defer unlockfile(f)
- now := time.Now().Unix()
- utime := syscall.Utimbuf{now, now}
- return syscall.Utime(p, &utime)
+ ts := syscall.NsecToTimespec(time.Now().UnixNano())
+ return syscall.UtimesNano(p, []syscall.Timespec{ts, ts})
}
// Mtime returns the stored timestamp for the given locator.
@@ -353,7 +352,7 @@ func (v *UnixVolume) IndexTo(prefix string, w io.Writer) error {
_, err = fmt.Fprint(w,
name,
"+", fileInfo[0].Size(),
- " ", fileInfo[0].ModTime().Unix(),
+ " ", fileInfo[0].ModTime().UnixNano(),
"\n")
}
blockdir.Close()
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list