[ARVADOS] created: 1.3.3-4-g9624cb485
Git user
git at public.arvados.org
Thu Jan 6 17:44:51 UTC 2022
at 9624cb485531064e94159af31b0a2c1d197cbb31 (commit)
commit 9624cb485531064e94159af31b0a2c1d197cbb31
Author: Ward Vandewege <ward at curii.com>
Date: Mon Aug 10 13:55:27 2020 -0400
documentation: the arvados k8s repo has changed name.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/install/arvados-on-kubernetes.html.textile.liquid b/doc/install/arvados-on-kubernetes.html.textile.liquid
index 01999f0c2..4e7b8b73f 100644
--- a/doc/install/arvados-on-kubernetes.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes.html.textile.liquid
@@ -55,11 +55,11 @@ There should be no errors. The command will return nothing.
h2(#git). Clone the repository
-Clone the repository and nagivate to the @arvados-kubernetes/charts/arvados@ directory:
+Clone the repository and nagivate to the @arvados-k8s/charts/arvados@ directory:
<pre>
-$ git clone https://github.com/curoverse/arvados-kubernetes.git
-$ cd arvados-kubernetes/charts/arvados
+$ git clone https://github.com/arvados/arvados-k8s.git
+$ cd arvados-k8s/charts/arvados
</pre>
h2(#Start). Start the Arvados cluster
commit b4bda0c138d28e9e2268cc900dedac06b8445aee
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Wed Jun 19 15:02:10 2019 -0400
Keep relative URL when switching between versions in our documentation.
refs #15138
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/doc/_includes/_navbar_top.liquid b/doc/_includes/_navbar_top.liquid
index e227703a5..39c70471a 100644
--- a/doc/_includes/_navbar_top.liquid
+++ b/doc/_includes/_navbar_top.liquid
@@ -28,7 +28,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
{% if site.all_versions != [] %}
<div class="dropdown-menu" aria-labelledby="versionMenuLink">
{% for version in site.all_versions %}
- <a href="/{{ version }}/" class="dropdown-item">{{ version }}</a><br/>
+ <a href="/{{ version }}{{ page.url }}" class="dropdown-item">{{ version }}</a><br/>
{% endfor %}
</div>
{% endif %}
commit 08e361c63d9254e59f688b68a24282af16005c89
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Tue Jun 11 13:42:40 2019 -0400
15138: Include logic to support building multiple versions of our documentation.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/doc/_config.yml b/doc/_config.yml
index 94c953996..bb1535b0f 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -11,6 +11,8 @@
# $ rake generate baseurl=/example arvados_api_host=example.com
baseurl:
+current_version:
+all_versions:
arvados_api_host: localhost
arvados_workbench_host: http://localhost
diff --git a/doc/_includes/_navbar_top.liquid b/doc/_includes/_navbar_top.liquid
index 477a0dd81..e227703a5 100644
--- a/doc/_includes/_navbar_top.liquid
+++ b/doc/_includes/_navbar_top.liquid
@@ -24,6 +24,15 @@ SPDX-License-Identifier: CC-BY-SA-3.0
<li {% if page.navsection == 'api' %} class="active" {% endif %}><a href="{{ site.baseurl }}/api/index.html">API</a></li>
<li {% if page.navsection == 'admin' %} class="active" {% endif %}><a href="{{ site.baseurl }}/admin/index.html">Admin</a></li>
<li {% if page.navsection == 'installguide' %} class="active" {% endif %}><a href="{{ site.baseurl }}/install/index.html">Install</a></li>
+ <li><a href="#" class="dropdown-toggle" role="button" id="versionMenuLink" data-toggle="dropdown" aria-haspopup="true" aria-expanded="false">{{ site.current_version }}{% if site.all_versions != [] %} ▾{% endif %}</a>
+ {% if site.all_versions != [] %}
+ <div class="dropdown-menu" aria-labelledby="versionMenuLink">
+ {% for version in site.all_versions %}
+ <a href="/{{ version }}/" class="dropdown-item">{{ version }}</a><br/>
+ {% endfor %}
+ </div>
+ {% endif %}
+ </li>
<li><a href="https://arvados.org" style="padding-left: 2em">arvados.org »</a></li>
</ul>
commit 8cf8dc7df529f83aa98aaefc238e1a104200fea5
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Mon Jun 10 10:41:22 2019 -0400
15138: Show the TM notice a little bit higher.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/doc/_includes/_navbar_top.liquid b/doc/_includes/_navbar_top.liquid
index 7d96ea011..477a0dd81 100644
--- a/doc/_includes/_navbar_top.liquid
+++ b/doc/_includes/_navbar_top.liquid
@@ -13,7 +13,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
<span class="icon-bar"></span>
<span class="icon-bar"></span>
</button>
- <a class="navbar-brand" href="{{ site.baseurl }}/">Arvados™ Docs</a>
+ <a class="navbar-brand" href="{{ site.baseurl }}/">Arvados<sup>™</sup> Docs</a>
</div>
<div class="collapse navbar-collapse" id="bs-navbar-collapse">
<ul class="nav navbar-nav">
diff --git a/doc/user/index.html.textile.liquid b/doc/user/index.html.textile.liquid
index 202e297a2..909394ef4 100644
--- a/doc/user/index.html.textile.liquid
+++ b/doc/user/index.html.textile.liquid
@@ -1,7 +1,7 @@
---
layout: default
navsection: userguide
-title: Welcome to Arvados™!
+title: Welcome to Arvados<sup>™</sup>!
...
{% comment %}
Copyright (C) The Arvados Authors. All rights reserved.
commit 926380be966b968aa7223cade0476c2a70acfbc4
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Wed May 1 16:03:17 2019 -0400
Add missing key file.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/docker/jobs/1078ECD7.key b/docker/jobs/1078ECD7.key
new file mode 100644
index 000000000..edc62f48f
--- /dev/null
+++ b/docker/jobs/1078ECD7.key
@@ -0,0 +1,30 @@
+-----BEGIN PGP PUBLIC KEY BLOCK-----
+
+mQENBEzhgeoBCAChhoK1dqpWzNyDWqRGEvdFdkJaA9D2HRwKPfBfjAoePX6ZyrpA
+ItlUsvt/8s/DRiTiPEFQR4S7VqocmU6whJc3gDEGyOM6b1NF873lIfSVwUoE42QE
+a76dO8woOYgLUyxu2mKG+bJgGMumjBJt6ZOndYVjTYB/7sEeVxwmMVulfZe0s6zg
+ut0+SoTYg2R36qIqeIcWllYt97sEYnyy1qXMis4/3IZnuWkS/frsPR3aeUI4W+o2
+NDN1kj49+LMe7Fb5b7jZY08rZbAWXi1rU1hQx4jC9RvYqlT4HNld4Bn7os1IvOOA
+wNiR0oiVdiuDbBxcMvRPktxMrFVjowusRLq/ABEBAAG0PUN1cm92ZXJzZSwgSW5j
+IEF1dG9tYXRpYyBTaWduaW5nIEtleSA8c3lzYWRtaW5AY3Vyb3ZlcnNlLmNvbT6J
+ATgEEwECACIFAlNgYIECGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEFcW
+WREQeOzXPkEH/jQJDIYI1dxWcYiA+hczmpaZvN2/pc/kwIW/6a03+6zqmSNkebOE
+TgoDILacSYc17hy20R1/rWyUstOMKcEgFDBlSehhHyl0f7q/w7d8Ais6MabzsPfx
+IceJpsjUg87+BR7qWhgQ0sxmtIF2TKuTFLs+nkGsgSsiBOEF4NvHxuj3HD4y8F27
+HNqrkqwjLS8xJwwH5Gp2uMEVr1AXIH3iSRjJ8X124s8iEP97Q/3IazoYRf9/MCSm
+QEx8KzxwDX6t4bW6O4D01K+e9gdkTY70dcMgJoqm5IsX7yxjEubiOunphtlJnZ9d
+Oi1yBN5UM3pWKAdcfRj4rcfV9Simvpx9av+5AQ0ETOGB6gEIAMAA0HVMG0BbdnU7
+wWgl5eFdT0AUSrXK/WdcKqVEGGv+c68NETSHWZOJX7O46Eao4gY4cTYprVMBzxpY
+/BtQSYLpE0HLvBc1fcFd61Yz4H/9rGSNY0GcIQEbOjbJY5mr8qFsQ1K/mAf3aUL3
+b6ni4sHVicRiRr0Gl4Ihorlskpfu1SHs/C5tvTSVNF9p4vtl5892y1yILQeVpcBs
+NCR7MUpdS49xCpvnAWsDZX+ij6LTR3lzCm/ZLCg4gNuZkjgU9oqVfGkqysW7WZ8S
+OLvzAwUw7i1EIFX8q6QdudGoezxz8m8OgZM1v8AFpYEKlhEPf1W0MSfaRDwrj866
+8nCLruEAEQEAAYkBHwQYAQIACQUCTOGB6gIbDAAKCRBXFlkREHjs199EB/4+p0G1
+3PHxt6rLWSCGXobDOu4ZOA/qnv0D/JhOLroFds5TzQv6vnS8eAkhCTjHVA+b58cm
+kXpI0oYcD4ZP+KK1CHKq2rGfwou7HfAF+icnNqYkeBOkjjbCgkvBlcCInuAuU8JX
+DZMkfFk52+eBKwTjS/J/fQp0vDru8bHLp98WgdRHWfJQ3mc3gz4A5sR6zhrGPW6/
+ssnROS4dC2Ohp35GpgN1KjD3EmEw5RoSBYlyrARCaMsivgIKMxGUEyFZWhuJt3N1
+2MTddRwz28hbmYCi+MzHYDbRv+cSyUDmvXaWhfkNKBepClBA1rTWBcldit5vvlqr
+yPet6wIKrtLGhAqZ
+=CLkG
+-----END PGP PUBLIC KEY BLOCK-----
commit fbbe70d89471efca8813b31739ebefbe44bd0c28
Author: Ward Vandewege <wvandewege at veritasgenetics.com>
Date: Wed May 1 15:53:15 2019 -0400
Add argument to the arvados/jobs build script to choose which package repo to
build from.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <wvandewege at veritasgenetics.com>
diff --git a/.licenseignore b/.licenseignore
index 83c81b2fc..b62364bb1 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -15,7 +15,8 @@ build/package-test-dockerfiles/ubuntu1604/etc-apt-preferences.d-arvados
doc/fonts/*
doc/user/cwl/federated/*
*/docker_image
-docker/jobs/apt.arvados.org.list
+docker/jobs/apt.arvados.org*.list
+docker/jobs/1078ECD7.key
*/en.bootstrap.yml
*font-awesome.css
*.gif
diff --git a/build/run-build-docker-jobs-image.sh b/build/run-build-docker-jobs-image.sh
index 83bb5ae71..5f774ce76 100755
--- a/build/run-build-docker-jobs-image.sh
+++ b/build/run-build-docker-jobs-image.sh
@@ -5,21 +5,24 @@
function usage {
echo >&2
- echo >&2 "usage: $0 [options]"
+ echo >&2 "usage: WORKSPACE=/path/to/arvados $0 [options]"
echo >&2
echo >&2 "$0 options:"
echo >&2 " -t, --tags version tag for docker"
+ echo >&2 " -r, --repo Arvados package repo to use: dev, testing, stable (default: dev)"
echo >&2 " -u, --upload Upload the images (docker push)"
echo >&2 " --no-cache Don't use build cache"
echo >&2 " -h, --help Display this help and exit"
echo >&2
- echo >&2 " If no options are given, just builds the images."
+ echo >&2 " WORKSPACE=path Path to the Arvados source tree to build from"
+ echo >&2
}
upload=false
+REPO=dev
# NOTE: This requires GNU getopt (part of the util-linux package on Debian-based distros).
-TEMP=`getopt -o hut: \
- --long help,upload,no-cache,tags: \
+TEMP=`getopt -o hut:r: \
+ --long help,upload,no-cache,tags:,repo: \
-n "$0" -- "$@"`
if [ $? != 0 ] ; then echo "Use -h for help"; exit 1 ; fi
@@ -50,6 +53,19 @@ do
;;
esac
;;
+ -r | --repo)
+ case "$2" in
+ "")
+ echo "ERROR: --repo needs a parameter";
+ usage;
+ exit 1
+ ;;
+ *)
+ REPO="$2";
+ shift 2
+ ;;
+ esac
+ ;;
--)
shift
break
@@ -69,6 +85,16 @@ exit_cleanly() {
exit $EXITCODE
}
+# Sanity check
+if ! [[ -n "$WORKSPACE" ]]; then
+ usage;
+ echo >&2 "Error: WORKSPACE environment variable not set"
+ echo >&2
+ exit 1
+fi
+
+echo $WORKSPACE
+
COLUMNS=80
. $WORKSPACE/build/run-library.sh
@@ -88,16 +114,6 @@ docker_push () {
checkexit $ECODE "docker push $*"
}
-# Sanity check
-if ! [[ -n "$WORKSPACE" ]]; then
- echo >&2
- echo >&2 "Error: WORKSPACE environment variable not set"
- echo >&2
- exit 1
-fi
-
-echo $WORKSPACE
-
# find the docker binary
DOCKER=`which docker.io`
@@ -153,6 +169,7 @@ cd docker/jobs
docker build $NOCACHE \
--build-arg python_sdk_version=${python_sdk_version} \
--build-arg cwl_runner_version=${cwl_runner_version} \
+ --build-arg repo_version=${REPO} \
-t arvados/jobs:$cwl_runner_version_orig .
ECODE=$?
@@ -175,6 +192,9 @@ if docker --version |grep " 1\.[0-9]\." ; then
# -f flag removed in Docker 1.12
FORCE=-f
fi
+
+#docker export arvados/jobs:$cwl_runner_version_orig | docker import - arvados/jobs:$cwl_runner_version_orig
+
if ! [[ -z "$version_tag" ]]; then
docker tag $FORCE arvados/jobs:$cwl_runner_version_orig arvados/jobs:"$version_tag"
else
diff --git a/docker/jobs/Dockerfile b/docker/jobs/Dockerfile
index 12c68564b..c7bcf988d 100644
--- a/docker/jobs/Dockerfile
+++ b/docker/jobs/Dockerfile
@@ -4,13 +4,19 @@
# Based on Debian Jessie
FROM debian:jessie
-MAINTAINER Ward Vandewege <ward at curoverse.com>
+MAINTAINER Ward Vandewege <wvandewege at veritasgenetics.com>
ENV DEBIAN_FRONTEND noninteractive
-ADD apt.arvados.org.list /etc/apt/sources.list.d/
-RUN apt-key adv --keyserver pool.sks-keyservers.net --recv 1078ECD7
-RUN gpg --keyserver pool.sks-keyservers.net --recv-keys D39DC0E3
+RUN apt-get update -q
+RUN apt-get install -yq --no-install-recommends gnupg
+
+ARG repo_version
+RUN echo repo_version $repo_version
+ADD apt.arvados.org-$repo_version.list /etc/apt/sources.list.d/
+
+ADD 1078ECD7.key /tmp/
+RUN cat /tmp/1078ECD7.key | apt-key add -
ARG python_sdk_version
ARG cwl_runner_version
diff --git a/docker/jobs/apt.arvados.org.list b/docker/jobs/apt.arvados.org-dev.list
similarity index 100%
rename from docker/jobs/apt.arvados.org.list
rename to docker/jobs/apt.arvados.org-dev.list
diff --git a/docker/jobs/apt.arvados.org-stable.list b/docker/jobs/apt.arvados.org-stable.list
new file mode 100644
index 000000000..3ae6df421
--- /dev/null
+++ b/docker/jobs/apt.arvados.org-stable.list
@@ -0,0 +1,2 @@
+# apt.arvados.org
+deb http://apt.arvados.org/ jessie main
diff --git a/docker/jobs/apt.arvados.org-testing.list b/docker/jobs/apt.arvados.org-testing.list
new file mode 100644
index 000000000..27b3e0bc1
--- /dev/null
+++ b/docker/jobs/apt.arvados.org-testing.list
@@ -0,0 +1,2 @@
+# apt.arvados.org
+deb http://apt.arvados.org/ jessie-testing main
commit 675237bec9e6ecf768b63504b20e58162ff5352d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Fri Apr 26 16:06:39 2019 -0400
15148: Include list of affected PDHs in LostBlocksFile.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index adf9b9b3c..08ed7d2a7 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -453,7 +453,13 @@ func (bal *Balancer) addCollection(coll arvados.Collection) error {
repl = *coll.ReplicationDesired
}
debugf("%v: %d block x%d", coll.UUID, len(blkids), repl)
- bal.BlockStateMap.IncreaseDesired(coll.StorageClassesDesired, repl, blkids)
+ // Pass pdh to IncreaseDesired only if LostBlocksFile is being
+ // written -- otherwise it's just a waste of memory.
+ pdh := ""
+ if bal.LostBlocksFile != "" {
+ pdh = coll.PortableDataHash
+ }
+ bal.BlockStateMap.IncreaseDesired(pdh, coll.StorageClassesDesired, repl, blkids)
return nil
}
@@ -927,7 +933,11 @@ func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
s.lost.replicas -= surplus
s.lost.blocks++
s.lost.bytes += bytes * int64(-surplus)
- fmt.Fprintf(bal.lostBlocks, "%s\n", strings.SplitN(string(result.blkid), "+", 2)[0])
+ fmt.Fprintf(bal.lostBlocks, "%s", strings.SplitN(string(result.blkid), "+", 2)[0])
+ for pdh := range result.blk.Refs {
+ fmt.Fprintf(bal.lostBlocks, " %s", pdh)
+ }
+ fmt.Fprint(bal.lostBlocks, "\n")
case surplus < 0:
s.underrep.replicas -= surplus
s.underrep.blocks++
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index de368239b..8bfbd952b 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -452,7 +452,7 @@ func (s *runSuite) TestWriteLostBlocks(c *check.C) {
c.Check(err, check.IsNil)
lost, err := ioutil.ReadFile(lostf.Name())
c.Assert(err, check.IsNil)
- c.Check(string(lost), check.Equals, "37b51d194a7513e45b56f6524f2d51f2\n")
+ c.Check(string(lost), check.Equals, "37b51d194a7513e45b56f6524f2d51f2 fa7aeb5140e2848d39b416daeef4ffc5+45\n")
}
func (s *runSuite) TestDryRun(c *check.C) {
diff --git a/services/keep-balance/block_state.go b/services/keep-balance/block_state.go
index 46e69059c..d9338d0f9 100644
--- a/services/keep-balance/block_state.go
+++ b/services/keep-balance/block_state.go
@@ -23,6 +23,7 @@ type Replica struct {
// replicas actually stored (according to the keepstore indexes we
// know about).
type BlockState struct {
+ Refs map[string]bool // pdh => true (only tracked when len(Replicas)==0)
RefCount int
Replicas []Replica
Desired map[string]int
@@ -40,9 +41,21 @@ var defaultClasses = []string{"default"}
func (bs *BlockState) addReplica(r Replica) {
bs.Replicas = append(bs.Replicas, r)
+ // Free up memory wasted by tracking PDHs that will never be
+ // reported (see comment in increaseDesired)
+ bs.Refs = nil
}
-func (bs *BlockState) increaseDesired(classes []string, n int) {
+func (bs *BlockState) increaseDesired(pdh string, classes []string, n int) {
+ if pdh != "" && len(bs.Replicas) == 0 {
+ // Note we only track PDHs if there's a possibility
+ // that we will report the list of referring PDHs,
+ // i.e., if we haven't yet seen a replica.
+ if bs.Refs == nil {
+ bs.Refs = map[string]bool{}
+ }
+ bs.Refs[pdh] = true
+ }
bs.RefCount++
if len(classes) == 0 {
classes = defaultClasses
@@ -109,11 +122,14 @@ func (bsm *BlockStateMap) AddReplicas(mnt *KeepMount, idx []arvados.KeepServiceI
// IncreaseDesired updates the map to indicate the desired replication
// for the given blocks in the given storage class is at least n.
-func (bsm *BlockStateMap) IncreaseDesired(classes []string, n int, blocks []arvados.SizedDigest) {
+//
+// If pdh is non-empty, it will be tracked and reported in the "lost
+// blocks" report.
+func (bsm *BlockStateMap) IncreaseDesired(pdh string, classes []string, n int, blocks []arvados.SizedDigest) {
bsm.mutex.Lock()
defer bsm.mutex.Unlock()
for _, blkid := range blocks {
- bsm.get(blkid).increaseDesired(classes, n)
+ bsm.get(blkid).increaseDesired(pdh, classes, n)
}
}
commit 72ef97707af66bb064004ca8f86ca539d2439ddc
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Apr 25 10:04:04 2019 -0400
Change arvados-python-client to >=1.3.1, <=1.3.99
refs #15080
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/sdk/cwl/setup.py b/sdk/cwl/setup.py
index 493ce799c..04b40524a 100644
--- a/sdk/cwl/setup.py
+++ b/sdk/cwl/setup.py
@@ -37,7 +37,7 @@ setup(name='arvados-cwl-runner',
'schema-salad==2.7.20181116024232',
'typing >= 3.6.4',
'ruamel.yaml >=0.15.54, <=0.15.77',
- 'arvados-python-client==1.3.1',
+ 'arvados-python-client >=1.3.1, <=1.3.99',
'setuptools',
'ciso8601 >=1.0.6, <2.0.0',
'subprocess32>=3.5.1',
commit 483066a2a107712e7b01833191afe93002954f55
Author: Fernando Monserrat <fernando.monserrat at gmail.com>
Date: Thu Mar 7 11:52:25 2019 -0300
refs #14891 .Merge branch '14891-upload-to-testing-repo'
Arvados-DCO-1.1-Signed-off-by: Fernando Monserrat <fmonserrat at veritasgenetics.com>
diff --git a/build/run-build-test-packages-one-target.sh b/build/run-build-test-packages-one-target.sh
index 7bdaacac5..b98a4c0a7 100755
--- a/build/run-build-test-packages-one-target.sh
+++ b/build/run-build-test-packages-one-target.sh
@@ -14,6 +14,8 @@ Syntax:
--upload
If the build and test steps are successful, upload the packages
to a remote apt repository (default: false)
+--rc
+ Optional Parameter to build Release Candidate
--build-version <version>
Version to build (default:
\$ARVADOS_BUILDING_VERSION-\$ARVADOS_BUILDING_ITERATION or
@@ -40,7 +42,7 @@ if ! [[ -d "$WORKSPACE" ]]; then
fi
PARSEDOPTS=$(getopt --name "$0" --longoptions \
- help,upload,target:,build-version: \
+ help,upload,rc,target:,build-version: \
-- "" "$@")
if [ $? -ne 0 ]; then
exit 1
@@ -48,6 +50,7 @@ fi
TARGET=debian8
UPLOAD=0
+RC=0
declare -a build_args=()
@@ -65,6 +68,9 @@ while [ $# -gt 0 ]; do
--upload)
UPLOAD=1
;;
+ --rc)
+ RC=1
+ ;;
--build-version)
build_args+=("$1" "$2")
shift
@@ -115,8 +121,13 @@ if [[ "$UPLOAD" != 0 ]]; then
timer_reset
if [ ${#failures[@]} -eq 0 ]; then
- echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt at apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
- /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt at apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ if [[ "$RC" != 0 ]]; then
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages_testing.py -H jenkinsapt at apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
+ /usr/local/arvados-dev/jenkins/run_upload_packages_testing.py -H jenkinsapt at apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ else
+ echo "/usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt at apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET"
+ /usr/local/arvados-dev/jenkins/run_upload_packages.py -H jenkinsapt at apt.arvados.org -o Port=2222 --workspace $WORKSPACE $TARGET
+ fi
else
echo "Skipping package upload, there were errors building and/or testing the packages"
fi
@@ -124,4 +135,4 @@ if [[ "$UPLOAD" != 0 ]]; then
title "End of upload packages (`timer`)"
fi
-exit_cleanly
+exit_cleanly
\ No newline at end of file
commit c2ed4aab77cb1fa25487be495b6971107e69aab4
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Apr 24 16:42:06 2019 -0400
Merge branch '15112-save-lost-blocks-file'
refs #15112
refs #15148
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/doc/install/install-keep-balance.html.textile.liquid b/doc/install/install-keep-balance.html.textile.liquid
index 68bf07a4a..4a35f448e 100644
--- a/doc/install/install-keep-balance.html.textile.liquid
+++ b/doc/install/install-keep-balance.html.textile.liquid
@@ -81,11 +81,11 @@ Client:
AuthToken: zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz
KeepServiceTypes:
- disk
-Listen: :9005
ManagementToken: <span class="userinput">xyzzy</span>
RunPeriod: 10m
CollectionBatchSize: 100000
CollectionBuffers: 1000
+LostBlocksFile: /tmp/keep-balance-lost-blocks.txt # If given, this file will be updated atomically during each successful run.
</code></pre>
</notextile>
diff --git a/services/keep-balance/balance.go b/services/keep-balance/balance.go
index ab8aadca0..adf9b9b3c 100644
--- a/services/keep-balance/balance.go
+++ b/services/keep-balance/balance.go
@@ -8,12 +8,16 @@ import (
"bytes"
"crypto/md5"
"fmt"
+ "io"
+ "io/ioutil"
"log"
"math"
+ "os"
"runtime"
"sort"
"strings"
"sync"
+ "syscall"
"time"
"git.curoverse.com/arvados.git/sdk/go/arvados"
@@ -35,6 +39,8 @@ type Balancer struct {
Dumper *logrus.Logger
Metrics *metrics
+ LostBlocksFile string
+
*BlockStateMap
KeepServices map[string]*KeepService
DefaultReplication int
@@ -48,6 +54,7 @@ type Balancer struct {
errors []error
stats balancerStats
mutex sync.Mutex
+ lostBlocks io.Writer
}
// Run performs a balance operation using the given config and
@@ -64,6 +71,30 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (nextRunOptions R
defer bal.time("sweep", "wall clock time to run one full sweep")()
+ var lbFile *os.File
+ if bal.LostBlocksFile != "" {
+ tmpfn := bal.LostBlocksFile + ".tmp"
+ lbFile, err = os.OpenFile(tmpfn, os.O_CREATE|os.O_WRONLY, 0777)
+ if err != nil {
+ return
+ }
+ defer lbFile.Close()
+ err = syscall.Flock(int(lbFile.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
+ if err != nil {
+ return
+ }
+ defer func() {
+ // Remove the tempfile only if we didn't get
+ // as far as successfully renaming it.
+ if lbFile != nil {
+ os.Remove(tmpfn)
+ }
+ }()
+ bal.lostBlocks = lbFile
+ } else {
+ bal.lostBlocks = ioutil.Discard
+ }
+
if len(config.KeepServiceList.Items) > 0 {
err = bal.SetKeepServices(config.KeepServiceList)
} else {
@@ -107,6 +138,17 @@ func (bal *Balancer) Run(config Config, runOptions RunOptions) (nextRunOptions R
if err = bal.CheckSanityLate(); err != nil {
return
}
+ if lbFile != nil {
+ err = lbFile.Sync()
+ if err != nil {
+ return
+ }
+ err = os.Rename(bal.LostBlocksFile+".tmp", bal.LostBlocksFile)
+ if err != nil {
+ return
+ }
+ lbFile = nil
+ }
if runOptions.CommitPulls {
err = bal.CommitPulls(&config.Client)
if err != nil {
@@ -885,6 +927,7 @@ func (bal *Balancer) collectStatistics(results <-chan balanceResult) {
s.lost.replicas -= surplus
s.lost.blocks++
s.lost.bytes += bytes * int64(-surplus)
+ fmt.Fprintf(bal.lostBlocks, "%s\n", strings.SplitN(string(result.blkid), "+", 2)[0])
case surplus < 0:
s.underrep.replicas -= surplus
s.underrep.blocks++
diff --git a/services/keep-balance/balance_run_test.go b/services/keep-balance/balance_run_test.go
index 545aeca86..de368239b 100644
--- a/services/keep-balance/balance_run_test.go
+++ b/services/keep-balance/balance_run_test.go
@@ -11,6 +11,7 @@ import (
"io/ioutil"
"net/http"
"net/http/httptest"
+ "os"
"strings"
"sync"
"time"
@@ -270,6 +271,28 @@ func (s *stubServer) serveKeepstoreIndexFoo4Bar1() *reqTracker {
return rt
}
+func (s *stubServer) serveKeepstoreIndexFoo1() *reqTracker {
+ rt := &reqTracker{}
+ s.mux.HandleFunc("/index/", func(w http.ResponseWriter, r *http.Request) {
+ rt.Add(r)
+ io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
+ })
+ for _, mounts := range stubMounts {
+ for i, mnt := range mounts {
+ i := i
+ s.mux.HandleFunc(fmt.Sprintf("/mounts/%s/blocks", mnt.UUID), func(w http.ResponseWriter, r *http.Request) {
+ rt.Add(r)
+ if i == 0 {
+ io.WriteString(w, "acbd18db4cc2f85cedef654fccc4a4d8+3 12345678\n\n")
+ } else {
+ io.WriteString(w, "\n")
+ }
+ })
+ }
+ }
+ return rt
+}
+
func (s *stubServer) serveKeepstoreTrash() *reqTracker {
return s.serveStatic("/trash", `{}`)
}
@@ -406,6 +429,32 @@ func (s *runSuite) TestDetectSkippedCollections(c *check.C) {
c.Check(pullReqs.Count(), check.Equals, 0)
}
+func (s *runSuite) TestWriteLostBlocks(c *check.C) {
+ lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
+ c.Assert(err, check.IsNil)
+ s.config.LostBlocksFile = lostf.Name()
+ defer os.Remove(lostf.Name())
+ opts := RunOptions{
+ CommitPulls: true,
+ CommitTrash: true,
+ Logger: s.logger(c),
+ }
+ s.stub.serveCurrentUserAdmin()
+ s.stub.serveFooBarFileCollections()
+ s.stub.serveKeepServices(stubServices)
+ s.stub.serveKeepstoreMounts()
+ s.stub.serveKeepstoreIndexFoo1()
+ s.stub.serveKeepstoreTrash()
+ s.stub.serveKeepstorePull()
+ srv, err := NewServer(s.config, opts)
+ c.Assert(err, check.IsNil)
+ _, err = srv.Run()
+ c.Check(err, check.IsNil)
+ lost, err := ioutil.ReadFile(lostf.Name())
+ c.Assert(err, check.IsNil)
+ c.Check(string(lost), check.Equals, "37b51d194a7513e45b56f6524f2d51f2\n")
+}
+
func (s *runSuite) TestDryRun(c *check.C) {
opts := RunOptions{
CommitPulls: false,
@@ -435,6 +484,11 @@ func (s *runSuite) TestDryRun(c *check.C) {
}
func (s *runSuite) TestCommit(c *check.C) {
+ lostf, err := ioutil.TempFile("", "keep-balance-lost-blocks-test-")
+ c.Assert(err, check.IsNil)
+ s.config.LostBlocksFile = lostf.Name()
+ defer os.Remove(lostf.Name())
+
s.config.Listen = ":"
s.config.ManagementToken = "xyzzy"
opts := RunOptions{
@@ -462,6 +516,10 @@ func (s *runSuite) TestCommit(c *check.C) {
// in a poor rendezvous position
c.Check(bal.stats.pulls, check.Equals, 2)
+ lost, err := ioutil.ReadFile(lostf.Name())
+ c.Assert(err, check.IsNil)
+ c.Check(string(lost), check.Equals, "")
+
metrics := s.getMetrics(c, srv)
c.Check(metrics, check.Matches, `(?ms).*\narvados_keep_total_bytes 15\n.*`)
c.Check(metrics, check.Matches, `(?ms).*\narvados_keepbalance_changeset_compute_seconds_sum [0-9\.]+\n.*`)
diff --git a/services/keep-balance/server.go b/services/keep-balance/server.go
index ad13be751..4a5f4acb3 100644
--- a/services/keep-balance/server.go
+++ b/services/keep-balance/server.go
@@ -57,6 +57,10 @@ type Config struct {
// Timeout for outgoing http request/response cycle.
RequestTimeout arvados.Duration
+
+ // Destination filename for the list of lost block hashes, one
+ // per line. Updated atomically during each successful run.
+ LostBlocksFile string
}
// RunOptions controls runtime behavior. The flags/options that belong
@@ -142,9 +146,10 @@ func (srv *Server) start() error {
func (srv *Server) Run() (*Balancer, error) {
bal := &Balancer{
- Logger: srv.Logger,
- Dumper: srv.Dumper,
- Metrics: srv.metrics,
+ Logger: srv.Logger,
+ Dumper: srv.Dumper,
+ Metrics: srv.metrics,
+ LostBlocksFile: srv.config.LostBlocksFile,
}
var err error
srv.runOptions, err = bal.Run(srv.config, srv.runOptions)
commit 3553f63170e747052452ec0640a4453fb462f437
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Apr 22 14:52:39 2019 -0400
15112: Update structure.sql for last migration.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/db/structure.sql b/services/api/db/structure.sql
index aa29a1cbb..c36e19f8b 100644
--- a/services/api/db/structure.sql
+++ b/services/api/db/structure.sql
@@ -3201,3 +3201,5 @@ INSERT INTO schema_migrations (version) VALUES ('20181005192222');
INSERT INTO schema_migrations (version) VALUES ('20181011184200');
+INSERT INTO schema_migrations (version) VALUES ('20190422144631');
+
commit 4b4f2356cc1be1d4f79a20dcd7686acb8376cd64
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Mon Apr 22 11:28:50 2019 -0400
15112: Fill in missing modified_at values from before #14595.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/services/api/db/migrate/20190422144631_fill_missing_modified_at.rb b/services/api/db/migrate/20190422144631_fill_missing_modified_at.rb
new file mode 100644
index 000000000..a83c25a39
--- /dev/null
+++ b/services/api/db/migrate/20190422144631_fill_missing_modified_at.rb
@@ -0,0 +1,11 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: AGPL-3.0
+
+class FillMissingModifiedAt < ActiveRecord::Migration
+ def up
+ Collection.where('modified_at is null').update_all('modified_at = created_at')
+ end
+ def down
+ end
+end
commit 7533ce8c161f3f02ae356b524b332b6740967e89
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Dec 13 15:08:53 2018 -0500
14595: Fix comment.
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/lib/arvados_model_updates.rb b/services/api/lib/arvados_model_updates.rb
index 0a07574a5..7f0d7c290 100644
--- a/services/api/lib/arvados_model_updates.rb
+++ b/services/api/lib/arvados_model_updates.rb
@@ -20,7 +20,7 @@ module ArvadosModelUpdates
end
# ArvadosModel checks this to decide whether it should update the
- # 'modified_at_uuid' field.
+ # 'modified_at' field.
def timeless_updater
Thread.current[:timeless_updater] || false
end
commit 232b4e63bbfa5b398a35a2e952968d643605272f
Author: Peter Amstutz <pamstutz at veritasgenetics.com>
Date: Thu Dec 13 14:37:29 2018 -0500
14595: Add leave_modified_at_alone
leave_modified_by_user_alone does not affect modified_at
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <pamstutz at veritasgenetics.com>
diff --git a/services/api/app/models/arvados_model.rb b/services/api/app/models/arvados_model.rb
index 93d5b9a02..eea95e2be 100644
--- a/services/api/app/models/arvados_model.rb
+++ b/services/api/app/models/arvados_model.rb
@@ -557,6 +557,8 @@ class ArvadosModel < ActiveRecord::Base
self.owner_uuid ||= current_default_owner if self.respond_to? :owner_uuid=
if !anonymous_updater
self.modified_by_user_uuid = current_user ? current_user.uuid : nil
+ end
+ if !timeless_updater
self.modified_at = current_time
end
self.modified_by_client_uuid = current_api_client ? current_api_client.uuid : nil
diff --git a/services/api/app/models/collection.rb b/services/api/app/models/collection.rb
index 33c811319..6147b79f9 100644
--- a/services/api/app/models/collection.rb
+++ b/services/api/app/models/collection.rb
@@ -288,7 +288,9 @@ class Collection < ArvadosModel
# Use a different validation context to skip the 'old_versions_cannot_be_updated'
# validator, as on this case it is legal to update some fields.
leave_modified_by_user_alone do
- c.save(context: :update_old_versions)
+ leave_modified_at_alone do
+ c.save(context: :update_old_versions)
+ end
end
end
end
diff --git a/services/api/lib/arvados_model_updates.rb b/services/api/lib/arvados_model_updates.rb
index b456bd395..0a07574a5 100644
--- a/services/api/lib/arvados_model_updates.rb
+++ b/services/api/lib/arvados_model_updates.rb
@@ -18,4 +18,21 @@ module ArvadosModelUpdates
Thread.current[:anonymous_updater] = anonymous_updater_was
end
end
+
+ # ArvadosModel checks this to decide whether it should update the
+ # 'modified_at_uuid' field.
+ def timeless_updater
+ Thread.current[:timeless_updater] || false
+ end
+
+ def leave_modified_at_alone
+ timeless_updater_was = timeless_updater
+ begin
+ Thread.current[:timeless_updater] = true
+ yield
+ ensure
+ Thread.current[:timeless_updater] = timeless_updater_was
+ end
+ end
+
end
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list