[ARVADOS] created: 1.4.2-17-g1dee29f4b
Git user
git at public.arvados.org
Thu Jan 6 17:47:19 UTC 2022
at 1dee29f4b17ba38fd0fd99e8d8748190a87109ba (commit)
commit 1dee29f4b17ba38fd0fd99e8d8748190a87109ba
Author: Ward Vandewege <ward at curii.com>
Date: Mon Aug 10 13:55:27 2020 -0400
documentation: the arvados k8s repo has changed name.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/doc/install/arvados-on-kubernetes.html.textile.liquid b/doc/install/arvados-on-kubernetes.html.textile.liquid
index 01999f0c2..4e7b8b73f 100644
--- a/doc/install/arvados-on-kubernetes.html.textile.liquid
+++ b/doc/install/arvados-on-kubernetes.html.textile.liquid
@@ -55,11 +55,11 @@ There should be no errors. The command will return nothing.
h2(#git). Clone the repository
-Clone the repository and nagivate to the @arvados-kubernetes/charts/arvados@ directory:
+Clone the repository and nagivate to the @arvados-k8s/charts/arvados@ directory:
<pre>
-$ git clone https://github.com/curoverse/arvados-kubernetes.git
-$ cd arvados-kubernetes/charts/arvados
+$ git clone https://github.com/arvados/arvados-k8s.git
+$ cd arvados-k8s/charts/arvados
</pre>
h2(#Start). Start the Arvados cluster
commit a282597c417ac4f670055eee42c90828d212bb05
Author: Ward Vandewege <ward at jhvc.com>
Date: Tue Jan 21 09:04:45 2020 -0500
Add missing definition of `all_users_group_uuid` on the compute node
install page.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at jhvc.com>
diff --git a/doc/_includes/_install_compute_docker.liquid b/doc/_includes/_install_compute_docker.liquid
index ea3640e52..4a600724b 100644
--- a/doc/_includes/_install_compute_docker.liquid
+++ b/doc/_includes/_install_compute_docker.liquid
@@ -56,6 +56,7 @@ Here we create a default project for the standard Arvados Docker images, and giv
<notextile>
<pre><code>~$ <span class="userinput">uuid_prefix=`arv --format=uuid user current | cut -d- -f1`</span>
+~$ <span class="userinput">all_users_group_uuid="$uuid_prefix-j7d0g-fffffffffffffff"</span>
~$ <span class="userinput">project_uuid=`arv --format=uuid group create --group "{\"owner_uuid\":\"$uuid_prefix-tpzed-000000000000000\", \"group_class\":\"project\", \"name\":\"Arvados Standard Docker Images\"}"`</span>
~$ <span class="userinput">echo "Arvados project uuid is '$project_uuid'"</span>
~$ <span class="userinput">read -rd $'\000' newlink <<EOF; arv link create --link "$newlink"</span>
diff --git a/sdk/cli/Gemfile b/sdk/cli/Gemfile
index f34204e02..1234eb271 100644
--- a/sdk/cli/Gemfile
+++ b/sdk/cli/Gemfile
@@ -5,4 +5,5 @@
source 'https://rubygems.org'
gemspec
gem 'minitest', '>= 5.0.0'
+gem 'signet', '< 0.12.0'
gem 'rake'
commit 502eeae62a21874fb8b5d377b06b6e75c1c429a8
Author: Ward Vandewege <ward at jhvc.com>
Date: Wed Jan 15 15:31:56 2020 -0500
Make the arvados-controller install docs less misleading for the 1.4
release.
No issue #
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at jhvc.com>
diff --git a/doc/install/install-controller.html.textile.liquid b/doc/install/install-controller.html.textile.liquid
index 3e94b290d..af17e67b1 100644
--- a/doc/install/install-controller.html.textile.liquid
+++ b/doc/install/install-controller.html.textile.liquid
@@ -25,16 +25,6 @@ On Red Hat-based systems:
</code></pre>
</notextile>
-Verify the @arvados-controller@ program is functional:
-
-<notextile>
-<pre><code>~$ <span class="userinput">arvados-controller -h</span>
-Usage:
- -config file
-[...]
-</code></pre>
-</notextile>
-
h3. Configure Nginx to route requests to the controller
Add @upstream@ and @server@ definitions inside the @http@ section of your Nginx configuration using the following template.
@@ -116,6 +106,18 @@ Create the host configuration file @/etc/arvados/environment at .
</code></pre>
</notextile>
+Verify the @arvados-controller@ program is functional:
+
+<notextile>
+<pre><code>~$ <span class="userinput">. /etc/arvados/environment</span>
+~$ <span class="userinput">arvados-controller -h</span>
+Usage:
+ -config file
+[...]
+</code></pre>
+</notextile>
+
+
h3. Start the service (option 1: systemd)
If your system does not use systemd, skip this section and follow the "runit instructions":#runit instead.
commit 4d0bfd26812595cde3416768889bf644df65036f
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Jan 3 15:44:28 2020 -0500
setuptools >=44 doesn't support Python 3.4 on Debian jessie
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/build/run-library.sh b/build/run-library.sh
index 1aa3e3cfd..bb789eabf 100755
--- a/build/run-library.sh
+++ b/build/run-library.sh
@@ -435,8 +435,8 @@ fpm_build_virtualenv () {
rm -rf dist/*
- # Get the latest setuptools
- if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools; then
+ # Get setuptools, but <44 because >=44 doesn't support python 3.4
+ if ! $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U 'setuptools<44'; then
echo "Error, unable to upgrade setuptools with"
echo " $pip install $DASHQ_UNLESS_DEBUG $CACHE_FLAG -U setuptools"
exit 1
commit d1757cb278f370eabc16fc0314afe01a3f932656
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Jan 3 15:40:02 2020 -0500
Update bundles to a commit that can be installed with git 2.1.4.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/apps/workbench/Gemfile b/apps/workbench/Gemfile
index 136e1948b..26188676a 100644
--- a/apps/workbench/Gemfile
+++ b/apps/workbench/Gemfile
@@ -5,7 +5,7 @@
source 'https://rubygems.org'
gem 'rails', '~> 5.0.0'
-gem 'arvados', git: 'https://github.com/curoverse/arvados.git', glob: 'sdk/ruby/arvados.gemspec', branch: '15900-gem-version-142'
+gem 'arvados', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/ruby/arvados.gemspec', branch: '15900-gem-version-143'
gem 'activerecord-nulldb-adapter', git: 'https://github.com/curoverse/nulldb'
gem 'multi_json'
diff --git a/apps/workbench/Gemfile.lock b/apps/workbench/Gemfile.lock
index af6dc647f..d01cfd20f 100644
--- a/apps/workbench/Gemfile.lock
+++ b/apps/workbench/Gemfile.lock
@@ -1,10 +1,10 @@
GIT
- remote: https://github.com/curoverse/arvados.git
- revision: 2c952e4c3635e02b231019ed1bca2700c35a5682
- branch: 15900-gem-version-142
+ remote: https://github.com/arvados/arvados.git
+ revision: 311443795cf368f62c984838d2ed385f6a95694c
+ branch: 15900-gem-version-143
glob: sdk/ruby/arvados.gemspec
specs:
- arvados (1.4.3.pre20191203162527)
+ arvados (1.4.3.pre20200103202408)
activesupport (>= 3)
andand (~> 1.3, >= 1.3.3)
arvados-google-api-client (>= 0.7, < 0.8.9)
diff --git a/services/api/Gemfile b/services/api/Gemfile
index 29829c159..df37bf297 100644
--- a/services/api/Gemfile
+++ b/services/api/Gemfile
@@ -59,8 +59,8 @@ gem 'themes_for_rails', git: 'https://github.com/curoverse/themes_for_rails'
# two gems with the same "git" url but different "glob" values, hence
# the use of a wildcard here instead of literal paths
# (sdk/cli/arvados-cli.gem and sdk/ruby/arvados.gem).
-gem 'arvados-cli', git: 'https://github.com/curoverse/arvados.git', glob: 'sdk/*/*.gemspec', branch: '15900-gem-version-142'
-gem 'arvados', git: 'https://github.com/curoverse/arvados.git', glob: 'sdk/*/*.gemspec', branch: '15900-gem-version-142'
+gem 'arvados-cli', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/*/*.gemspec', branch: '15900-gem-version-143'
+gem 'arvados', git: 'https://github.com/arvados/arvados.git', glob: 'sdk/*/*.gemspec', branch: '15900-gem-version-143'
gem 'httpclient'
gem 'sshkey'
diff --git a/services/api/Gemfile.lock b/services/api/Gemfile.lock
index 204c509bd..07347a400 100644
--- a/services/api/Gemfile.lock
+++ b/services/api/Gemfile.lock
@@ -1,10 +1,10 @@
GIT
- remote: https://github.com/curoverse/arvados.git
- revision: 2c952e4c3635e02b231019ed1bca2700c35a5682
- branch: 15900-gem-version-142
+ remote: https://github.com/arvados/arvados.git
+ revision: 311443795cf368f62c984838d2ed385f6a95694c
+ branch: 15900-gem-version-143
glob: sdk/*/*.gemspec
specs:
- arvados (1.4.3.pre20191203162527)
+ arvados (1.4.3.pre20200103202408)
activesupport (>= 3)
andand (~> 1.3, >= 1.3.3)
arvados-google-api-client (>= 0.7, < 0.8.9)
@@ -12,7 +12,7 @@ GIT
i18n (~> 0)
json (>= 1.7.7, < 3)
jwt (>= 0.1.5, < 2)
- arvados-cli (1.4.3.pre20191203162527)
+ arvados-cli (1.4.3.pre20200103202408)
activesupport (>= 3.2.13, < 5.1)
andand (~> 1.3, >= 1.3.3)
arvados (>= 1.4.1.20190320201707)
commit 311443795cf368f62c984838d2ed385f6a95694c
Author: Tom Clegg <tom at tomclegg.ca>
Date: Fri Jan 3 15:24:08 2020 -0500
Update version number script to work with git 2.1.4.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/build/version-at-commit.sh b/build/version-at-commit.sh
index 2f514c82d..3a52c4b20 100755
--- a/build/version-at-commit.sh
+++ b/build/version-at-commit.sh
@@ -12,6 +12,7 @@ if ! git describe --exact-match --match "$versionglob" "$commit" 2>/dev/null; th
# x.y.(z+1).preTIMESTAMP, where x.y.z is the latest released ancestor of $commit
v=$(git describe --abbrev=0 --match "$versionglob" "$commit" | perl -pe 's/(\d+)$/$1+1/e')
fi
- ts=$(TZ=UTC git log -n1 --format=%cd --date="format-local:%Y%m%d%H%M%S" "$commit")
+ isodate=$(TZ=UTC git log -n1 --format=%cd --date=iso "$commit")
+ ts=$(TZ=UTC date --date="$isodate" "+%Y%m%d%H%M%S")
echo "$v.pre$ts"
fi
commit d9007d9fcab69b97f3eb8d85e4eb17b93c475b0b
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Thu Jan 2 15:53:56 2020 -0500
15965: Decode as UTF-8 to avoid "not ASCII" error on Python 2
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/sdk/python/arvados/commands/keepdocker.py b/sdk/python/arvados/commands/keepdocker.py
index d4fecc47b..62b9a9ed1 100644
--- a/sdk/python/arvados/commands/keepdocker.py
+++ b/sdk/python/arvados/commands/keepdocker.py
@@ -105,7 +105,7 @@ def docker_image_format(image_hash):
cmd = popen_docker(['inspect', '--format={{.Id}}', image_hash],
stdout=subprocess.PIPE)
try:
- image_id = next(cmd.stdout).decode().strip()
+ image_id = next(cmd.stdout).decode('utf-8').strip()
if image_id.startswith('sha256:'):
return 'v2'
elif ':' not in image_id:
@@ -136,7 +136,7 @@ def docker_images():
next(list_output) # Ignore the header line
for line in list_output:
words = line.split()
- words = [word.decode() for word in words]
+ words = [word.decode('utf-8') for word in words]
size_index = len(words) - 2
repo, tag, imageid = words[:3]
ctime = ' '.join(words[3:size_index])
@@ -513,7 +513,7 @@ def main(arguments=None, stdout=sys.stdout, install_sig_handlers=True, api=None)
else:
json_filename = raw_image_hash + '/json'
json_file = image_tar.extractfile(image_tar.getmember(json_filename))
- image_metadata = json.loads(json_file.read().decode())
+ image_metadata = json.loads(json_file.read().decode('utf-8'))
json_file.close()
image_tar.close()
link_base = {'head_uuid': coll_uuid, 'properties': {}}
commit 7892b534a9bdf317e61f470300065ef235edc117
Author: Tom Clegg <tom at tomclegg.ca>
Date: Sat Dec 21 19:20:45 2019 -0500
15942: Fix deadlock caused by unclosed "done" channel.
Also, stop relying on the flushing goroutine to set flushing=nil when
finished; closing the channel is now enough to indicate work is done.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 609eeb6e9..0aef95402 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -565,7 +565,6 @@ func (fn *filenode) pruneMemSegments() {
// A new seg.buf has been allocated.
return
}
- seg.flushing = nil
if err != nil {
// TODO: stall (or return errors from)
// subsequent writes until flushing
@@ -668,9 +667,10 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
offsets := make([]int, 0, len(refs)) // location of segment's data within block
for _, ref := range refs {
seg := ref.fn.segments[ref.idx].(*memSegment)
- if seg.flushing != nil && !sync {
+ if !sync && seg.flushingUnfinished() {
// Let the other flushing goroutine finish. If
// it fails, we'll try again next time.
+ close(done)
return nil
} else {
// In sync mode, we proceed regardless of
@@ -697,15 +697,6 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
defer close(errs)
locator, _, err := dn.fs.PutB(block)
dn.fs.throttle().Release()
- {
- defer func() {
- for _, seg := range segs {
- if seg.flushing == done {
- seg.flushing = nil
- }
- }
- }()
- }
if err != nil {
errs <- err
return
@@ -1195,13 +1186,26 @@ type segment interface {
type memSegment struct {
buf []byte
- // If flushing is not nil, then a) buf is being shared by a
- // pruneMemSegments goroutine, and must be copied on write;
- // and b) the flushing channel will close when the goroutine
- // finishes, whether it succeeds or not.
+ // If flushing is not nil and not ready/closed, then a) buf is
+ // being shared by a pruneMemSegments goroutine, and must be
+ // copied on write; and b) the flushing channel will close
+ // when the goroutine finishes, whether it succeeds or not.
flushing <-chan struct{}
}
+func (me *memSegment) flushingUnfinished() bool {
+ if me.flushing == nil {
+ return false
+ }
+ select {
+ case <-me.flushing:
+ me.flushing = nil
+ return false
+ default:
+ return true
+ }
+}
+
func (me *memSegment) Len() int {
return len(me.buf)
}
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index 6ef7627db..a1a888b3e 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -17,6 +17,7 @@ import (
"os"
"regexp"
"runtime"
+ "runtime/pprof"
"strings"
"sync"
"sync/atomic"
@@ -1281,6 +1282,47 @@ func (s *CollectionFSSuite) TestMaxUnflushed(c *check.C) {
fs.Flush("", true)
}
+func (s *CollectionFSSuite) TestFlushStress(c *check.C) {
+ done := false
+ defer func() { done = true }()
+ time.AfterFunc(10*time.Second, func() {
+ if !done {
+ pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
+ panic("timeout")
+ }
+ })
+
+ wrote := 0
+ s.kc.onPut = func(p []byte) {
+ s.kc.Lock()
+ s.kc.blocks = map[string][]byte{}
+ wrote++
+ defer c.Logf("wrote block %d, %d bytes", wrote, len(p))
+ s.kc.Unlock()
+ time.Sleep(20 * time.Millisecond)
+ }
+
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+
+ data := make([]byte, 1<<20)
+ for i := 0; i < 3; i++ {
+ dir := fmt.Sprintf("dir%d", i)
+ fs.Mkdir(dir, 0755)
+ for j := 0; j < 200; j++ {
+ data[0] = byte(j)
+ f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, j), os.O_WRONLY|os.O_CREATE, 0)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write(data)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ fs.Flush(dir, false)
+ }
+ _, err := fs.MarshalManifest(".")
+ c.Check(err, check.IsNil)
+ }
+}
+
func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
s.kc.onPut = func([]byte) {
s.kc.Lock()
@@ -1302,6 +1344,9 @@ func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
f.Close()
fs.Flush(dir, false)
}
+ fs.Flush(dir, true)
+ _, err := fs.MarshalManifest(".")
+ c.Check(err, check.IsNil)
}
}
commit dbd8c8df5bdfcd2c3717918abf8e6e7a2ec65dec
Author: Ward Vandewege <ward at jhvc.com>
Date: Tue Dec 17 15:54:12 2019 -0500
Update CI url, and add Curii Corporation to the AUTHORS file.
No issue #
diff --git a/AUTHORS b/AUTHORS
index 9a861a631..436a504c3 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -18,3 +18,4 @@ President and Fellows of Harvard College <*@harvard.edu>
Thomas Mooney <tmooney at genome.wustl.edu>
Chen Chen <aflyhorse at gmail.com>
Veritas Genetics, Inc. <*@veritasgenetics.com>
+Curii Corporation, Inc. <*@curii.com>
diff --git a/README.md b/README.md
index 08e645037..fd359fa61 100644
--- a/README.md
+++ b/README.md
@@ -58,7 +58,7 @@ contributors to Arvados.
## Development
-[](https://ci.curoverse.com/job/run-tests/)
+[](https://ci.arvados.org/job/run-tests/)
[](https://goreportcard.com/report/github.com/curoverse/arvados)
The Arvados public bug tracker is located at https://dev.arvados.org/projects/arvados/issues
commit 947370baeacd86738446bb2ea6a209390e0d83dc
Author: Tom Clegg <tom at tomclegg.ca>
Date: Wed Dec 11 10:40:01 2019 -0500
15928: Fix deadlock.
Locking the parent dir isn't necessary to finish async writes, and
deadlocks if a waitPrune() is waiting for the async writes to finish.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at tomclegg.ca>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index d206cb3b2..609eeb6e9 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -695,21 +695,9 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
go func() {
defer close(done)
defer close(errs)
- locked := map[*filenode]bool{}
locator, _, err := dn.fs.PutB(block)
dn.fs.throttle().Release()
{
- if !sync {
- dn.Lock()
- defer dn.Unlock()
- for _, name := range dn.sortedNames() {
- if fn, ok := dn.inodes[name].(*filenode); ok {
- fn.Lock()
- defer fn.Unlock()
- locked[fn] = true
- }
- }
- }
defer func() {
for _, seg := range segs {
if seg.flushing == done {
@@ -724,6 +712,7 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
}
for idx, ref := range refs {
if !sync {
+ ref.fn.Lock()
// In async mode, fn's lock was
// released while we were waiting for
// PutB(); lots of things might have
@@ -732,17 +721,15 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
// file segments have
// rearranged or changed in
// some way
+ ref.fn.Unlock()
continue
} else if seg, ok := ref.fn.segments[ref.idx].(*memSegment); !ok || seg != segs[idx] {
// segment has been replaced
+ ref.fn.Unlock()
continue
} else if seg.flushing != done {
// seg.buf has been replaced
- continue
- } else if !locked[ref.fn] {
- // file was renamed, moved, or
- // deleted since we called
- // PutB
+ ref.fn.Unlock()
continue
}
}
@@ -760,6 +747,9 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
// lock, writing different segments from the
// same file.
atomic.AddInt64(&ref.fn.memsize, -int64(len(data)))
+ if !sync {
+ ref.fn.Unlock()
+ }
}
}()
if sync {
commit 3dfdd2f3767314e8014b45ba590efb7453710c77
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Dec 9 15:43:52 2019 -0500
Update gitter link to arvados/community
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/README.md b/README.md
index 78a4846c7..08e645037 100644
--- a/README.md
+++ b/README.md
@@ -42,9 +42,9 @@ doc/README.textile for instructions.
## Community
-[](https://gitter.im/curoverse/arvados?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[](https://gitter.im/arvados/community?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
-The [curoverse/arvados channel](https://gitter.im/curoverse/arvados)
+The [arvados community channel](https://gitter.im/arvados/community)
channel at [gitter.im](https://gitter.im) is available for live
discussion and support.
diff --git a/doc/admin/upgrading.html.textile.liquid b/doc/admin/upgrading.html.textile.liquid
index 894e31b86..e4c6e1270 100644
--- a/doc/admin/upgrading.html.textile.liquid
+++ b/doc/admin/upgrading.html.textile.liquid
@@ -141,7 +141,13 @@ h4. New configuration
Arvados is migrating to a centralized configuration file for all components. During the migration, legacy configuration files will continue to be loaded. See "Migrating Configuration":config-migration.html for details.
-h3. v1.3.0 (2018-12-05)
+h3(#v1_3_3). v1.3.3 (2019-05-14)
+
+This release corrects a potential data loss issue, if you are running Arvados 1.3.0 or 1.3.1 we strongly recommended disabling @keep-balance@ until you can upgrade to 1.3.3 or 1.4.0. With keep-balance disabled, there is no chance of data loss.
+
+We've put together a "wiki page":https://dev.arvados.org/projects/arvados/wiki/Recovering_lost_data which outlines how to recover blocks which have been put in the trash, but not yet deleted, as well as how to identify any collections which have missing blocks so that they can be regenerated. The keep-balance component has been enhanced to provide a list of missing blocks and affected collections and we've provided a "utility script":https://github.com/arvados/arvados/blob/master/tools/keep-xref/keep-xref.py which can be used to identify the workflows that generated those collections and who ran those workflows, so that they can be rerun.
+
+h3(#v1_3_0). v1.3.0 (2018-12-05)
This release includes several database migrations, which will be executed automatically as part of the API server upgrade. On large Arvados installations, these migrations will take a while. We've seen the upgrade take 30 minutes or more on installations with a lot of collections.
diff --git a/doc/index.html.liquid b/doc/index.html.liquid
index ec0639236..54bc2ff22 100644
--- a/doc/index.html.liquid
+++ b/doc/index.html.liquid
@@ -34,7 +34,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
<a name="Support"></a>
<p><strong>Support and Community</strong></p>
- <p>The <a href="https://gitter.im/curoverse/arvados">#arvados channel</a> at gitter.im is available for live discussion and community support. There is also a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>.
+ <p>The <a href="https://gitter.im/arvados/community">arvados community channel</a> at gitter.im is available for live discussion and community support. There is also a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>.
</p>
<p>Curii Corporation provides managed Arvados installations as well as commercial support for Arvados. Please contact <a href="mailto:info at curii.com">info at curii.com</a> for more information.</p>
diff --git a/doc/user/cwl/cwl-runner.html.textile.liquid b/doc/user/cwl/cwl-runner.html.textile.liquid
index fbce8e17b..f38cbf1de 100644
--- a/doc/user/cwl/cwl-runner.html.textile.liquid
+++ b/doc/user/cwl/cwl-runner.html.textile.liquid
@@ -25,10 +25,10 @@ h2. Running arvados-cwl-runner
h3. Get the example files
-The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/curoverse/arvados/tree/master/doc/user/cwl/bwa-mem
+The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/arvados/arvados/tree/master/doc/user/cwl/bwa-mem
<notextile>
-<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados</span>
+<pre><code>~$ <span class="userinput">git clone https://github.com/arvados/arvados</span>
~$ <span class="userinput">cd arvados/doc/user/cwl/bwa-mem</span>
</code></pre>
</notextile>
diff --git a/doc/user/cwl/federated-workflows.html.textile.liquid b/doc/user/cwl/federated-workflows.html.textile.liquid
index 7e2150dcc..eb03c1587 100644
--- a/doc/user/cwl/federated-workflows.html.textile.liquid
+++ b/doc/user/cwl/federated-workflows.html.textile.liquid
@@ -17,10 +17,10 @@ For more information, visit the "architecture":{{site.baseurl}}/architecture/fed
h2. Get the example files
-The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/curoverse/arvados/tree/master/doc/user/cwl/federated or "see below":#fed-example
+The tutorial files are located in the "documentation section of the Arvados source repository:":https://github.com/arvados/arvados/tree/master/doc/user/cwl/federated or "see below":#fed-example
<notextile>
-<pre><code>~$ <span class="userinput">git clone https://github.com/curoverse/arvados</span>
+<pre><code>~$ <span class="userinput">git clone https://github.com/arvados/arvados</span>
~$ <span class="userinput">cd arvados/doc/user/cwl/federated</span>
</code></pre>
</notextile>
diff --git a/doc/user/getting_started/community.html.textile.liquid b/doc/user/getting_started/community.html.textile.liquid
index 40c67ad11..b85b55612 100644
--- a/doc/user/getting_started/community.html.textile.liquid
+++ b/doc/user/getting_started/community.html.textile.liquid
@@ -19,7 +19,7 @@ The "Arvados user mailing list":http://lists.arvados.org/mailman/listinfo/arvado
h2. Chat
-The "curoverse/arvados channel":https://gitter.im/curoverse/arvados channel at "gitter.im":https://gitter.im is available for live discussion and support.
+The "arvados community channel":https://gitter.im/arvados/community channel at "gitter.im":https://gitter.im is available for live discussion and support.
h2. Bug tracking
commit df06f668f22689e5b6c2a11e5f875f8e52c16427
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Dec 9 14:38:09 2019 -0500
Remove biostars link
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/doc/index.html.liquid b/doc/index.html.liquid
index 6e7704945..ec0639236 100644
--- a/doc/index.html.liquid
+++ b/doc/index.html.liquid
@@ -34,9 +34,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
<a name="Support"></a>
<p><strong>Support and Community</strong></p>
-<p>The recommended place to ask a question about Arvados is on Biostars. After you have <a href="//www.biostars.org/t/arvados/">read previous questions and answers</a> you can <a href="https://www.biostars.org/p/new/post/?tag_val=arvados">post your question using the 'arvados' tag</a>.</p>
-
- <p>There is a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>. The <a href="https://gitter.im/curoverse/arvados">#arvados channel</a> at gitter.im is available for live discussion and community support.
+ <p>The <a href="https://gitter.im/curoverse/arvados">#arvados channel</a> at gitter.im is available for live discussion and community support. There is also a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>.
</p>
<p>Curii Corporation provides managed Arvados installations as well as commercial support for Arvados. Please contact <a href="mailto:info at curii.com">info at curii.com</a> for more information.</p>
commit cd526a8473061071fc636e0180ca2d9c0f0e8435
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Mon Dec 9 14:26:31 2019 -0500
Fix support links
no issue #
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/README.md b/README.md
index 5843bb84d..78a4846c7 100644
--- a/README.md
+++ b/README.md
@@ -19,8 +19,6 @@ Arvados consists of:
## Quick start
-Veritas Genetics maintains a public installation of Arvados for evaluation and trial use, the [Arvados Playground](https://playground.arvados.org). A Google account is required to log in.
-
To try out Arvados on your local workstation, you can use Arvbox, which
provides Arvados components pre-installed in a Docker container (requires
Docker 1.9+). After cloning the Arvados git repository:
@@ -65,7 +63,7 @@ contributors to Arvados.
The Arvados public bug tracker is located at https://dev.arvados.org/projects/arvados/issues
-Continuous integration is hosted at https://ci.curoverse.com/
+Continuous integration is hosted at https://ci.arvados.org/
Instructions for setting up a development environment and working on specific
components can be found on the
diff --git a/doc/index.html.liquid b/doc/index.html.liquid
index ee3538778..6e7704945 100644
--- a/doc/index.html.liquid
+++ b/doc/index.html.liquid
@@ -39,7 +39,7 @@ SPDX-License-Identifier: CC-BY-SA-3.0
<p>There is a <a href="http://lists.arvados.org/mailman/listinfo/arvados">mailing list</a>. The <a href="https://gitter.im/curoverse/arvados">#arvados channel</a> at gitter.im is available for live discussion and community support.
</p>
- <p>Curoverse, a Veritas Genetics company, provides managed Arvados installations as well as commercial support for Arvados. Please visit <a href="https://curoverse.com">curoverse.com</a> or contact <a href="mailto:researchsales at veritasgenetics.com">researchsales at veritasgenetics.com</a> for more information.</p>
+ <p>Curii Corporation provides managed Arvados installations as well as commercial support for Arvados. Please contact <a href="mailto:info at curii.com">info at curii.com</a> for more information.</p>
<p><strong>Contributing</strong></p>
<p>Please visit the <a href="https://dev.arvados.org/projects/arvados/wiki/Wiki#Contributing-and-hacking">developer site</a>. Arvados is 100% free and open source software, check out the code on <a href="https://github.com/curoverse/arvados">github</a>.
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index c31b2ed43..527b62cf3 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -9,7 +9,7 @@ Copyright (C) The Arvados Authors. All rights reserved.
SPDX-License-Identifier: CC-BY-SA-3.0
{% endcomment %}
-Arvados components run on GNU/Linux systems, and supports multiple cloud operating stacks. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS.
+Arvados components run on GNU/Linux systems, and supports multiple cloud operating stacks. Arvados supports Debian and derivatives such as Ubuntu, as well as Red Hat and derivatives such as CentOS. "Arvados is Free Software":{{site.baseurl}}/copying/copying.html and self-install installations are not limited in any way. Commercial support and development are also available from "Curii Corporation.":mailto:info at curii.com
Arvados components can be installed and configured in a number of different ways.
@@ -20,10 +20,9 @@ table(table table-bordered table-condensed).
|"Arvados-in-a-box":arvbox.html (arvbox)|Easy|no|yes|no|yes|yes|
|"Arvados on Kubernetes":arvados-on-kubernetes.html|Easy ^1^|yes|yes ^2^|no ^2^|no|yes|
|"Manual installation":install-manual-prerequisites.html|Complicated|yes|yes|yes|no|no|
-|"Arvados Playground":https://playground.arvados.org hosted by Veritas Genetics|N/A ^3^|yes|yes|no|no|no|
-|"Cluster Operation Subscription":https://curoverse.com/products supported by Veritas Genetics|N/A ^3^|yes|yes|yes|yes|yes|
+|"Cluster Operation Subscription supported by Curii":mailto:info at curii.com|N/A ^3^|yes|yes|yes|yes|yes|
</div>
* ^1^ Assumes a Kubernetes cluster is available
* ^2^ Arvados on Kubernetes is under development and not yet ready for production use
-* ^3^ No installation necessary, Veritas Genetics run and managed
+* ^3^ No user installation necessary, Curii run and managed
commit 89f1571a26a5c90c79d241d049df73fb16b90954
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 4 11:28:38 2019 -0500
15910: Fix races in collectionfs flush/sync.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/sdk/go/arvados/fs_collection.go b/sdk/go/arvados/fs_collection.go
index 578d73f5a..d206cb3b2 100644
--- a/sdk/go/arvados/fs_collection.go
+++ b/sdk/go/arvados/fs_collection.go
@@ -16,6 +16,7 @@ import (
"strconv"
"strings"
"sync"
+ "sync/atomic"
"time"
)
@@ -688,6 +689,7 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
}
segs = append(segs, seg)
}
+ blocksize := len(block)
dn.fs.throttle().Acquire()
errs := make(chan error, 1)
go func() {
@@ -698,6 +700,8 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
dn.fs.throttle().Release()
{
if !sync {
+ dn.Lock()
+ defer dn.Unlock()
for _, name := range dn.sortedNames() {
if fn, ok := dn.inodes[name].(*filenode); ok {
fn.Lock()
@@ -746,11 +750,16 @@ func (dn *dirnode) commitBlock(ctx context.Context, refs []fnSegmentRef, bufsize
ref.fn.segments[ref.idx] = storedSegment{
kc: dn.fs,
locator: locator,
- size: len(block),
+ size: blocksize,
offset: offsets[idx],
length: len(data),
}
- ref.fn.memsize -= int64(len(data))
+ // atomic is needed here despite caller having
+ // lock: caller might be running concurrent
+ // commitBlock() goroutines using the same
+ // lock, writing different segments from the
+ // same file.
+ atomic.AddInt64(&ref.fn.memsize, -int64(len(data)))
}
}()
if sync {
diff --git a/sdk/go/arvados/fs_collection_test.go b/sdk/go/arvados/fs_collection_test.go
index e5cea0639..6ef7627db 100644
--- a/sdk/go/arvados/fs_collection_test.go
+++ b/sdk/go/arvados/fs_collection_test.go
@@ -1281,6 +1281,30 @@ func (s *CollectionFSSuite) TestMaxUnflushed(c *check.C) {
fs.Flush("", true)
}
+func (s *CollectionFSSuite) TestFlushShort(c *check.C) {
+ s.kc.onPut = func([]byte) {
+ s.kc.Lock()
+ s.kc.blocks = map[string][]byte{}
+ s.kc.Unlock()
+ }
+ fs, err := (&Collection{}).FileSystem(s.client, s.kc)
+ c.Assert(err, check.IsNil)
+ for _, blocksize := range []int{8, 1000000} {
+ dir := fmt.Sprintf("dir%d", blocksize)
+ err = fs.Mkdir(dir, 0755)
+ c.Assert(err, check.IsNil)
+ data := make([]byte, blocksize)
+ for i := 0; i < 100; i++ {
+ f, err := fs.OpenFile(fmt.Sprintf("%s/file%d", dir, i), os.O_WRONLY|os.O_CREATE, 0)
+ c.Assert(err, check.IsNil)
+ _, err = f.Write(data)
+ c.Assert(err, check.IsNil)
+ f.Close()
+ fs.Flush(dir, false)
+ }
+ }
+}
+
func (s *CollectionFSSuite) TestBrokenManifests(c *check.C) {
for _, txt := range []string{
"\n",
commit b702e93a96fededd1cfb46e862f9209596063156
Merge: 40eb23ae0 831828849
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Wed Dec 4 13:55:49 2019 -0500
Merge branch '15900-gem-version-142' into 1.4-dev
refs #15900
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list