[ARVADOS] created: 1.3.0-3274-g9deed27d5
Git user
git at public.arvados.org
Wed Oct 7 01:07:58 UTC 2020
at 9deed27d5b61b2e51de1c70ae8baf06f18588e4d (commit)
commit 9deed27d5b61b2e51de1c70ae8baf06f18588e4d
Author: Ward Vandewege <ward at curii.com>
Date: Tue Oct 6 21:06:01 2020 -0400
16955: a number of fixes to make the federation migration tests pass
with arvbox again. Followup to #16267.
Arvados-DCO-1.1-Signed-off-by: Ward Vandewege <ward at curii.com>
diff --git a/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl b/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl
index e1cacdcaf..abd97ed8f 100644
--- a/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl
+++ b/sdk/cwl/tests/federation/arvboxcwl/fed-config.cwl
@@ -65,7 +65,7 @@ requirements:
arguments:
- shellQuote: false
valueFrom: |
- docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados
+ docker cp cluster_config.yml.override $(inputs.container_name):/var/lib/arvados-arvbox
docker cp application.yml.override $(inputs.container_name):/usr/src/arvados/services/api/config
$(inputs.arvbox_bin.path) sv restart api
$(inputs.arvbox_bin.path) sv restart controller
diff --git a/sdk/cwl/tests/federation/arvboxcwl/start.cwl b/sdk/cwl/tests/federation/arvboxcwl/start.cwl
index c933de254..2c453f768 100644
--- a/sdk/cwl/tests/federation/arvboxcwl/start.cwl
+++ b/sdk/cwl/tests/federation/arvboxcwl/start.cwl
@@ -98,4 +98,4 @@ arguments:
$(inputs.arvbox_bin.path) restart $(inputs.arvbox_mode)
fi
$(inputs.arvbox_bin.path) status > status.txt
- $(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token > superuser_token.txt
+ $(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/superuser_token > superuser_token.txt
diff --git a/sdk/python/tests/fed-migrate/fed-migrate.cwlex b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
index e0beaa91d..4c1db0f43 100644
--- a/sdk/python/tests/fed-migrate/fed-migrate.cwlex
+++ b/sdk/python/tests/fed-migrate/fed-migrate.cwlex
@@ -34,8 +34,8 @@ $(inputs.arvbox_bin.path) hotreset
while ! curl --fail --insecure --silent https://$(inputs.host)/discovery/v1/apis/arvados/v1/rest >/dev/null ; do sleep 3 ; done
-ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/vm-uuid)
-ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados/superuser_token)
+ARVADOS_VIRTUAL_MACHINE_UUID=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/vm-uuid)
+ARVADOS_API_TOKEN=\$($(inputs.arvbox_bin.path) cat /var/lib/arvados-arvbox/superuser_token)
while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_TOKEN" https://$(inputs.host)/arvados/v1/virtual_machines/$ARVADOS_VIRTUAL_MACHINE_UUID >/dev/null ; do sleep 3 ; done
>>>
@@ -47,4 +47,4 @@ while ! curl --fail --insecure --silent -H "Authorization: Bearer $ARVADOS_API_T
report = run_test(arvados_api_hosts, superuser_tokens=supertok, fed_migrate)
return supertok, report
-}
\ No newline at end of file
+}
diff --git a/tools/arvbox/bin/arvbox b/tools/arvbox/bin/arvbox
index 3040cb793..cd72ec06b 100755
--- a/tools/arvbox/bin/arvbox
+++ b/tools/arvbox/bin/arvbox
@@ -625,8 +625,6 @@ export RAILS_ENV=development
bundle exec rake db:drop
rm $ARVADOS_CONTAINER_PATH/api_database_setup
rm $ARVADOS_CONTAINER_PATH/superuser_token
-rm $ARVADOS_CONTAINER_PATH/keep0-uuid
-rm $ARVADOS_CONTAINER_PATH/keep1-uuid
rm $ARVADOS_CONTAINER_PATH/keepproxy-uuid
sv start api
sv start controller
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.base b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
index 115754cac..79f0d3f4f 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.base
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.base
@@ -133,6 +133,8 @@ COPY $workdir/runit /etc/runit
# put everything (/var/lib/arvados)
ENV ARVADOS_CONTAINER_PATH /var/lib/arvados-arvbox
+RUN /bin/ln -s /var/lib/arvados/bin/ruby /usr/local/bin/
+
# Start the supervisor.
ENV SVDIR /etc/service
STOPSIGNAL SIGINT
diff --git a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
index 777c71356..92d4e7088 100644
--- a/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
+++ b/tools/arvbox/lib/arvbox/docker/Dockerfile.demo
@@ -28,6 +28,17 @@ RUN mkdir -p $ARVADOS_CONTAINER_PATH
RUN echo "production" > $ARVADOS_CONTAINER_PATH/api_rails_env
RUN echo "production" > $ARVADOS_CONTAINER_PATH/workbench_rails_env
+# for the federation tests, the dev server watches a lot of files,
+# and we run three instances of the docker container. Bump up the
+# inotify limit from 8192, to avoid errors like
+# events.js:183
+# throw er; // Unhandled 'error' event
+# ^
+#
+# Error: watch /usr/src/workbench2/public ENOSPC
+# cf. https://github.com/facebook/jest/issues/3254
+RUN echo fs.inotify.max_user_watches=524288 >> /etc/sysctl.conf
+
RUN /usr/local/lib/arvbox/createusers.sh
RUN sudo -u arvbox /var/lib/arvbox/service/api/run-service --only-deps
diff --git a/tools/arvbox/lib/arvbox/docker/api-setup.sh b/tools/arvbox/lib/arvbox/docker/api-setup.sh
index f20278a69..4ad2aed0c 100755
--- a/tools/arvbox/lib/arvbox/docker/api-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/api-setup.sh
@@ -56,16 +56,16 @@ EOF
fi
if ! test -f $ARVADOS_CONTAINER_PATH/api_database_setup ; then
- bundle exec rake db:setup
+ flock $GEM_HOME/gems.lock bundle exec rake db:setup
touch $ARVADOS_CONTAINER_PATH/api_database_setup
fi
if ! test -s $ARVADOS_CONTAINER_PATH/superuser_token ; then
- superuser_tok=$(bundle exec ./script/create_superuser_token.rb)
+ superuser_tok=$(flock $GEM_HOME/gems.lock bundle exec ./script/create_superuser_token.rb)
echo "$superuser_tok" > $ARVADOS_CONTAINER_PATH/superuser_token
fi
rm -rf tmp
mkdir -p tmp/cache
-bundle exec rake db:migrate
+flock $GEM_HOME/gems.lock bundle exec rake db:migrate
diff --git a/tools/arvbox/lib/arvbox/docker/go-setup.sh b/tools/arvbox/lib/arvbox/docker/go-setup.sh
index 21be0ccd6..5bdc5207a 100644
--- a/tools/arvbox/lib/arvbox/docker/go-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/go-setup.sh
@@ -8,14 +8,11 @@ mkdir -p $GOPATH
cd /usr/src/arvados
if [[ $UID = 0 ]] ; then
- /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go mod download
- if [[ ! -f /usr/local/bin/arvados-server ]]; then
- /usr/local/lib/arvbox/runsu.sh flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
- fi
-else
- flock /var/lib/gopath/gopath.lock go mod download
- if [[ ! -f /usr/local/bin/arvados-server ]]; then
- flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
- fi
+ RUNSU="/usr/local/lib/arvbox/runsu.sh"
+fi
+
+if [[ ! -f /usr/local/bin/arvados-server ]]; then
+ $RUNSU flock /var/lib/gopath/gopath.lock go mod download
+ $RUNSU flock /var/lib/gopath/gopath.lock go install git.arvados.org/arvados.git/cmd/arvados-server
+ $RUNSU flock /var/lib/gopath/gopath.lock install $GOPATH/bin/arvados-server /usr/local/bin
fi
-install $GOPATH/bin/arvados-server /usr/local/bin
diff --git a/tools/arvbox/lib/arvbox/docker/keep-setup.sh b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
index 657a9a260..cb64f8406 100755
--- a/tools/arvbox/lib/arvbox/docker/keep-setup.sh
+++ b/tools/arvbox/lib/arvbox/docker/keep-setup.sh
@@ -19,40 +19,4 @@ fi
mkdir -p $ARVADOS_CONTAINER_PATH/$1
-export ARVADOS_API_HOST=$localip:${services[controller-ssl]}
-export ARVADOS_API_HOST_INSECURE=1
-export ARVADOS_API_TOKEN=$(cat $ARVADOS_CONTAINER_PATH/superuser_token)
-
-set +e
-read -rd $'\000' keepservice <<EOF
-{
- "service_host":"localhost",
- "service_port":$2,
- "service_ssl_flag":false,
- "service_type":"disk"
-}
-EOF
-set -e
-
-if test -s $ARVADOS_CONTAINER_PATH/$1-uuid ; then
- keep_uuid=$(cat $ARVADOS_CONTAINER_PATH/$1-uuid)
- arv keep_service update --uuid $keep_uuid --keep-service "$keepservice"
-else
- UUID=$(arv --format=uuid keep_service create --keep-service "$keepservice")
- echo $UUID > $ARVADOS_CONTAINER_PATH/$1-uuid
-fi
-
-management_token=$(cat $ARVADOS_CONTAINER_PATH/management_token)
-
-set +e
-sv hup /var/lib/arvbox/service/keepproxy
-
-cat >$ARVADOS_CONTAINER_PATH/$1.yml <<EOF
-Listen: "localhost:$2"
-BlobSigningKeyFile: $ARVADOS_CONTAINER_PATH/blob_signing_key
-SystemAuthTokenFile: $ARVADOS_CONTAINER_PATH/superuser_token
-ManagementToken: $management_token
-MaxBuffers: 20
-EOF
-
-exec /usr/local/bin/keepstore -config=$ARVADOS_CONTAINER_PATH/$1.yml
+exec /usr/local/bin/keepstore
diff --git a/tools/arvbox/lib/arvbox/docker/service/api/run-service b/tools/arvbox/lib/arvbox/docker/service/api/run-service
index 7df7b2820..b586afb65 100755
--- a/tools/arvbox/lib/arvbox/docker/service/api/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/api/run-service
@@ -17,8 +17,8 @@ else
fi
run_bundler --without=development
-bundle exec passenger-config build-native-support
-bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
if test "$1" = "--only-deps" ; then
exit
diff --git a/tools/arvbox/lib/arvbox/docker/service/doc/run-service b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
index 66a4a28ec..fa7aed1ef 100755
--- a/tools/arvbox/lib/arvbox/docker/service/doc/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/doc/run-service
@@ -24,4 +24,4 @@ if test "$1" = "--only-deps" ; then
fi
cd /usr/src/arvados/doc
-bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
+flock $GEM_HOME/gems.lock bundle exec rake generate baseurl=http://$localip:${services[doc]} arvados_api_host=$localip:${services[controller-ssl]} arvados_workbench_host=http://$localip
diff --git a/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service b/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
index e91386b67..967c75f90 100755
--- a/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/gitolite/run-service
@@ -119,6 +119,6 @@ $RAILS_ENV:
EOF
while true ; do
- bundle exec script/arvados-git-sync.rb $RAILS_ENV
+ flock $GEM_HOME/gems.lock bundle exec script/arvados-git-sync.rb $RAILS_ENV
sleep 120
done
diff --git a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
index 51b9420ee..b40d4f569 100755
--- a/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
+++ b/tools/arvbox/lib/arvbox/docker/service/workbench/run-service
@@ -17,8 +17,8 @@ else
fi
run_bundler --without=development
-bundle exec passenger-config build-native-support
-bundle exec passenger-config install-standalone-runtime
+flock $GEM_HOME/gems.lock bundle exec passenger-config build-native-support
+flock $GEM_HOME/gems.lock bundle exec passenger-config install-standalone-runtime
mkdir -p /usr/src/arvados/apps/workbench/tmp
if test "$1" = "--only-deps" ; then
@@ -28,7 +28,7 @@ cat >config/application.yml <<EOF
$RAILS_ENV:
keep_web_url: https://example.com/c=%{uuid_or_pdh}
EOF
- RAILS_GROUPS=assets bundle exec rake npm:install
+ RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
rm config/application.yml
exit
fi
@@ -57,5 +57,5 @@ EOF
(cd config && /usr/local/lib/arvbox/yml_override.py application.yml)
fi
-RAILS_GROUPS=assets bundle exec rake npm:install
-bundle exec rake assets:precompile
+RAILS_GROUPS=assets flock $GEM_HOME/gems.lock bundle exec rake npm:install
+flock $GEM_HOME/gems.lock bundle exec rake assets:precompile
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list