[ARVADOS] created: 4f257e48f16b8e846119d9eab9a7ec02e1487986
git at public.curoverse.com
git at public.curoverse.com
Sun Nov 16 15:03:27 EST 2014
at 4f257e48f16b8e846119d9eab9a7ec02e1487986 (commit)
commit 4f257e48f16b8e846119d9eab9a7ec02e1487986
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 16 13:34:24 2014 -0500
3824: Create "all users" group in seeds.rb instead of asking a human to do it.
diff --git a/doc/install/create-standard-objects.html.textile.liquid b/doc/install/create-standard-objects.html.textile.liquid
index 92b0ade..e977bb7 100644
--- a/doc/install/create-standard-objects.html.textile.liquid
+++ b/doc/install/create-standard-objects.html.textile.liquid
@@ -1,29 +1,12 @@
---
layout: default
navsection: installguide
-title: Create standard objects
+title: Add an arvados repository
...
-Next, we're going to use the Arvados CLI tools on the <strong>shell server</strong> to create some standard objects.
-
-h3. "All users" group
-
-The convention is to add every active user to this group. We give it a distinctive UUID that looks like an IP broadcast address.
-
-<notextile>
-<pre><code>~$ <span class="userinput">prefix=`arv --format=uuid user current | cut -d- -f1`</span>
-~$ <span class="userinput">echo "Site prefix is '$prefix'"</span>
-~$ <span class="userinput">read -rd $'\000' newgroup <<EOF; arv group create --group "$newgroup"</span>
-<span class="userinput">{
- "uuid":"$prefix-j7d0g-fffffffffffffff",
- "name":"All users"
-}</span>
-EOF
-</code></pre></notextile>
-
-h3. "arvados" repository
+Next, we're going to use the Arvados CLI tools on the <strong>shell server</strong> to set up a locally hosted clone of the arvados repository.
This will be readable by the "All users" group, and therefore by every active user. This makes it possible for users to run the bundled Crunch scripts by specifying @"script_version":"master","repository":"arvados"@ rather than pulling the Arvados source tree into their own repositories.
diff --git a/docker/api/setup.sh.in b/docker/api/setup.sh.in
index cba4759..7af6afb 100755
--- a/docker/api/setup.sh.in
+++ b/docker/api/setup.sh.in
@@ -8,15 +8,6 @@ export ARVADOS_API_HOST=api
export ARVADOS_API_HOST_INSECURE=yes
export ARVADOS_API_TOKEN=@@API_SUPERUSER_SECRET@@
-# All users group
-prefix=`arv --format=uuid user current | cut -d- -f1`
-read -rd $'\000' newgroup <<EOF; arv group create --group "$newgroup"
-{
- "uuid":"$prefix-j7d0g-fffffffffffffff",
- "name":"All users"
-}
-EOF
-
# Arvados repository object
all_users_group_uuid="$prefix-j7d0g-fffffffffffffff"
repo_uuid=`arv --format=uuid repository create --repository '{"name":"arvados","fetch_url":"git at api:arvados.git","push_url":"git at api:arvados.git"}'`
diff --git a/services/api/app/models/user.rb b/services/api/app/models/user.rb
index b939d07..f55e24b 100644
--- a/services/api/app/models/user.rb
+++ b/services/api/app/models/user.rb
@@ -399,34 +399,15 @@ class User < ArvadosModel
# add the user to the 'All users' group
def create_user_group_link
- # Look up the "All users" group (we expect uuid *-*-fffffffffffffff).
- group = Group.where(name: 'All users').select do |g|
- g[:uuid].match /-f+$/
- end.first
-
- if not group
- logger.warn "No 'All users' group with uuid '*-*-fffffffffffffff'."
- raise "No 'All users' group with uuid '*-*-fffffffffffffff' is found"
- else
- logger.info { "\"All users\" group uuid: " + group[:uuid] }
-
- group_perms = Link.where(tail_uuid: self.uuid,
- head_uuid: group[:uuid],
- link_class: 'permission',
- name: 'can_read')
-
- if !group_perms.any?
- group_perm = Link.create(tail_uuid: self.uuid,
- head_uuid: group[:uuid],
- link_class: 'permission',
- name: 'can_read')
- logger.info { "group permission: " + group_perm[:uuid] }
- else
- group_perm = group_perms.first
- end
-
- return group_perm
- end
+ return (Link.where(tail_uuid: self.uuid,
+ head_uuid: all_users_group[:uuid],
+ link_class: 'permission',
+ name: 'can_read').first
+ or
+ Link.create(tail_uuid: self.uuid,
+ head_uuid: all_users_group[:uuid],
+ link_class: 'permission',
+ name: 'can_read'))
end
# Give the special "System group" permission to manage this user and
diff --git a/services/api/db/seeds.rb b/services/api/db/seeds.rb
index d397b91..0a13ae6 100644
--- a/services/api/db/seeds.rb
+++ b/services/api/db/seeds.rb
@@ -7,6 +7,7 @@
include CurrentApiClient
system_user
system_group
+all_users_group
anonymous_group
anonymous_user
empty_collection
diff --git a/services/api/lib/current_api_client.rb b/services/api/lib/current_api_client.rb
index 4c26010..eaf54da 100644
--- a/services/api/lib/current_api_client.rb
+++ b/services/api/lib/current_api_client.rb
@@ -98,6 +98,27 @@ module CurrentApiClient
$system_group
end
+ def all_users_group_uuid
+ [Server::Application.config.uuid_prefix,
+ Group.uuid_prefix,
+ 'fffffffffffffff'].join('-')
+ end
+
+ def all_users_group
+ if not $all_users_group
+ act_as_system_user do
+ ActiveRecord::Base.transaction do
+ $system_group = Group.
+ where(uuid: all_users_group_uuid).first_or_create do |g|
+ g.update_attributes(name: "All users",
+ description: "All users")
+ end
+ end
+ end
+ end
+ $all_users_group
+ end
+
def act_as_system_user
if block_given?
act_as_user system_user do
commit e4e47b5f631befe382a48d00122234e248f155ad
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 16 13:33:08 2014 -0500
3824: autorestart non-detached services regardless of exit code.
diff --git a/docker/api/supervisor.conf b/docker/api/supervisor.conf
index e85bb72..e684ff8 100644
--- a/docker/api/supervisor.conf
+++ b/docker/api/supervisor.conf
@@ -6,10 +6,12 @@ startsecs=0
[program:postgres]
user=postgres
command=/usr/lib/postgresql/9.1/bin/postgres -D /var/lib/postgresql/9.1/main -c config_file=/etc/postgresql/9.1/main/postgresql.conf
+autorestart=true
[program:apache2]
command=/etc/apache2/foreground.sh
stopsignal=6
+autorestart=true
[program:munge]
user=root
@@ -39,3 +41,4 @@ startsecs=0
[program:crunch-dispatch]
user=root
command=/usr/local/bin/crunch-dispatch-run.sh
+autorestart=true
commit 3ca79283e89ba51a0ab98d0f84f2efd52f9c3550
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 16 13:31:48 2014 -0500
3824: Enable websockets.
diff --git a/docker/api/apache2_vhost.in b/docker/api/apache2_vhost.in
index fdbb2f9..520b154 100644
--- a/docker/api/apache2_vhost.in
+++ b/docker/api/apache2_vhost.in
@@ -19,6 +19,7 @@
RailsEnv production
RackBaseURI /
RailsAppSpawnerIdleTime 1200
+ SetEnv ARVADOS_WEBSOCKETS true
# Enable streaming
PassengerBufferResponse off
commit 5662a056ff6e3328966272a4a7852d0d3f572739
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 16 13:31:09 2014 -0500
3824: docker issue 3172 is closed, build works without -q.
diff --git a/docker/build_tools/Makefile b/docker/build_tools/Makefile
index f3dd90c..d92349c 100644
--- a/docker/build_tools/Makefile
+++ b/docker/build_tools/Makefile
@@ -148,10 +148,7 @@ $(SSO_GENERATED): $(SSO_GENERATED_IN)
$(KEEP_GENERATED): $(KEEP_GENERATED_IN)
$(CONFIG_RB) keep
-# The docker build -q option suppresses verbose build output.
-# Necessary to prevent failure on building warehouse; see
-# https://github.com/dotcloud/docker/issues/3172
-DOCKER_BUILD = $(DOCKER) build -q --rm=true
+DOCKER_BUILD = $(DOCKER) build --rm=true
# ============================================================
# The main Arvados servers: api, doc, workbench, compute
commit 11374252ee3c26240420bb3aa0d0433fad71731f
Author: Tom Clegg <tom at curoverse.com>
Date: Sun Nov 16 04:16:20 2014 -0500
3824: Run build_script (and create *_WORK) inside the container.
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 081d745..7205382 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -551,7 +551,9 @@ else {
freeze_if_want_freeze ($installpid);
select (undef, undef, undef, 0.1);
}
- Log (undef, "Install script exited ".exit_status_s($?));
+ my $install_exited = $?;
+ Log (undef, "Install script exited ".exit_status_s($install_exited));
+ exit (1) if $install_exited != 0;
}
if (!$have_slurm)
@@ -699,17 +701,10 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
qw(-n1 -c1 -N1 -D), $ENV{'TMPDIR'},
"--job-name=$job_id.$id.$$",
);
- my $build_script_to_send = "";
my $command =
"if [ -e $ENV{TASK_WORK} ]; then rm -rf $ENV{TASK_WORK}; fi; "
."mkdir -p $ENV{CRUNCH_TMP} $ENV{JOB_WORK} $ENV{TASK_WORK} $ENV{TASK_KEEPMOUNT} "
."&& cd $ENV{CRUNCH_TMP} ";
- if ($build_script)
- {
- $build_script_to_send = $build_script;
- $command .=
- "&& perl -";
- }
$command .= "&& exec arv-mount --by-id --allow-other $ENV{TASK_KEEPMOUNT} --exec ";
if ($docker_hash)
{
@@ -738,18 +733,32 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
$command .= "--volume=\Q$ENV{TASK_KEEPMOUNT}:/keep:ro\E ";
$ENV{TASK_KEEPMOUNT} = "/keep";
- # TASK_WORK is a plain docker data volume: it starts out empty,
- # is writable, and persists until no containers use it any
- # more. We don't use --volumes-from to share it with other
- # containers: it is only accessible to this task, and it goes
- # away when this task stops.
- $command .= "--volume=\Q$ENV{TASK_WORK}\E ";
-
- # JOB_WORK is also a plain docker data volume for now. TODO:
- # Share a single JOB_WORK volume across all task containers on a
- # given worker node, and delete it when the job ends (and, in
- # case that doesn't work, when the next job starts).
- $command .= "--volume=\Q$ENV{JOB_WORK}\E ";
+ # TASK_WORK is almost exactly like a docker data volume: it
+ # starts out empty, is writable, and persists until no
+ # containers use it any more. We don't use --volumes-from to
+ # share it with other containers: it is only accessible to this
+ # task, and it goes away when this task stops.
+ #
+ # However, a docker data volume is writable only by root unless
+ # the mount point already happens to exist in the container with
+ # different permissions. Therefore, we [1] assume /tmp already
+ # exists in the image and is writable by the crunch user; [2]
+ # avoid putting TASK_WORK inside CRUNCH_TMP (which won't be
+ # writable if they are created by docker while setting up the
+ # other --volumes); and [3] create $TASK_WORK inside the
+ # container using $build_script.
+ $command .= "--volume=/tmp ";
+ $ENV{"TASK_WORK"} = "/tmp/crunch-job-task-work/$childslotname";
+ $ENV{"HOME"} = $ENV{"TASK_WORK"};
+ $ENV{"TASK_TMPDIR"} = $ENV{"TASK_WORK"}; # deprecated
+
+ # TODO: Share a single JOB_WORK volume across all task
+ # containers on a given worker node, and delete it when the job
+ # ends (and, in case that doesn't work, when the next job
+ # starts).
+ #
+ # For now, use the same approach as TASK_WORK above.
+ $ENV{"JOB_WORK"} = "/tmp/crunch-job-work";
while (my ($env_key, $env_val) = each %ENV)
{
@@ -760,16 +769,16 @@ for (my $todo_ptr = 0; $todo_ptr <= $#jobstep_todo; $todo_ptr ++)
$command .= "--env=\QHOME=$ENV{HOME}\E ";
$command .= "\Q$docker_hash\E ";
$command .= "stdbuf --output=0 --error=0 ";
- $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+ $command .= "perl - $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
} else {
# Non-docker run
$command .= "crunchstat -cgroup-root=/sys/fs/cgroup -poll=10000 ";
$command .= "stdbuf --output=0 --error=0 ";
- $command .= "$ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
+ $command .= "perl - $ENV{CRUNCH_SRC}/crunch_scripts/" . $Job->{"script"};
}
my @execargs = ('bash', '-c', $command);
- srun (\@srunargs, \@execargs, undef, $build_script_to_send);
+ srun (\@srunargs, \@execargs, undef, $build_script);
# exec() failed, we assume nothing happened.
die "srun() failed on build script\n";
}
@@ -1730,9 +1739,10 @@ use File::Path qw( make_path remove_tree );
my $destdir = $ENV{"CRUNCH_SRC"};
my $commit = $ENV{"CRUNCH_SRC_COMMIT"};
my $repo = $ENV{"CRUNCH_SRC_URL"};
+my $job_work = $ENV{"JOB_WORK"};
my $task_work = $ENV{"TASK_WORK"};
-for my $dir ($destdir, $task_work) {
+for my $dir ($destdir, $job_work, $task_work) {
if ($dir) {
make_path $dir;
-e $dir or die "Failed to create temporary directory ($dir): $!";
@@ -1747,12 +1757,8 @@ if ($task_work) {
open L, ">", "$destdir.lock" or die "$destdir.lock: $!";
flock L, LOCK_EX;
if (readlink ("$destdir.commit") eq $commit && -d $destdir) {
- if (@ARGV) {
- exec(@ARGV);
- die "Cannot exec `@ARGV`: $!";
- } else {
- exit 0;
- }
+ # This version already installed -> nothing to do.
+ run_argv_and_exit();
}
unlink "$destdir.commit";
@@ -1762,12 +1768,15 @@ open STDERR, ">&STDOUT";
mkdir $destdir;
my @git_archive_data = <DATA>;
-if (@git_archive_data) {
- open TARX, "|-", "tar", "-C", $destdir, "-xf", "-";
- print TARX @git_archive_data;
- if(!close(TARX)) {
- die "'tar -C $destdir -xf -' exited $?: $!";
- }
+if (!@git_archive_data) {
+ # Nothing to extract -> nothing to install.
+ run_argv_and_exit();
+}
+
+open TARX, "|-", "tar", "-C", $destdir, "-xf", "-";
+print TARX @git_archive_data;
+if(!close(TARX)) {
+ die "'tar -C $destdir -xf -' exited $?: $!";
}
my $pwd;
@@ -1799,11 +1808,16 @@ if ($commit) {
close L;
-if (@ARGV) {
+run_argv_and_exit();
+
+sub run_argv_and_exit
+{
+ if (@ARGV) {
exec(@ARGV);
die "Cannot exec `@ARGV`: $!";
-} else {
+ } else {
exit 0;
+ }
}
sub shell_or_die
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list