[ARVADOS] updated: 8a274345e3279b561d17d546d4f3581862811125
git at public.curoverse.com
git at public.curoverse.com
Fri Mar 14 17:19:28 EDT 2014
Summary of changes:
docker/api/Dockerfile | 1 +
docker/build.rb | 185 ------------------------------
docker/build.sh | 2 +-
docker/{ => build_tools}/Makefile | 12 +-
docker/build_tools/build.rb | 221 ++++++++++++++++++++++++++++++++++++
docker/{ => build_tools}/config.rb | 0
docker/docker_build | 18 ---
sdk/cli/bin/crunch-job | 8 +-
8 files changed, 236 insertions(+), 211 deletions(-)
delete mode 100755 docker/build.rb
rename docker/{ => build_tools}/Makefile (97%)
create mode 100755 docker/build_tools/build.rb
rename docker/{ => build_tools}/config.rb (100%)
delete mode 100755 docker/docker_build
via 8a274345e3279b561d17d546d4f3581862811125 (commit)
via cf8b27749cf44aee74914622dc8bc2a9690204dc (commit)
from 3690225e29161191cc5daabe4a335a0ec5737d3d (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 8a274345e3279b561d17d546d4f3581862811125
Author: Tim Pierce <twp at curoverse.com>
Date: Fri Mar 14 17:20:37 2014 -0400
Reorganize build tools to be less confusing.
diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index 6f3b792..0ecd7d0 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -37,6 +37,7 @@ RUN bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
./script/create_superuser_token.rb $(cat /tmp/superuser_token) && \
chown www-data:www-data config.ru && \
chown www-data:www-data log -R && \
+ mkdir tmp && \
chown www-data:www-data tmp -R
# Configure Apache and Passenger.
diff --git a/docker/build.rb b/docker/build.rb
deleted file mode 100755
index ee2cc78..0000000
--- a/docker/build.rb
+++ /dev/null
@@ -1,185 +0,0 @@
-#! /usr/bin/env ruby
-
-require 'tempfile'
-require 'yaml'
-
-def sudo(*cmd)
- # user can pass a single list in as an argument
- # to allow usage like: sudo %w(apt-get install foo)
- warn "You may need to enter your password here."
- if cmd.length == 1 and cmd[0].class == Array
- cmd = cmd[0]
- end
- system '/usr/bin/sudo', *cmd
-end
-
-def is_valid_email? str
- str.match /^\S+@\S+\.\S+$/
-end
-
-def generate_api_hostname
- rand(2**256).to_s(36)[0...5]
-end
-
-# ip_forwarding_enabled?
-# Returns 'true' if IP forwarding is enabled in the kernel
-#
-def ip_forwarding_enabled?
- %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n"
-end
-
-def debootstrap_ok?
- return system '/usr/sbin/debootstrap --version > /dev/null 2>&1'
-end
-
-def docker_ok?
- return system 'docker images > /dev/null 2>&1'
-end
-
-# find_or_create_ssh_key arvados_name
-# Return the SSH public key appropriate for this Arvados instance,
-# generating one if necessary.
-#
-def find_or_create_ssh_key arvados_name
- ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
- unless File.exists? ssh_key_file
- system 'ssh-keygen',
- '-f', ssh_key_file,
- '-C', "arvados@#{arvados_name}",
- '-P', ''
- end
-
- return "#{ssh_key_file}.pub"
-end
-
-if not ip_forwarding_enabled?
- warn "NOTE: IP forwarding must be enabled in the kernel."
- warn "Turning IP forwarding on now."
- sudo %w(/sbin/sysctl net.ipv4.ip_forward=1)
-end
-
-# Check that:
-# * Docker is installed and can be found in the user's path
-# * Docker can be run as a non-root user
-# - TODO: put the user is in the docker group if necessary
-# - TODO: mount cgroup automatically
-# - TODO: start the docker service if not started
-
-docker_path = %x(which docker).chomp
-if docker_path.empty?
- warn "Docker not found."
- warn ""
- warn "Please make sure that Docker has been installed and"
- warn "can be found in your PATH."
- warn ""
- warn "Installation instructions for a variety of platforms can be found at"
- warn "http://docs.docker.io/en/latest/installation/"
- exit
-elsif not docker_ok?
- warn "WARNING: docker could not be run."
- warn "Please make sure that:"
- warn " * You have permission to read and write /var/run/docker.sock"
- warn " * a 'cgroup' volume is mounted on your machine"
- warn " * the docker daemon is running"
- exit
-end
-
-# Check that debootstrap is installed.
-if not debootstrap_ok?
- warn "Installing debootstrap."
- sudo '/usr/bin/apt-get', 'install', 'debootstrap'
-end
-
-# Generate a config.yml if it does not exist or is empty
-if not File.size? 'config.yml'
- print "Generating config.yml.\n"
- print "Arvados needs to know the email address of the administrative user,\n"
- print "so that when that user logs in they are automatically made an admin.\n"
- print "This should be the email address you use to log in to Google.\n"
- print "\n"
- admin_email_address = ""
- until is_valid_email? admin_email_address
- print "Enter your Google ID email address here: "
- admin_email_address = gets.strip
- if not is_valid_email? admin_email_address
- print "That doesn't look like a valid email address. Please try again.\n"
- end
- end
-
- File.open 'config.yml', 'w' do |config_out|
- config = YAML.load_file 'config.yml.example'
- config['API_AUTO_ADMIN_USER'] = admin_email_address
- config['API_HOSTNAME'] = generate_api_hostname
- config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
- config.each_key do |var|
- if var.end_with?('_PW') or var.end_with?('_SECRET')
- config[var] = rand(2**256).to_s(36)
- end
- config_out.write "#{var}: #{config[var]}\n"
- end
- end
-end
-
-# If all prerequisites are met, go ahead and build.
-if ip_forwarding_enabled? and
- docker_ok? and
- debootstrap_ok? and
- File.exists? 'config.yml'
- warn "Building Arvados."
- system '/usr/bin/make', *ARGV
-end
-
-# install_docker
-# Determine which Docker package is suitable for this Linux distro
-# and install, resolving any dependencies.
-# NOTE: not in use yet.
-
-def install_docker
- linux_distro = %x(lsb_release --id).split.last
- linux_release = %x(lsb_release --release).split.last
- linux_version = linux_distro + " " + linux_release
- kernel_release = `uname -r`
-
- case linux_distro
- when 'Ubuntu'
- if not linux_release.match '^1[234]\.'
- warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)."
- warn "Your system is Ubuntu #{linux_release}."
- exit
- end
- if linux_release.match '^12' and kernel_release.start_with? '3.2'
- # Ubuntu Precise ships with a 3.2 kernel and must be upgraded.
- warn "Your kernel #{kernel_release} must be upgraded to run Docker."
- warn "To do this:"
- warn " sudo apt-get update"
- warn " sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
- warn " sudo reboot"
- exit
- else
- # install AUFS
- sudo 'apt-get', 'update'
- sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}"
- end
-
- # add Docker repository
- sudo %w(/usr/bin/apt-key adv
- --keyserver keyserver.ubuntu.com
- --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9)
- source_file = Tempfile.new('arv')
- source_file.write("deb http://get.docker.io/ubuntu docker main\n")
- source_file.close
- sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list'
- sudo %w(/usr/bin/apt-get update)
- sudo %w(/usr/bin/apt-get install lxc-docker)
-
- # Set up for non-root access
- sudo %w(/usr/sbin/groupadd docker)
- sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'
- sudo %w(/usr/sbin/service docker restart)
- when 'Debian'
- else
- warn "Must be running a Debian or Ubuntu release in order to run Docker."
- exit
- end
-end
-
diff --git a/docker/build.sh b/docker/build.sh
index 0e3d4cb..6b2cc6b 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -11,4 +11,4 @@ then
sudo apt-get -y install ruby1.9.3
fi
-./build.rb $*
+build_tools/build.rb $*
diff --git a/docker/Makefile b/docker/build_tools/Makefile
similarity index 97%
rename from docker/Makefile
rename to docker/build_tools/Makefile
index d949db0..7fdad91 100644
--- a/docker/Makefile
+++ b/docker/build_tools/Makefile
@@ -17,6 +17,8 @@ realclean: clean
# Dependencies for */generated files which are prerequisites
# for building docker images.
+CONFIG_RB = build_tools/config.rb
+
BASE_DEPS = base/Dockerfile $(BASE_GENERATED)
API_DEPS = api/Dockerfile $(API_GENERATED)
@@ -78,23 +80,23 @@ SSO_GENERATED_IN = \
sso/secret_token.rb.in
$(BASE_GENERATED): config.yml
- ./config.rb
+ $(CONFIG_RB)
mkdir -p base/generated
tar -c -z -f base/generated/arvados.tar.gz -C .. . \
--exclude=services/api/log/* --exclude=docker/*
$(API_GENERATED): config.yml $(API_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(WORKBENCH_GENERATED): config.yml $(WORKBENCH_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(WAREHOUSE_GENERATED): config.yml $(WAREHOUSE_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
$(SSO_GENERATED): config.yml $(SSO_GENERATED_IN)
- ./config.rb
+ $(CONFIG_RB)
# The docker build -q option suppresses verbose build output.
# Necessary to prevent failure on building warehouse; see
diff --git a/docker/build_tools/build.rb b/docker/build_tools/build.rb
new file mode 100755
index 0000000..65f44ad
--- /dev/null
+++ b/docker/build_tools/build.rb
@@ -0,0 +1,221 @@
+#! /usr/bin/env ruby
+
+require 'optparse'
+require 'tempfile'
+require 'yaml'
+
+def main options
+ if not ip_forwarding_enabled?
+ warn "NOTE: IP forwarding must be enabled in the kernel."
+ warn "Turning IP forwarding on now."
+ sudo %w(/sbin/sysctl net.ipv4.ip_forward=1)
+ end
+
+ # Check that:
+ # * Docker is installed and can be found in the user's path
+ # * Docker can be run as a non-root user
+ # - TODO: put the user is in the docker group if necessary
+ # - TODO: mount cgroup automatically
+ # - TODO: start the docker service if not started
+
+ docker_path = %x(which docker).chomp
+ if docker_path.empty?
+ warn "Docker not found."
+ warn ""
+ warn "Please make sure that Docker has been installed and"
+ warn "can be found in your PATH."
+ warn ""
+ warn "Installation instructions for a variety of platforms can be found at"
+ warn "http://docs.docker.io/en/latest/installation/"
+ exit
+ elsif not docker_ok?
+ warn "WARNING: docker could not be run."
+ warn "Please make sure that:"
+ warn " * You have permission to read and write /var/run/docker.sock"
+ warn " * a 'cgroup' volume is mounted on your machine"
+ warn " * the docker daemon is running"
+ exit
+ end
+
+ # Check that debootstrap is installed.
+ if not debootstrap_ok?
+ warn "Installing debootstrap."
+ sudo '/usr/bin/apt-get', 'install', 'debootstrap'
+ end
+
+ # Generate a config.yml if it does not exist or is empty
+ if not File.size? 'config.yml'
+ print "Generating config.yml.\n"
+ print "Arvados needs to know the email address of the administrative user,\n"
+ print "so that when that user logs in they are automatically made an admin.\n"
+ print "This should be the email address you use to log in to Google.\n"
+ print "\n"
+ admin_email_address = ""
+ until is_valid_email? admin_email_address
+ print "Enter your Google ID email address here: "
+ admin_email_address = gets.strip
+ if not is_valid_email? admin_email_address
+ print "That doesn't look like a valid email address. Please try again.\n"
+ end
+ end
+
+ File.open 'config.yml', 'w' do |config_out|
+ config = YAML.load_file 'config.yml.example'
+ config['API_AUTO_ADMIN_USER'] = admin_email_address
+ config['API_HOSTNAME'] = generate_api_hostname
+ config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
+ config.each_key do |var|
+ if var.end_with?('_PW') or var.end_with?('_SECRET')
+ config[var] = rand(2**256).to_s(36)
+ end
+ config_out.write "#{var}: #{config[var]}\n"
+ end
+ end
+ end
+
+ # If all prerequisites are met, go ahead and build.
+ if ip_forwarding_enabled? and
+ docker_ok? and
+ debootstrap_ok? and
+ File.exists? 'config.yml'
+ warn "Building Arvados."
+ system '/usr/bin/make', '-f', options[:makefile], *ARGV
+ end
+end
+
+# sudo
+# Execute the arg list 'cmd' under sudo.
+# cmd can be passed either as a series of arguments or as a
+# single argument consisting of a list, e.g.:
+# sudo 'apt-get', 'update'
+# sudo(['/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'])
+# sudo %w(/usr/bin/apt-get install lxc-docker)
+#
+def sudo(*cmd)
+ # user can pass a single list in as an argument
+ # to allow usage like: sudo %w(apt-get install foo)
+ warn "You may need to enter your password here."
+ if cmd.length == 1 and cmd[0].class == Array
+ cmd = cmd[0]
+ end
+ system '/usr/bin/sudo', *cmd
+end
+
+# is_valid_email?
+# Returns true if its arg looks like a valid email address.
+# This is a very very loose sanity check.
+#
+def is_valid_email? str
+ str.match /^\S+@\S+\.\S+$/
+end
+
+# generate_api_hostname
+# Generates a 5-character randomly chosen API hostname.
+#
+def generate_api_hostname
+ rand(2**256).to_s(36)[0...5]
+end
+
+# ip_forwarding_enabled?
+# Returns 'true' if IP forwarding is enabled in the kernel
+#
+def ip_forwarding_enabled?
+ %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n"
+end
+
+# debootstrap_ok?
+# Returns 'true' if debootstrap is installed and working.
+#
+def debootstrap_ok?
+ return system '/usr/sbin/debootstrap --version > /dev/null 2>&1'
+end
+
+# docker_ok?
+# Returns 'true' if docker can be run as the current user.
+#
+def docker_ok?
+ return system 'docker images > /dev/null 2>&1'
+end
+
+# find_or_create_ssh_key arvados_name
+# Returns the SSH public key appropriate for this Arvados instance,
+# generating one if necessary.
+#
+def find_or_create_ssh_key arvados_name
+ ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
+ unless File.exists? ssh_key_file
+ system 'ssh-keygen',
+ '-f', ssh_key_file,
+ '-C', "arvados@#{arvados_name}",
+ '-P', ''
+ end
+
+ return "#{ssh_key_file}.pub"
+end
+
+# install_docker
+# Determines which Docker package is suitable for this Linux distro
+# and installs it, resolving any dependencies.
+# NOTE: not in use yet.
+
+def install_docker
+ linux_distro = %x(lsb_release --id).split.last
+ linux_release = %x(lsb_release --release).split.last
+ linux_version = linux_distro + " " + linux_release
+ kernel_release = `uname -r`
+
+ case linux_distro
+ when 'Ubuntu'
+ if not linux_release.match '^1[234]\.'
+ warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)."
+ warn "Your system is Ubuntu #{linux_release}."
+ exit
+ end
+ if linux_release.match '^12' and kernel_release.start_with? '3.2'
+ # Ubuntu Precise ships with a 3.2 kernel and must be upgraded.
+ warn "Your kernel #{kernel_release} must be upgraded to run Docker."
+ warn "To do this:"
+ warn " sudo apt-get update"
+ warn " sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
+ warn " sudo reboot"
+ exit
+ else
+ # install AUFS
+ sudo 'apt-get', 'update'
+ sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}"
+ end
+
+ # add Docker repository
+ sudo %w(/usr/bin/apt-key adv
+ --keyserver keyserver.ubuntu.com
+ --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9)
+ source_file = Tempfile.new('arv')
+ source_file.write("deb http://get.docker.io/ubuntu docker main\n")
+ source_file.close
+ sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list'
+ sudo %w(/usr/bin/apt-get update)
+ sudo %w(/usr/bin/apt-get install lxc-docker)
+
+ # Set up for non-root access
+ sudo %w(/usr/sbin/groupadd docker)
+ sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'
+ sudo %w(/usr/sbin/service docker restart)
+ when 'Debian'
+ else
+ warn "Must be running a Debian or Ubuntu release in order to run Docker."
+ exit
+ end
+end
+
+
+if __FILE__ == $PROGRAM_NAME
+ options = { :makefile => Dir.pwd + "/build_tools/Makefile" }
+ OptionParser.new do |opts|
+ opts.on('-m', '--makefile MAKEFILE-PATH',
+ 'Path to the Makefile used to build Arvados Docker images') do |mk|
+ options[:makefile] = mk
+ end
+ end
+
+ main options
+end
diff --git a/docker/config.rb b/docker/build_tools/config.rb
similarity index 100%
rename from docker/config.rb
rename to docker/build_tools/config.rb
diff --git a/docker/docker_build b/docker/docker_build
deleted file mode 100755
index 0c0fd18..0000000
--- a/docker/docker_build
+++ /dev/null
@@ -1,18 +0,0 @@
-#! /bin/bash
-
-# Wrapper script for `docker build'.
-# This is a workaround for https://github.com/dotcloud/docker/issues/1875.
-
-tmpfile=$(mktemp)
-trap "rm $tmpfile; exit 1" SIGHUP SIGINT SIGTERM
-
-docker build $* | tee ${tmpfile}
-if $(grep -q 'Error build' ${tmpfile})
-then
- result=1
-else
- result=0
-fi
-
-rm $tmpfile
-exit $result
commit cf8b27749cf44aee74914622dc8bc2a9690204dc
Author: Tim Pierce <twp at curoverse.com>
Date: Fri Mar 14 17:19:33 2014 -0400
Incorporate code review comments (refs #2221, fixes #2325)
diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 07b4943..2b31531 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1071,7 +1071,7 @@ sub collate_output
}
else
{
- print $child_in "XXX fetch_block($output) failed XXX\n";
+ Log (undef, "XXX fetch_block($output) failed XXX");
$main::success = 0;
}
}
@@ -1079,7 +1079,11 @@ sub collate_output
if (!defined $joboutput) {
my $s = IO::Select->new($child_out);
- sysread($child_out, $joboutput, 64 * 1024 * 1024) if $s->can_read(5);
+ if ($s->can_read(120)) {
+ sysread($child_out, $joboutput, 64 * 1024 * 1024);
+ } else {
+ Log (undef, "timed out reading from 'arv keep put'");
+ }
}
waitpid($pid, 0);
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list