[ARVADOS] updated: 8a274345e3279b561d17d546d4f3581862811125

git at public.curoverse.com git at public.curoverse.com
Fri Mar 14 17:19:28 EDT 2014


Summary of changes:
 docker/api/Dockerfile              |    1 +
 docker/build.rb                    |  185 ------------------------------
 docker/build.sh                    |    2 +-
 docker/{ => build_tools}/Makefile  |   12 +-
 docker/build_tools/build.rb        |  221 ++++++++++++++++++++++++++++++++++++
 docker/{ => build_tools}/config.rb |    0
 docker/docker_build                |   18 ---
 sdk/cli/bin/crunch-job             |    8 +-
 8 files changed, 236 insertions(+), 211 deletions(-)
 delete mode 100755 docker/build.rb
 rename docker/{ => build_tools}/Makefile (97%)
 create mode 100755 docker/build_tools/build.rb
 rename docker/{ => build_tools}/config.rb (100%)
 delete mode 100755 docker/docker_build

       via  8a274345e3279b561d17d546d4f3581862811125 (commit)
       via  cf8b27749cf44aee74914622dc8bc2a9690204dc (commit)
      from  3690225e29161191cc5daabe4a335a0ec5737d3d (commit)

Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.


commit 8a274345e3279b561d17d546d4f3581862811125
Author: Tim Pierce <twp at curoverse.com>
Date:   Fri Mar 14 17:20:37 2014 -0400

    Reorganize build tools to be less confusing.

diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile
index 6f3b792..0ecd7d0 100644
--- a/docker/api/Dockerfile
+++ b/docker/api/Dockerfile
@@ -37,6 +37,7 @@ RUN bundle install --gemfile=/usr/src/arvados/services/api/Gemfile && \
     ./script/create_superuser_token.rb $(cat /tmp/superuser_token) && \
     chown www-data:www-data config.ru && \
     chown www-data:www-data log -R && \
+    mkdir tmp && \
     chown www-data:www-data tmp -R
 
 # Configure Apache and Passenger.
diff --git a/docker/build.rb b/docker/build.rb
deleted file mode 100755
index ee2cc78..0000000
--- a/docker/build.rb
+++ /dev/null
@@ -1,185 +0,0 @@
-#! /usr/bin/env ruby
-
-require 'tempfile'
-require 'yaml'
-
-def sudo(*cmd)
-  # user can pass a single list in as an argument
-  # to allow usage like: sudo %w(apt-get install foo)
-  warn "You may need to enter your password here."
-  if cmd.length == 1 and cmd[0].class == Array
-    cmd = cmd[0]
-  end
-  system '/usr/bin/sudo', *cmd
-end
-
-def is_valid_email? str
-  str.match /^\S+@\S+\.\S+$/
-end
-
-def generate_api_hostname
-  rand(2**256).to_s(36)[0...5]
-end
-
-# ip_forwarding_enabled?
-#   Returns 'true' if IP forwarding is enabled in the kernel
-#
-def ip_forwarding_enabled?
-  %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n"
-end
-
-def debootstrap_ok?
-  return system '/usr/sbin/debootstrap --version > /dev/null 2>&1'
-end
-
-def docker_ok?
-  return system 'docker images > /dev/null 2>&1'
-end
-
-# find_or_create_ssh_key arvados_name
-#   Return the SSH public key appropriate for this Arvados instance,
-#   generating one if necessary.
-#
-def find_or_create_ssh_key arvados_name
-  ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
-  unless File.exists? ssh_key_file
-    system 'ssh-keygen',
-           '-f', ssh_key_file,
-           '-C', "arvados@#{arvados_name}",
-           '-P', ''
-  end
-
-  return "#{ssh_key_file}.pub"
-end
-
-if not ip_forwarding_enabled?
-  warn "NOTE: IP forwarding must be enabled in the kernel."
-  warn "Turning IP forwarding on now."
-  sudo %w(/sbin/sysctl net.ipv4.ip_forward=1)
-end
-
-# Check that:
-#   * Docker is installed and can be found in the user's path
-#   * Docker can be run as a non-root user
-#      - TODO: put the user is in the docker group if necessary
-#      - TODO: mount cgroup automatically
-#      - TODO: start the docker service if not started
-
-docker_path = %x(which docker).chomp
-if docker_path.empty?
-  warn "Docker not found."
-  warn ""
-  warn "Please make sure that Docker has been installed and"
-  warn "can be found in your PATH."
-  warn ""
-  warn "Installation instructions for a variety of platforms can be found at"
-  warn "http://docs.docker.io/en/latest/installation/"
-  exit
-elsif not docker_ok?
-  warn "WARNING: docker could not be run."
-  warn "Please make sure that:"
-  warn "  * You have permission to read and write /var/run/docker.sock"
-  warn "  * a 'cgroup' volume is mounted on your machine"
-  warn "  * the docker daemon is running"
-  exit
-end
-
-# Check that debootstrap is installed.
-if not debootstrap_ok?
-  warn "Installing debootstrap."
-  sudo '/usr/bin/apt-get', 'install', 'debootstrap'
-end
-
-# Generate a config.yml if it does not exist or is empty
-if not File.size? 'config.yml'
-  print "Generating config.yml.\n"
-  print "Arvados needs to know the email address of the administrative user,\n"
-  print "so that when that user logs in they are automatically made an admin.\n"
-  print "This should be the email address you use to log in to Google.\n"
-  print "\n"
-  admin_email_address = ""
-  until is_valid_email? admin_email_address
-    print "Enter your Google ID email address here: "
-    admin_email_address = gets.strip
-    if not is_valid_email? admin_email_address
-      print "That doesn't look like a valid email address. Please try again.\n"
-    end
-  end
-
-  File.open 'config.yml', 'w' do |config_out|
-    config = YAML.load_file 'config.yml.example'
-    config['API_AUTO_ADMIN_USER'] = admin_email_address
-    config['API_HOSTNAME'] = generate_api_hostname
-    config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
-    config.each_key do |var|
-      if var.end_with?('_PW') or var.end_with?('_SECRET')
-        config[var] = rand(2**256).to_s(36)
-      end
-      config_out.write "#{var}: #{config[var]}\n"
-    end
-  end
-end
-
-# If all prerequisites are met, go ahead and build.
-if ip_forwarding_enabled? and
-    docker_ok? and
-    debootstrap_ok? and
-    File.exists? 'config.yml'
-  warn "Building Arvados."
-  system '/usr/bin/make', *ARGV
-end
-
-# install_docker
-#   Determine which Docker package is suitable for this Linux distro
-#   and install, resolving any dependencies.
-#   NOTE: not in use yet.
-
-def install_docker
-  linux_distro = %x(lsb_release --id).split.last
-  linux_release = %x(lsb_release --release).split.last
-  linux_version = linux_distro + " " + linux_release
-  kernel_release = `uname -r`
-  
-  case linux_distro
-  when 'Ubuntu'
-    if not linux_release.match '^1[234]\.'
-      warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)."
-      warn "Your system is Ubuntu #{linux_release}."
-      exit
-    end
-    if linux_release.match '^12' and kernel_release.start_with? '3.2'
-      # Ubuntu Precise ships with a 3.2 kernel and must be upgraded.
-      warn "Your kernel #{kernel_release} must be upgraded to run Docker."
-      warn "To do this:"
-      warn "  sudo apt-get update"
-      warn "  sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
-      warn "  sudo reboot"
-      exit
-    else
-      # install AUFS
-      sudo 'apt-get', 'update'
-      sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}"
-    end
-
-    # add Docker repository
-    sudo %w(/usr/bin/apt-key adv
-              --keyserver keyserver.ubuntu.com
-              --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9)
-    source_file = Tempfile.new('arv')
-    source_file.write("deb http://get.docker.io/ubuntu docker main\n")
-    source_file.close
-    sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list'
-    sudo %w(/usr/bin/apt-get update)
-    sudo %w(/usr/bin/apt-get install lxc-docker)
-
-    # Set up for non-root access
-    sudo %w(/usr/sbin/groupadd docker)
-    sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'
-    sudo %w(/usr/sbin/service docker restart)
-  when 'Debian'
-  else
-    warn "Must be running a Debian or Ubuntu release in order to run Docker."
-    exit
-  end
-end
-
diff --git a/docker/build.sh b/docker/build.sh
index 0e3d4cb..6b2cc6b 100755
--- a/docker/build.sh
+++ b/docker/build.sh
@@ -11,4 +11,4 @@ then
     sudo apt-get -y install ruby1.9.3
 fi
 
-./build.rb $*
+build_tools/build.rb $*
diff --git a/docker/Makefile b/docker/build_tools/Makefile
similarity index 97%
rename from docker/Makefile
rename to docker/build_tools/Makefile
index d949db0..7fdad91 100644
--- a/docker/Makefile
+++ b/docker/build_tools/Makefile
@@ -17,6 +17,8 @@ realclean: clean
 # Dependencies for */generated files which are prerequisites
 # for building docker images.
 
+CONFIG_RB = build_tools/config.rb
+
 BASE_DEPS = base/Dockerfile $(BASE_GENERATED)
 
 API_DEPS = api/Dockerfile $(API_GENERATED)
@@ -78,23 +80,23 @@ SSO_GENERATED_IN = \
         sso/secret_token.rb.in
 
 $(BASE_GENERATED): config.yml
-	./config.rb
+	$(CONFIG_RB)
 	mkdir -p base/generated
 	tar -c -z -f base/generated/arvados.tar.gz -C .. . \
 	  --exclude=services/api/log/* --exclude=docker/*
 
 
 $(API_GENERATED): config.yml $(API_GENERATED_IN)
-	./config.rb
+	$(CONFIG_RB)
 
 $(WORKBENCH_GENERATED): config.yml $(WORKBENCH_GENERATED_IN)
-	./config.rb
+	$(CONFIG_RB)
 
 $(WAREHOUSE_GENERATED): config.yml $(WAREHOUSE_GENERATED_IN)
-	./config.rb
+	$(CONFIG_RB)
 
 $(SSO_GENERATED): config.yml $(SSO_GENERATED_IN)
-	./config.rb
+	$(CONFIG_RB)
 
 # The docker build -q option suppresses verbose build output.
 # Necessary to prevent failure on building warehouse; see
diff --git a/docker/build_tools/build.rb b/docker/build_tools/build.rb
new file mode 100755
index 0000000..65f44ad
--- /dev/null
+++ b/docker/build_tools/build.rb
@@ -0,0 +1,221 @@
+#! /usr/bin/env ruby
+
+require 'optparse'
+require 'tempfile'
+require 'yaml'
+
+def main options
+  if not ip_forwarding_enabled?
+    warn "NOTE: IP forwarding must be enabled in the kernel."
+    warn "Turning IP forwarding on now."
+    sudo %w(/sbin/sysctl net.ipv4.ip_forward=1)
+  end
+
+  # Check that:
+  #   * Docker is installed and can be found in the user's path
+  #   * Docker can be run as a non-root user
+  #      - TODO: put the user is in the docker group if necessary
+  #      - TODO: mount cgroup automatically
+  #      - TODO: start the docker service if not started
+
+  docker_path = %x(which docker).chomp
+  if docker_path.empty?
+    warn "Docker not found."
+    warn ""
+    warn "Please make sure that Docker has been installed and"
+    warn "can be found in your PATH."
+    warn ""
+    warn "Installation instructions for a variety of platforms can be found at"
+    warn "http://docs.docker.io/en/latest/installation/"
+    exit
+  elsif not docker_ok?
+    warn "WARNING: docker could not be run."
+    warn "Please make sure that:"
+    warn "  * You have permission to read and write /var/run/docker.sock"
+    warn "  * a 'cgroup' volume is mounted on your machine"
+    warn "  * the docker daemon is running"
+    exit
+  end
+
+  # Check that debootstrap is installed.
+  if not debootstrap_ok?
+    warn "Installing debootstrap."
+    sudo '/usr/bin/apt-get', 'install', 'debootstrap'
+  end
+
+  # Generate a config.yml if it does not exist or is empty
+  if not File.size? 'config.yml'
+    print "Generating config.yml.\n"
+    print "Arvados needs to know the email address of the administrative user,\n"
+    print "so that when that user logs in they are automatically made an admin.\n"
+    print "This should be the email address you use to log in to Google.\n"
+    print "\n"
+    admin_email_address = ""
+    until is_valid_email? admin_email_address
+      print "Enter your Google ID email address here: "
+      admin_email_address = gets.strip
+      if not is_valid_email? admin_email_address
+        print "That doesn't look like a valid email address. Please try again.\n"
+      end
+    end
+
+    File.open 'config.yml', 'w' do |config_out|
+      config = YAML.load_file 'config.yml.example'
+      config['API_AUTO_ADMIN_USER'] = admin_email_address
+      config['API_HOSTNAME'] = generate_api_hostname
+      config['PUBLIC_KEY_PATH'] = find_or_create_ssh_key(config['API_HOSTNAME'])
+      config.each_key do |var|
+        if var.end_with?('_PW') or var.end_with?('_SECRET')
+          config[var] = rand(2**256).to_s(36)
+        end
+        config_out.write "#{var}: #{config[var]}\n"
+      end
+    end
+  end
+
+  # If all prerequisites are met, go ahead and build.
+  if ip_forwarding_enabled? and
+      docker_ok? and
+      debootstrap_ok? and
+      File.exists? 'config.yml'
+    warn "Building Arvados."
+    system '/usr/bin/make', '-f', options[:makefile], *ARGV
+  end
+end
+
+# sudo
+#   Execute the arg list 'cmd' under sudo.
+#   cmd can be passed either as a series of arguments or as a
+#   single argument consisting of a list, e.g.:
+#     sudo 'apt-get', 'update'
+#     sudo(['/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'])
+#     sudo %w(/usr/bin/apt-get install lxc-docker)
+#
+def sudo(*cmd)
+  # user can pass a single list in as an argument
+  # to allow usage like: sudo %w(apt-get install foo)
+  warn "You may need to enter your password here."
+  if cmd.length == 1 and cmd[0].class == Array
+    cmd = cmd[0]
+  end
+  system '/usr/bin/sudo', *cmd
+end
+
+# is_valid_email?
+#   Returns true if its arg looks like a valid email address.
+#   This is a very very loose sanity check.
+#
+def is_valid_email? str
+  str.match /^\S+@\S+\.\S+$/
+end
+
+# generate_api_hostname
+#   Generates a 5-character randomly chosen API hostname.
+#
+def generate_api_hostname
+  rand(2**256).to_s(36)[0...5]
+end
+
+# ip_forwarding_enabled?
+#   Returns 'true' if IP forwarding is enabled in the kernel
+#
+def ip_forwarding_enabled?
+  %x(/sbin/sysctl -n net.ipv4.ip_forward) == "1\n"
+end
+
+# debootstrap_ok?
+#   Returns 'true' if debootstrap is installed and working.
+#
+def debootstrap_ok?
+  return system '/usr/sbin/debootstrap --version > /dev/null 2>&1'
+end
+
+# docker_ok?
+#   Returns 'true' if docker can be run as the current user.
+#
+def docker_ok?
+  return system 'docker images > /dev/null 2>&1'
+end
+
+# find_or_create_ssh_key arvados_name
+#   Returns the SSH public key appropriate for this Arvados instance,
+#   generating one if necessary.
+#
+def find_or_create_ssh_key arvados_name
+  ssh_key_file = "#{ENV['HOME']}/.ssh/arvados_#{arvados_name}_id_rsa"
+  unless File.exists? ssh_key_file
+    system 'ssh-keygen',
+           '-f', ssh_key_file,
+           '-C', "arvados@#{arvados_name}",
+           '-P', ''
+  end
+
+  return "#{ssh_key_file}.pub"
+end
+
+# install_docker
+#   Determines which Docker package is suitable for this Linux distro
+#   and installs it, resolving any dependencies.
+#   NOTE: not in use yet.
+
+def install_docker
+  linux_distro = %x(lsb_release --id).split.last
+  linux_release = %x(lsb_release --release).split.last
+  linux_version = linux_distro + " " + linux_release
+  kernel_release = `uname -r`
+  
+  case linux_distro
+  when 'Ubuntu'
+    if not linux_release.match '^1[234]\.'
+      warn "Arvados requires at least Ubuntu 12.04 (Precise Pangolin)."
+      warn "Your system is Ubuntu #{linux_release}."
+      exit
+    end
+    if linux_release.match '^12' and kernel_release.start_with? '3.2'
+      # Ubuntu Precise ships with a 3.2 kernel and must be upgraded.
+      warn "Your kernel #{kernel_release} must be upgraded to run Docker."
+      warn "To do this:"
+      warn "  sudo apt-get update"
+      warn "  sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring"
+      warn "  sudo reboot"
+      exit
+    else
+      # install AUFS
+      sudo 'apt-get', 'update'
+      sudo 'apt-get', 'install', "linux-image-extra-#{kernel_release}"
+    end
+
+    # add Docker repository
+    sudo %w(/usr/bin/apt-key adv
+              --keyserver keyserver.ubuntu.com
+              --recv-keys 36A1D7869245C8950F966E92D8576A8BA88D21E9)
+    source_file = Tempfile.new('arv')
+    source_file.write("deb http://get.docker.io/ubuntu docker main\n")
+    source_file.close
+    sudo '/bin/mv', source_file.path, '/etc/apt/sources.list.d/docker.list'
+    sudo %w(/usr/bin/apt-get update)
+    sudo %w(/usr/bin/apt-get install lxc-docker)
+
+    # Set up for non-root access
+    sudo %w(/usr/sbin/groupadd docker)
+    sudo '/usr/bin/gpasswd', '-a', ENV['USER'], 'docker'
+    sudo %w(/usr/sbin/service docker restart)
+  when 'Debian'
+  else
+    warn "Must be running a Debian or Ubuntu release in order to run Docker."
+    exit
+  end
+end
+
+
+if __FILE__ == $PROGRAM_NAME
+  options = { :makefile => Dir.pwd + "/build_tools/Makefile" }
+  OptionParser.new do |opts|
+    opts.on('-m', '--makefile MAKEFILE-PATH',
+            'Path to the Makefile used to build Arvados Docker images') do |mk|
+      options[:makefile] = mk
+    end
+  end
+
+  main options
+end
diff --git a/docker/config.rb b/docker/build_tools/config.rb
similarity index 100%
rename from docker/config.rb
rename to docker/build_tools/config.rb
diff --git a/docker/docker_build b/docker/docker_build
deleted file mode 100755
index 0c0fd18..0000000
--- a/docker/docker_build
+++ /dev/null
@@ -1,18 +0,0 @@
-#! /bin/bash
-
-# Wrapper script for `docker build'.
-# This is a workaround for https://github.com/dotcloud/docker/issues/1875.
-
-tmpfile=$(mktemp)
-trap "rm $tmpfile; exit 1" SIGHUP SIGINT SIGTERM
-
-docker build $* | tee ${tmpfile}
-if $(grep -q 'Error build' ${tmpfile})
-then
-  result=1
-else
-  result=0
-fi
-
-rm $tmpfile
-exit $result

commit cf8b27749cf44aee74914622dc8bc2a9690204dc
Author: Tim Pierce <twp at curoverse.com>
Date:   Fri Mar 14 17:19:33 2014 -0400

    Incorporate code review comments (refs #2221, fixes #2325)

diff --git a/sdk/cli/bin/crunch-job b/sdk/cli/bin/crunch-job
index 07b4943..2b31531 100755
--- a/sdk/cli/bin/crunch-job
+++ b/sdk/cli/bin/crunch-job
@@ -1071,7 +1071,7 @@ sub collate_output
     }
     else
     {
-      print $child_in "XXX fetch_block($output) failed XXX\n";
+      Log (undef, "XXX fetch_block($output) failed XXX");
       $main::success = 0;
     }
   }
@@ -1079,7 +1079,11 @@ sub collate_output
 
   if (!defined $joboutput) {
     my $s = IO::Select->new($child_out);
-    sysread($child_out, $joboutput, 64 * 1024 * 1024) if $s->can_read(5);
+    if ($s->can_read(120)) {
+      sysread($child_out, $joboutput, 64 * 1024 * 1024);
+    } else {
+      Log (undef, "timed out reading from 'arv keep put'");
+    }
   }
   waitpid($pid, 0);
 

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list