[ARVADOS] created: 97c9b2b49f94735c6d4d5b62b772089ef7e0e3bf

git at public.curoverse.com git at public.curoverse.com
Tue Jul 1 22:16:10 EDT 2014 

        at  97c9b2b49f94735c6d4d5b62b772089ef7e0e3bf (commit)


commit 97c9b2b49f94735c6d4d5b62b772089ef7e0e3bf
Author: Ward Vandewege <ward at curoverse.com>
Date:   Tue Jul 1 22:12:34 2014 -0400

    Do not print an error when hitting 'not found' with a the MockVolume
    backend.

diff --git a/services/keep/src/keep/keep.go b/services/keep/src/keep/keep.go
index 7d86202..c9047b0 100644
--- a/services/keep/src/keep/keep.go
+++ b/services/keep/src/keep/keep.go
@@ -575,12 +575,13 @@ func GetBlock(hash string) ([]byte, error) {
 	for _, vol := range KeepVM.Volumes() {
 		if buf, err := vol.Get(hash); err != nil {
 			// IsNotExist is an expected error and may be ignored.
+			// "not found" is the equivalent for the MockVolume backend.
 			// (If all volumes report IsNotExist, we return a NotFoundError)
 			// A CorruptError should be returned immediately.
 			// Any other errors should be logged but we continue trying to
 			// read.
 			switch {
-			case os.IsNotExist(err):
+			case (os.IsNotExist(err) || err.Error() == "not found"):
 				continue
 			default:
 				log.Printf("GetBlock: reading %s: %s\n", hash, err)

commit 7b23dea6aaf66492ddb0e204cf0eae4b159f2a1c
Author: Ward Vandewege <ward at curoverse.com>
Date:   Tue Jul 1 21:27:34 2014 -0400

    Move error logging for block-not-found from GetBlock to GetBlockHandler.
    
    It doesn't belong in GetBlock because GetBlock is also called from
    PutBlock, which checks to see if a block exists before it writes it
    again.
    
    In other words, in the most common scenario (a PUT of a new block), Keep
    should not print a bogus error message.

diff --git a/services/keep/src/keep/keep.go b/services/keep/src/keep/keep.go
index 429a7e0..7d86202 100644
--- a/services/keep/src/keep/keep.go
+++ b/services/keep/src/keep/keep.go
@@ -413,6 +413,9 @@ func GetBlockHandler(resp http.ResponseWriter, req *http.Request) {
 	if err != nil {
 		// This type assertion is safe because the only errors
 		// GetBlock can return are CorruptError or NotFoundError.
+		if err == NotFoundError {
+			log.Printf("%s: not found, giving up\n", hash)
+		}
 		http.Error(resp, err.Error(), err.(*KeepError).HTTPCode)
 		return
 	}
@@ -601,7 +604,6 @@ func GetBlock(hash string) ([]byte, error) {
 		}
 	}
 
-	log.Printf("%s: not found on any volumes, giving up\n", hash)
 	return nil, NotFoundError
 }
 

commit f76bc1b30f5bfc32722d2639e609c12531dc8319
Author: Ward Vandewege <ward at curoverse.com>
Date:   Tue Jul 1 21:12:47 2014 -0400

    UnixVolume has no business being loud about files that can not be found.
    Just pass on the error to the caller and be quiet.
    
    Back story: for every PUT, Keep checks if the block already exists.
    Prior to this patch, UnixVolume would complain loudly if the block did
    not already exist in the volume (i.e. the normal case for a PUT).

diff --git a/services/keep/src/keep/volume_unix.go b/services/keep/src/keep/volume_unix.go
index 7b711d2..aafc8de 100644
--- a/services/keep/src/keep/volume_unix.go
+++ b/services/keep/src/keep/volume_unix.go
@@ -111,12 +111,7 @@ func (v *UnixVolume) Put(loc string, block []byte) error {
 func (v *UnixVolume) Read(loc string) ([]byte, error) {
 	blockFilename := filepath.Join(v.root, loc[0:3], loc)
 	buf, err := ioutil.ReadFile(blockFilename)
-	if err != nil {
-		log.Printf("%s: reading %s: %s\n", v, blockFilename, err)
-		return nil, err
-	}
-
-	return buf, nil
+	return buf, err
 }
 
 // Write stores a block of data identified by the locator string

commit 3cc2c876251849002c1377c4631222c26545508d
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date:   Tue Jul 1 11:38:27 2014 -0400

    Change crunch-dispatch to use "git fetch-pack --all" insted of "git fetch" to
    fetch entire repository instead of just the history for HEAD.  This seems to be
    the only way to ensure that arbitrary script version hashes not in the history
    of the source's HEAD are fetched without using refs (tags or branches).
    no issue #

diff --git a/services/api/script/crunch-dispatch.rb b/services/api/script/crunch-dispatch.rb
index b1c0e7d..c39c8ea 100755
--- a/services/api/script/crunch-dispatch.rb
+++ b/services/api/script/crunch-dispatch.rb
@@ -230,7 +230,7 @@ class Dispatcher
         next
       end
 
-      $stderr.puts `cd #{arvados_internal.shellescape} && git fetch --no-tags #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
+      $stderr.puts `cd #{arvados_internal.shellescape} && git fetch-pack --all #{src_repo.shellescape} && git tag #{job.uuid.shellescape} #{job.script_version.shellescape}`
 
       cmd_args << crunch_job_bin
       cmd_args << '--job-api-token'

commit c56584d3bb97bc8e6652ac0d8cdd6c60b57f18ed
Author: Ward Vandewege <ward at curoverse.com>
Date:   Tue Jul 1 16:38:11 2014 -0400

    Add documentation to doc.arvados.org for the local Docker install.

diff --git a/doc/_config.yml b/doc/_config.yml
index f3d34cb..bf19c95 100644
--- a/doc/_config.yml
+++ b/doc/_config.yml
@@ -115,8 +115,11 @@ navbar:
       - admin/index.html.md.liquid
       - admin/cheat_sheet.html.textile.liquid
   installguide:
-    - Install:
+    - Overview:
       - install/index.html.textile.liquid
+    - Docker:
+      - install/install-docker.html.textile.liquid
+    - Manual installation:
       - install/install-sso.html.textile.liquid
       - install/install-api-server.html.textile.liquid
       - install/install-workbench-app.html.textile.liquid
diff --git a/doc/_includes/_alert_docker.liquid b/doc/_includes/_alert_docker.liquid
new file mode 100644
index 0000000..c80af90
--- /dev/null
+++ b/doc/_includes/_alert_docker.liquid
@@ -0,0 +1,4 @@
+<div class="alert alert-block alert-info">
+  <button type="button" class="close" data-dismiss="alert">×</button>
+  <p>The Docker installation is not feature complete. We do not have a Docker container yet for crunch-dispatch and the arvados compute nodes. This means that running pipelines from a Docker-based Arvados install is currently not supported without additional manual configuration. Without that manual configuration, it is possible to use arv-crunch-job to run a 'local' job against your Docker-based Arvados installation. To do this, please refer to the <a href="/user/topics/tutorial-job-debug.html">Debugging a Crunch script</a> page.</p>
+</div>
diff --git a/doc/install/index.html.textile.liquid b/doc/install/index.html.textile.liquid
index 5bf35f3..944a695 100644
--- a/doc/install/index.html.textile.liquid
+++ b/doc/install/index.html.textile.liquid
@@ -1,12 +1,24 @@
 ---
 layout: default
 navsection: installguide
-title: Overview
+title: Installation overview
 ...
 
-{% include 'alert_stub' %}
+Arvados can be installed in multiple ways. Arvados does not depend on any particular cloud operating stack. Arvados runs on one or more GNU/Linux system(s). Arvados is being developed on Debian and Ubuntu GNU/Linux.
+
+The simplest way to try out Arvados is to use the Docker-based installation, which installs Arvados in a series of Docker containers.
+
+For larger scale installations, a manual installation is more appropriate.
+
+h2. Docker
 
-h2. Installation Overview
+{% include 'alert_docker' %}
+
+"Installing with Docker":install-docker.html
+
+h2. Manual installation
+
+{% include 'alert_stub' %}
 
 # Set up a cluster, or use Amazon
 # Create and mount Keep volumes
diff --git a/doc/install/install-docker.html.textile.liquid b/doc/install/install-docker.html.textile.liquid
new file mode 100644
index 0000000..93629e9
--- /dev/null
+++ b/doc/install/install-docker.html.textile.liquid
@@ -0,0 +1,112 @@
+---
+layout: default
+navsection: installguide
+title: Installing with Docker
+...
+
+h2. Prerequisites:
+
+# A GNU/Linux (virtual) machine
+# A working Docker installation
+
+h2. Download the source tree
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd $HOME</span> # (or wherever you want to install)
+~$ <span class="userinput">git clone https://github.com/curoverse/arvados.git</span>
+</code></pre></notextile>
+
+See also: "Downloading the source code":https://arvados.org/projects/arvados/wiki/Download on the Arvados wiki.
+
+h2. Building the Arvados Docker containers
+
+First we need a suitable @config.yml@ file.
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/docker</span>
+~$ <span class="userinput">cp config.yml.example config.yml</span>
+</code></pre></notextile>
+
+Now it's time to edit the @config.yml@ file and fill in suitable values for at a minimum these parameters:
+
+<pre>
+PUBLIC_KEY_PATH
+API_HOSTNAME
+API_AUTO_ADMIN_USER
+</pre>
+
+h2. Running the Arvados Docker containers
+
+The @arvdock@ command can be used to start and stop the docker containers. It has a number of options:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/docker</span>
+~$ <span class="userinput">./arvdock -h</span>
+
+usage: ./arvdock (start|stop|restart|test) [options]
+
+./arvdock start/stop/restart options:
+  -d [port], --doc[=port]        Documentation server (default port 9898)
+  -w [port], --workbench[=port]  Workbench server (default port 9899)
+  -s [port], --sso[=port]        SSO server (default port 9901)
+  -a [port], --api[=port]        API server (default port 9900)
+  -k, --keep                     Keep servers
+  --ssh                          Enable SSH access to server containers
+  -h, --help                     Display this help and exit
+
+  If no options are given, the action is applied to all servers.
+
+./arvdock test [testname] [testname] ...
+  By default, all tests are run.
+</code>
+</pre>
+</notextile>
+
+The @--ssh@ option can be useful to debug issues with the Docker containers; it allows you to ssh into the running containers as the @root@ user, provided you have access to the private key that matches the public key specified in @config.yml@'s PUBLIC_KEY_PATH variable.
+
+Start the docker containers:
+
+<notextile>
+<pre><code>~$ <span class="userinput">cd arvados/docker</span>
+~$ <span class="userinput">./arvdock start</span>
+sso_server
+Starting container:
+  /usr/bin/docker.io run -d -i -t -p 9901:443 --name sso_server arvados/sso
+api_server
+Starting container:
+  /usr/bin/docker.io run -d -i -t -p 9900:443 --name api_server --link sso_server:sso arvados/api
+keep_server_0
+Starting container:
+  /usr/bin/docker.io run -d -i -t -p 25107:25107 --name keep_server_0 -v /tmp/tmp.aCSx8Pq6Wb:/dev/keep-0 --link api_server:api arvados/keep
+keep_server_1
+Starting container:
+  /usr/bin/docker.io run -d -i -t -p 25108:25107 --name keep_server_1 -v /tmp/tmp.m4OQ9WB73G:/dev/keep-0 --link api_server:api arvados/keep
+doc_server
+Starting container:
+  /usr/bin/docker.io run -d -i -t -p 9898:80 --name doc_server arvados/doc
+
+*****************************************************************
+You can access the Arvados documentation at http://localhost:9898
+*****************************************************************
+
+workbench_server
+Starting container:
+  /usr/bin/docker.io run -d -i -t -p 9899:80 --name workbench_server --link api_server:api arvados/workbench
+
+*****************************************************************
+You can access the Arvados workbench at http://localhost:9899
+*****************************************************************
+</code></pre></notextile>
+
+h2. Accessing workbench
+
+Point your browser to the Dockerized workbench:
+
+<notextile>
+<pre><code><span class="userinput">https://localhost:9899</span>
+</code></pre>
+</notextile>
+
+Now use the google account you specified as @API_AUTO_ADMIN_USER@ in @config.yml@ to log in. You will be prompted *twice* by your browser that you are accessing a site with an untrusted SSL certificate. This is normal; by default the Arvados Docker installation uses self-signed SSL certificates for the SSO and API servers, respectively.
+
+

-----------------------------------------------------------------------


hooks/post-receive
--

More information about the arvados-commits mailing list