[ARVADOS] created: 1.3.0-620-ge6a5f9527

Git user git at public.curoverse.com
Thu Mar 28 20:40:56 UTC 2019


        at  e6a5f9527903634fceb903925aea3193800a8324 (commit)


commit e6a5f9527903634fceb903925aea3193800a8324
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 16:21:26 2019 -0400

    15044: Specify test database collation.
    
    apps/workbench/test/controllers/projects_controller_test.rb:546 fails
    with other collations.
    
    Remove obsolete websocket comment.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/api/config/database.yml.example b/services/api/config/database.yml.example
index 80876888c..2ec90050a 100644
--- a/services/api/config/database.yml.example
+++ b/services/api/config/database.yml.example
@@ -15,6 +15,7 @@ test:
   adapter: postgresql
   template: template0
   encoding: utf8
+  collation: en_US.utf8
   database: arvados_test
   username: arvados
   password: xxxxxxxx
@@ -28,7 +29,4 @@ production:
   username: arvados
   password: xxxxxxxx
   host: localhost
-  # For the websockets server, prefer a larger database connection pool size since it
-  # multithreaded and can serve a large number of long-lived clients.  See also
-  # websocket_max_connections configuration option.
   pool: 50

commit 0a0237374957dd2dd690d86dc095cd902bbb4520
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 16:01:28 2019 -0400

    15044: Fix unnecessarily unsafe yaml load.
    
    ...and eliminate the associated warnings in test logs.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 61e8c3242..f84022a5b 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -909,7 +909,7 @@ install_services/api() {
     # database, so that we can drop it. This assumes the current user
     # is a postgresql superuser.
     cd "$WORKSPACE/services/api" \
-        && test_database=$(python -c "import yaml; print yaml.load(file('config/database.yml'))['test']['database']") \
+        && test_database=$(python -c "import yaml; print yaml.safe_load(file('config/database.yml'))['test']['database']") \
         && psql "$test_database" -c "SELECT pg_terminate_backend (pg_stat_activity.pid::int) FROM pg_stat_activity WHERE pg_stat_activity.datname = '$test_database';" 2>/dev/null
 
     mkdir -p "$WORKSPACE/services/api/tmp/pids"
diff --git a/sdk/python/tests/run_test_server.py b/sdk/python/tests/run_test_server.py
index 7b1f6059a..a69a1d465 100644
--- a/sdk/python/tests/run_test_server.py
+++ b/sdk/python/tests/run_test_server.py
@@ -738,7 +738,7 @@ def _getport(program):
 def _dbconfig(key):
     global _cached_db_config
     if not _cached_db_config:
-        _cached_db_config = yaml.load(open(os.path.join(
+        _cached_db_config = yaml.safe_load(open(os.path.join(
             SERVICES_SRC_DIR, 'api', 'config', 'database.yml')))
     return _cached_db_config['test'][key]
 
@@ -750,7 +750,7 @@ def _apiconfig(key):
         fullpath = os.path.join(SERVICES_SRC_DIR, 'api', 'config', f)
         if not required and not os.path.exists(fullpath):
             return {}
-        return yaml.load(fullpath)
+        return yaml.safe_load(fullpath)
     cdefault = _load('application.default.yml')
     csite = _load('application.yml', required=False)
     _cached_config = {}
@@ -769,7 +769,7 @@ def fixture(fix):
           yaml_file = yaml_file[0:trim_index]
         except ValueError:
           pass
-        return yaml.load(yaml_file)
+        return yaml.safe_load(yaml_file)
 
 def auth_token(token_name):
     return fixture("api_client_authorizations")[token_name]["api_token"]
diff --git a/sdk/python/tests/test_arv_put.py b/sdk/python/tests/test_arv_put.py
index 01a52a5e6..540e06c6c 100644
--- a/sdk/python/tests/test_arv_put.py
+++ b/sdk/python/tests/test_arv_put.py
@@ -859,7 +859,7 @@ class ArvPutIntegrationTest(run_test_server.TestCaseWithServers,
             if not mandatory and not os.path.exists(path):
                 continue
             with open(path) as f:
-                rails_config = yaml.load(f.read())
+                rails_config = yaml.safe_load(f.read())
                 for config_section in ['test', 'common']:
                     try:
                         key = rails_config[config_section]["blob_signing_key"]

commit ee32549dea929f31c2e00912b3e9e02f8659a094
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 15:58:10 2019 -0400

    15044: Fix workbench test args in interactive mode.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 94fabbc88..61e8c3242 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1058,17 +1058,17 @@ test_services/nodemanager_integration() {
 
 test_apps/workbench_units() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:units TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_units]}
 }
 
 test_apps/workbench_functionals() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:functionals TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_functionals]}
 }
 
 test_apps/workbench_integration() {
     cd "$WORKSPACE/apps/workbench" \
-        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]}
+        && env RAILS_ENV=test ${short:+RAILS_TEST_SHORT=1} bundle exec rake test:integration TESTOPTS=-v ${testargs[apps/workbench]} ${testargs[apps/workbench_integration]}
 }
 
 test_apps/workbench_benchmark() {

commit beef2016f895a7907dd9ef1f46a798789e028585
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 14:58:10 2019 -0400

    15044: Improve error reporting in test case.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/keep-web/server_test.go b/services/keep-web/server_test.go
index 8b689efbd..a9830bc1d 100644
--- a/services/keep-web/server_test.go
+++ b/services/keep-web/server_test.go
@@ -266,9 +266,10 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...
 	c.Log(fmt.Sprintf("curlArgs == %#v", curlArgs))
 	cmd := exec.Command("curl", curlArgs...)
 	stdout, err := cmd.StdoutPipe()
-	c.Assert(err, check.Equals, nil)
-	cmd.Stderr = cmd.Stdout
-	go cmd.Start()
+	c.Assert(err, check.IsNil)
+	cmd.Stderr = os.Stderr
+	err = cmd.Start()
+	c.Assert(err, check.IsNil)
 	buf := make([]byte, 2<<27)
 	n, err := io.ReadFull(stdout, buf)
 	// Discard (but measure size of) anything past 128 MiB.
@@ -276,9 +277,9 @@ func (s *IntegrationSuite) runCurl(c *check.C, token, host, uri string, args ...
 	if err == io.ErrUnexpectedEOF {
 		buf = buf[:n]
 	} else {
-		c.Assert(err, check.Equals, nil)
+		c.Assert(err, check.IsNil)
 		discarded, err = io.Copy(ioutil.Discard, stdout)
-		c.Assert(err, check.Equals, nil)
+		c.Assert(err, check.IsNil)
 	}
 	err = cmd.Wait()
 	// Without "-f", curl exits 0 as long as it gets a valid HTTP

commit 986212eb8c584df783c5836f1e60e721ce635e2c
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 14:12:37 2019 -0400

    15044: Always use the en_US.UTF-8 locale. Add sanity check.
    
    Some tests are sensitive to charset and collation.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 5c09f1eb2..94fabbc88 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -148,6 +148,7 @@ PYTHONPATH=
 GEMHOME=
 PERLINSTALLBASE=
 R_LIBS=
+export LANG=en_US.UTF-8
 
 short=
 only_install=
@@ -195,6 +196,9 @@ sanity_checks() {
         groups | grep -wq $g \
             || fatal "Must be in 'docker' and 'fuse' groups. Try: sudo adduser $USER $g (then log out, and log in again)"
     done
+    echo "locale: ${LANG}"
+    [[ "$(locale charmap)" = "UTF-8" ]] \
+        || fatal "Locale '${LANG}' is broken/missing. Try: echo ${LANG} | sudo tee -a /etc/locale.gen && sudo locale-gen"
     echo -n 'virtualenv: '
     virtualenv --version \
         || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"

commit 4fcb9f3b6b4f190ecec9f84a1d69220dbe305e68
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 13:27:52 2019 -0400

    15044: Listen on localhost instead of default dual-stack [::].
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/arv-git-httpd/gitolite_test.go b/services/arv-git-httpd/gitolite_test.go
index 0656cbf89..88cd221cb 100644
--- a/services/arv-git-httpd/gitolite_test.go
+++ b/services/arv-git-httpd/gitolite_test.go
@@ -52,7 +52,7 @@ func (s *GitoliteSuite) SetUpTest(c *check.C) {
 			APIHost:  arvadostest.APIHost(),
 			Insecure: true,
 		},
-		Listen:       ":0",
+		Listen:       "localhost:0",
 		GitCommand:   "/usr/share/gitolite3/gitolite-shell",
 		GitoliteHome: s.gitoliteHome,
 		RepoRoot:     s.tmpRepoRoot,
diff --git a/services/arv-git-httpd/integration_test.go b/services/arv-git-httpd/integration_test.go
index 10c69eedd..53b636dc0 100644
--- a/services/arv-git-httpd/integration_test.go
+++ b/services/arv-git-httpd/integration_test.go
@@ -77,7 +77,7 @@ func (s *IntegrationSuite) SetUpTest(c *check.C) {
 				APIHost:  arvadostest.APIHost(),
 				Insecure: true,
 			},
-			Listen:          ":0",
+			Listen:          "localhost:0",
 			GitCommand:      "/usr/bin/git",
 			RepoRoot:        s.tmpRepoRoot,
 			ManagementToken: arvadostest.ManagementToken,

commit 1476f136abdf96c5c205d160c233c80a0eec001d
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 11:12:44 2019 -0400

    15044: Test gofmt as a separate suite, instead of refusing to run Go tests.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 4ed183d4f..5c09f1eb2 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -705,7 +705,7 @@ do_test() {
         services/api)
             stop_services
             ;;
-        doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
+        gofmt | doc | lib/cli | lib/cloud/azure | lib/cloud/ec2 | lib/cmd | lib/dispatchcloud/ssh_executor | lib/dispatchcloud/worker)
             # don't care whether services are running
             ;;
         *)
@@ -738,7 +738,6 @@ do_test_once() {
         # compilation errors.
         go get -ldflags "-X main.version=${ARVADOS_VERSION:-$(git log -n1 --format=%H)-dev}" -t "git.curoverse.com/arvados.git/$1" && \
             cd "$GOPATH/src/git.curoverse.com/arvados.git/$1" && \
-            [[ -z "$(gofmt -e -d . | tee /dev/stderr)" ]] && \
             if [[ -n "${testargs[$1]}" ]]
         then
             # "go test -check.vv giturl" doesn't work, but this
@@ -755,6 +754,7 @@ do_test_once() {
             go tool cover -html="$WORKSPACE/tmp/.$covername.tmp" -o "$WORKSPACE/tmp/$covername.html"
             rm "$WORKSPACE/tmp/.$covername.tmp"
         fi
+        [[ $result = 0 ]] && gofmt -e -d *.go
     elif [[ "$2" == "pip" ]]
     then
         tries=0
@@ -1012,6 +1012,12 @@ test_doc() {
     )
 }
 
+test_gofmt() {
+    cd "$WORKSPACE" || return 1
+    dirs=$(ls -d */ | egrep -v 'vendor|tmp')
+    [[ -z "$(gofmt -e -d $dirs | tee -a /dev/stderr)" ]]
+}
+
 test_services/api() {
     rm -f "$WORKSPACE/services/api/git-commit.version"
     cd "$WORKSPACE/services/api" \
@@ -1127,6 +1133,7 @@ test_all() {
         exit_cleanly
     fi
 
+    do_test gofmt
     do_test doc
     do_test sdk/ruby
     do_test sdk/R

commit 82587c95f3cebfd086bbb80254e7c7c8d03ee003
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 10:48:32 2019 -0400

    15044: Add sanity check for docker and fuse group membership.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index e7573cea8..4ed183d4f 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -189,6 +189,12 @@ sanity_checks() {
     ( [[ -n "$WORKSPACE" ]] && [[ -d "$WORKSPACE/services" ]] ) \
         || fatal "WORKSPACE environment variable not set to a source directory (see: $0 --help)"
     echo Checking dependencies:
+    for g in docker fuse; do
+        egrep -q "^$g:" /etc/group \
+            || fatal "Must be in 'docker' and 'fuse' groups. Try: sudo addgroup $g && sudo adduser $USER $g (then log out, and log in again)"
+        groups | grep -wq $g \
+            || fatal "Must be in 'docker' and 'fuse' groups. Try: sudo adduser $USER $g (then log out, and log in again)"
+    done
     echo -n 'virtualenv: '
     virtualenv --version \
         || fatal "No virtualenv. Try: apt-get install virtualenv (on ubuntu: python-virtualenv)"

commit 2614d2fc20f30e1a83baaa0db52adfa628f8b4f2
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Thu Mar 28 10:37:13 2019 -0400

    15044: Fix error messages.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/services/crunch-run/crunchrun.go b/services/crunch-run/crunchrun.go
index 3925b0b7b..84b578a3e 100644
--- a/services/crunch-run/crunchrun.go
+++ b/services/crunch-run/crunchrun.go
@@ -987,7 +987,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
 		go func() {
 			_, err := io.Copy(response.Conn, stdinRdr)
 			if err != nil {
-				runner.CrunchLog.Print("While writing stdin collection to docker container %q", err)
+				runner.CrunchLog.Printf("While writing stdin collection to docker container %q", err)
 				runner.stop(nil)
 			}
 			stdinRdr.Close()
@@ -997,7 +997,7 @@ func (runner *ContainerRunner) AttachStreams() (err error) {
 		go func() {
 			_, err := io.Copy(response.Conn, bytes.NewReader(stdinJson))
 			if err != nil {
-				runner.CrunchLog.Print("While writing stdin json to docker container %q", err)
+				runner.CrunchLog.Printf("While writing stdin json to docker container %q", err)
 				runner.stop(nil)
 			}
 			response.CloseWrite()
diff --git a/services/keepproxy/keepproxy.go b/services/keepproxy/keepproxy.go
index fc4783eff..caeaa0e2c 100644
--- a/services/keepproxy/keepproxy.go
+++ b/services/keepproxy/keepproxy.go
@@ -152,7 +152,7 @@ func main() {
 		}
 		err = f.Sync()
 		if err != nil {
-			log.Fatal("sync(%s): %s", cfg.PIDFile, err)
+			log.Fatalf("sync(%s): %s", cfg.PIDFile, err)
 		}
 	}
 

commit c1794d8ca1c32300752221fc6c46016450bfd625
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Mar 27 16:46:04 2019 -0400

    15044: Fix interactive command handling.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index 9614ff5b1..e7573cea8 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -1208,32 +1208,32 @@ else
         read verb target opts <<<"${nextcmd}"
         target="${target%/}"
         target="${target/\/:/:}"
-        if [[ -z "${target}" ]]; then
-            verb=help
-        fi
         case "${verb}" in
-            "" | "help")
-                help_interactive
-                ;;
             "exit" | "quit")
                 exit_cleanly
                 ;;
             "reset")
                 stop_services
                 ;;
-            *)
-                testargs["$target"]="${opts}"
+            "test" | "install")
                 case "$target" in
+                    "")
+                        help_interactive
+                        ;;
                     all | deps)
                         ${verb}_${target}
                         ;;
                     *)
+                        testargs["$target"]="${opts}"
                         tt="${testfuncargs[${target}]}"
                         tt="${tt:-$target}"
                         do_$verb $tt
                         ;;
                 esac
                 ;;
+            "" | "help" | *)
+                help_interactive
+                ;;
         esac
         if [[ ${#successes[@]} -gt 0 || ${#failures[@]} -gt 0 ]]; then
             report_outcomes

commit 4398ef03604ddbc1fd3942246ae0d67d83d33fae
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date:   Wed Mar 27 16:42:25 2019 -0400

    15044: Override unusable nginx defaults. Test nginx startup success.
    
    Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>

diff --git a/build/run-tests.sh b/build/run-tests.sh
index dc4659453..9614ff5b1 100755
--- a/build/run-tests.sh
+++ b/build/run-tests.sh
@@ -392,6 +392,10 @@ start_services() {
         && (env | egrep ^ARVADOS) \
         || fail=1
     deactivate
+    if [[ $fail = 0 ]] && ! kill -0 "$(cat "$WORKSPACE/tmp/nginx.pid")"; then
+        echo >&2 "ERROR: nginx seems to have died already"
+        fail=1
+    fi
     if [[ $fail != 0 ]]; then
         unset ARVADOS_TEST_API_HOST
     fi
diff --git a/sdk/python/tests/nginx.conf b/sdk/python/tests/nginx.conf
index 130d8c964..1ef3b00c6 100644
--- a/sdk/python/tests/nginx.conf
+++ b/sdk/python/tests/nginx.conf
@@ -12,6 +12,10 @@ http {
     '"$http_referer" "$http_user_agent"';
   access_log "{{ACCESSLOG}}" customlog;
   client_body_temp_path "{{TMPDIR}}";
+  proxy_temp_path "{{TMPDIR}}";
+  fastcgi_temp_path "{{TMPDIR}}";
+  uwsgi_temp_path "{{TMPDIR}}";
+  scgi_temp_path "{{TMPDIR}}";
   upstream arv-git-http {
     server localhost:{{GITPORT}};
   }

-----------------------------------------------------------------------


hooks/post-receive
-- 




More information about the arvados-commits mailing list