[ARVADOS] updated: 53b1b5bbe6d6ae007e8ce546ba2539e0c061e25a
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 22 14:17:12 EDT 2015
Summary of changes:
sdk/go/arvadosclient/arvadosclient.go | 8 --
sdk/go/crunchrunner/crunchrunner.go | 174 +++++++++++++----------
sdk/go/crunchrunner/crunchrunner_test.go | 229 ++++++++++++++++++++++++-------
sdk/go/crunchrunner/upload.go | 19 ++-
sdk/go/crunchrunner/upload_test.go | 17 +--
5 files changed, 302 insertions(+), 145 deletions(-)
via 53b1b5bbe6d6ae007e8ce546ba2539e0c061e25a (commit)
from 0d428c09204f37f9c276c31e54bdb7acc5c80c02 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 53b1b5bbe6d6ae007e8ce546ba2539e0c061e25a
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 22 14:16:56 2015 -0400
7582: Add parameter substitution. Improve validity checking for filenames.
Adjust signal handling & added test. Tweak behavior on exit code handling.
Move IArvadosClient to crunchrunner.
diff --git a/sdk/go/arvadosclient/arvadosclient.go b/sdk/go/arvadosclient/arvadosclient.go
index 09170ba..1cce0a7 100644
--- a/sdk/go/arvadosclient/arvadosclient.go
+++ b/sdk/go/arvadosclient/arvadosclient.go
@@ -78,14 +78,6 @@ type ArvadosClient struct {
DiscoveryDoc Dict
}
-type IArvadosClient interface {
- Create(resourceType string, parameters Dict, output interface{}) error
- Delete(resource string, uuid string, parameters Dict, output interface{}) (err error)
- Update(resourceType string, uuid string, parameters Dict, output interface{}) (err error)
- Get(resourceType string, uuid string, parameters Dict, output interface{}) (err error)
- List(resource string, parameters Dict, output interface{}) (err error)
-}
-
// Create a new ArvadosClient, initialized with standard Arvados environment
// variables ARVADOS_API_HOST, ARVADOS_API_TOKEN, and (optionally)
// ARVADOS_API_HOST_INSECURE.
diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
index 0aacfeb..d3f6fcb 100644
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ b/sdk/go/crunchrunner/crunchrunner.go
@@ -1,24 +1,26 @@
package main
import (
+ "fmt"
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/keepclient"
"log"
"os"
"os/exec"
"os/signal"
+ "strings"
"syscall"
)
type TaskDef struct {
- commands []string `json:"commands"`
- env map[string]string `json:"env"`
- stdin string `json:"stdin"`
- stdout string `json:"stdout"`
- vwd map[string]string `json:"vwd"`
- successCodes []int `json:"successCodes"`
- permanentFailCodes []int `json:"permanentFailCodes"`
- temporaryFailCodes []int `json:"temporaryFailCodes"`
+ command []string `json:"command"`
+ env map[string]string `json:"task.env"`
+ stdin string `json:"task.stdin"`
+ stdout string `json:"task.stdout"`
+ vwd map[string]string `json:"task.vwd"`
+ successCodes []int `json:"task.successCodes"`
+ permanentFailCodes []int `json:"task.permanentFailCodes"`
+ temporaryFailCodes []int `json:"task.temporaryFailCodes"`
}
type Tasks struct {
@@ -39,57 +41,73 @@ type Task struct {
progress float32 `json:"sequence"`
}
-func setupDirectories(tmpdir, taskUuid string) (outdir string, err error) {
- err = os.Chdir(tmpdir)
- if err != nil {
- return "", err
- }
+type IArvadosClient interface {
+ Create(resourceType string, parameters arvadosclient.Dict, output interface{}) error
+ Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error)
+}
- err = os.Mkdir("tmpdir", 0700)
+func setupDirectories(crunchtmpdir, taskUuid string) (tmpdir, outdir string, err error) {
+ tmpdir = crunchtmpdir + "/tmpdir"
+ err = os.Mkdir(tmpdir, 0700)
if err != nil {
- return "", err
+ return "", "", err
}
- err = os.Mkdir(taskUuid, 0700)
+ outdir = crunchtmpdir + "/outdir"
+ err = os.Mkdir(outdir, 0700)
if err != nil {
- return "", err
+ return "", "", err
}
- os.Chdir(taskUuid)
- if err != nil {
- return "", err
- }
+ return tmpdir, outdir, nil
+}
- outdir, err = os.Getwd()
- if err != nil {
- return "", err
+func checkOutputFilename(outdir, fn string) error {
+ if strings.HasPrefix(fn, "/") || strings.HasSuffix(fn, "/") {
+ return fmt.Errorf("Path must not start or end with '/'")
+ }
+ if strings.Index("../", fn) != -1 {
+ return fmt.Errorf("Path must not contain '../'")
}
- return outdir, nil
+ sl := strings.LastIndex(fn, "/")
+ if sl != -1 {
+ os.MkdirAll(outdir+"/"+fn[0:sl], 0777)
+ }
+ return nil
}
-func setupCommand(cmd *exec.Cmd, taskp TaskDef, keepmount, outdir string) error {
- var err error
-
+func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[string]string) (stdin, stdout string, err error) {
if taskp.vwd != nil {
for k, v := range taskp.vwd {
- os.Symlink(keepmount+"/"+v, outdir+"/"+k)
+ v = substitute(v, replacements)
+ err = checkOutputFilename(outdir, k)
+ if err != nil {
+ return "", "", err
+ }
+ os.Symlink(v, outdir+"/"+k)
}
}
if taskp.stdin != "" {
// Set up stdin redirection
- cmd.Stdin, err = os.Open(keepmount + "/" + taskp.stdin)
+ stdin = substitute(taskp.stdin, replacements)
+ cmd.Stdin, err = os.Open(stdin)
if err != nil {
- return err
+ return "", "", err
}
}
if taskp.stdout != "" {
+ err = checkOutputFilename(outdir, taskp.stdout)
+ if err != nil {
+ return "", "", err
+ }
// Set up stdout redirection
- cmd.Stdout, err = os.Create(outdir + "/" + taskp.stdout)
+ stdout = outdir + "/" + taskp.stdout
+ cmd.Stdout, err = os.Create(stdout)
if err != nil {
- return err
+ return "", "", err
}
} else {
cmd.Stdout = os.Stdout
@@ -99,25 +117,25 @@ func setupCommand(cmd *exec.Cmd, taskp TaskDef, keepmount, outdir string) error
// Set up subprocess environment
cmd.Env = os.Environ()
for k, v := range taskp.env {
+ v = substitute(v, replacements)
cmd.Env = append(cmd.Env, k+"="+v)
}
}
- return nil
+ return stdin, stdout, nil
}
-func setupSignals(cmd *exec.Cmd) {
+func setupSignals(cmd *exec.Cmd) chan os.Signal {
// Set up signal handlers
// Forward SIGINT, SIGTERM and SIGQUIT to inner process
sigChan := make(chan os.Signal, 1)
go func(sig <-chan os.Signal) {
catch := <-sig
- if cmd.Process != nil {
- cmd.Process.Signal(catch)
- }
+ cmd.Process.Signal(catch)
}(sigChan)
signal.Notify(sigChan, syscall.SIGTERM)
signal.Notify(sigChan, syscall.SIGINT)
signal.Notify(sigChan, syscall.SIGQUIT)
+ return sigChan
}
func inCodes(code int, codes []int) bool {
@@ -133,20 +151,23 @@ func inCodes(code int, codes []int) bool {
const TASK_TEMPFAIL = 111
-type TempFail struct{ InnerError error }
+type TempFail struct{ error }
type PermFail struct{}
-func (s TempFail) Error() string {
- return s.InnerError.Error()
-}
-
func (s PermFail) Error() string {
return "PermFail"
}
-func runner(api arvadosclient.IArvadosClient,
+func substitute(inp string, subst map[string]string) string {
+ for k, v := range subst {
+ inp = strings.Replace(inp, k, v, -1)
+ }
+ return inp
+}
+
+func runner(api IArvadosClient,
kc IKeepClient,
- jobUuid, taskUuid, tmpdir, keepmount string,
+ jobUuid, taskUuid, crunchtmpdir, keepmount string,
jobStruct Job, taskStruct Task) error {
var err error
@@ -181,28 +202,46 @@ func runner(api arvadosclient.IArvadosClient,
}
}
- // Set up subprocess
- cmd := exec.Command(taskp.commands[0], taskp.commands[1:]...)
-
- var outdir string
- outdir, err = setupDirectories(tmpdir, taskUuid)
+ var tmpdir, outdir string
+ tmpdir, outdir, err = setupDirectories(crunchtmpdir, taskUuid)
if err != nil {
return TempFail{err}
}
+ replacements := map[string]string{
+ "$(task.tmpdir)": tmpdir,
+ "$(task.outdir)": outdir,
+ "$(task.keep)": keepmount}
+
+ // Set up subprocess
+ for k, v := range taskp.command {
+ taskp.command[k] = substitute(v, replacements)
+ }
+
+ cmd := exec.Command(taskp.command[0], taskp.command[1:]...)
+
cmd.Dir = outdir
- err = setupCommand(cmd, taskp, keepmount, outdir)
+ var stdin, stdout string
+ stdin, stdout, err = setupCommand(cmd, taskp, outdir, replacements)
if err != nil {
return err
}
- setupSignals(cmd)
-
// Run subprocess and wait for it to complete
- log.Printf("Running %v", cmd.Args)
+ if stdin != "" {
+ stdin = " < " + stdin
+ }
+ if stdout != "" {
+ stdout = " > " + stdout
+ }
+ log.Printf("Running %v%v%v", cmd.Args, stdin, stdout)
- err = cmd.Run()
+ err = cmd.Start()
+
+ signals := setupSignals(cmd)
+ err = cmd.Wait()
+ signal.Stop(signals)
if err != nil {
// Run() returns ExitError on non-zero exit code, but we handle
@@ -212,25 +251,20 @@ func runner(api arvadosclient.IArvadosClient,
}
}
- const success = 1
- const permfail = 2
- const tempfail = 2
- var status int
+ var success bool
exitCode := cmd.ProcessState.Sys().(syscall.WaitStatus).ExitStatus()
log.Printf("Completed with exit code %v", exitCode)
- if inCodes(exitCode, taskp.successCodes) {
- status = success
- } else if inCodes(exitCode, taskp.permanentFailCodes) {
- status = permfail
+ if inCodes(exitCode, taskp.permanentFailCodes) {
+ success = false
} else if inCodes(exitCode, taskp.temporaryFailCodes) {
- return TempFail{nil}
- } else if cmd.ProcessState.Success() {
- status = success
+ return TempFail{fmt.Errorf("Process tempfail with exit code %v", exitCode)}
+ } else if inCodes(exitCode, taskp.successCodes) || cmd.ProcessState.Success() {
+ success = true
} else {
- status = permfail
+ success = false
}
// Upload output directory
@@ -244,14 +278,14 @@ func runner(api arvadosclient.IArvadosClient,
map[string]interface{}{
"job_task": Task{
output: manifest,
- success: status == success,
+ success: success,
progress: 1}},
nil)
if err != nil {
return TempFail{err}
}
- if status == success {
+ if success {
return nil
} else {
return PermFail{}
@@ -259,8 +293,6 @@ func runner(api arvadosclient.IArvadosClient,
}
func main() {
- syscall.Umask(0077)
-
api, err := arvadosclient.MakeArvadosClient()
if err != nil {
log.Fatal(err)
diff --git a/sdk/go/crunchrunner/crunchrunner_test.go b/sdk/go/crunchrunner/crunchrunner_test.go
index e67c9ee..5c309b5 100644
--- a/sdk/go/crunchrunner/crunchrunner_test.go
+++ b/sdk/go/crunchrunner/crunchrunner_test.go
@@ -3,9 +3,13 @@ package main
import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
. "gopkg.in/check.v1"
+ "io"
"io/ioutil"
+ "log"
"os"
+ "syscall"
"testing"
+ "time"
)
// Gocheck boilerplate
@@ -28,10 +32,6 @@ func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict
return nil
}
-func (t ArvTestClient) Delete(resource string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
t.c.Check(resourceType, Equals, "job_tasks")
t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
@@ -41,14 +41,6 @@ func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvad
return nil
}
-func (t ArvTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
-func (t ArvTestClient) List(resource string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
func (s *TestSuite) TestSimpleRun(c *C) {
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
@@ -62,21 +54,25 @@ func (s *TestSuite) TestSimpleRun(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"echo", "foo"}}}}},
+ command: []string{"echo", "foo"}}}}},
Task{sequence: 0})
c.Check(err, IsNil)
-
}
func checkOutput(c *C, tmpdir string) {
- file, err := os.Open(tmpdir + "/zzzz-ot0gb-111111111111111/output.txt")
+ file, err := os.Open(tmpdir + "/outdir/output.txt")
c.Assert(err, IsNil)
data := make([]byte, 100)
var count int
- count, err = file.Read(data)
- c.Assert(err, IsNil)
- c.Check(string(data[0:count]), Equals, "foo\n")
+ err = nil
+ offset := 0
+ for err == nil {
+ count, err = file.Read(data[offset:])
+ offset += count
+ }
+ c.Assert(err, Equals, io.EOF)
+ c.Check(string(data[0:offset]), Equals, "foo\n")
}
func (s *TestSuite) TestSimpleRunSubtask(c *C) {
@@ -93,11 +89,11 @@ func (s *TestSuite) TestSimpleRunSubtask(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{
- TaskDef{commands: []string{"echo", "bar"}},
- TaskDef{commands: []string{"echo", "foo"}}}}},
+ TaskDef{command: []string{"echo", "bar"}},
+ TaskDef{command: []string{"echo", "foo"}}}}},
Task{parameters: TaskDef{
- commands: []string{"echo", "foo"},
- stdout: "output.txt"},
+ command: []string{"echo", "foo"},
+ stdout: "output.txt"},
sequence: 1})
c.Check(err, IsNil)
@@ -123,9 +119,9 @@ func (s *TestSuite) TestRedirect(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"cat"},
- stdout: "output.txt",
- stdin: tmpfile.Name()}}}},
+ command: []string{"cat"},
+ stdout: "output.txt",
+ stdin: tmpfile.Name()}}}},
Task{sequence: 0})
c.Check(err, IsNil)
@@ -145,12 +141,53 @@ func (s *TestSuite) TestEnv(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"/bin/sh", "-c", "echo $BAR"},
- stdout: "output.txt",
- env: map[string]string{"BAR": "foo"}}}}},
+ command: []string{"/bin/sh", "-c", "echo $BAR"},
+ stdout: "output.txt",
+ env: map[string]string{"BAR": "foo"}}}}},
Task{sequence: 0})
c.Check(err, IsNil)
+ checkOutput(c, tmpdir)
+}
+func (s *TestSuite) TestEnvSubstitute(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+ KeepTestClient{},
+ "zzzz-8i9sb-111111111111111",
+ "zzzz-ot0gb-111111111111111",
+ tmpdir,
+ "foo\n",
+ Job{script_parameters: Tasks{[]TaskDef{TaskDef{
+ command: []string{"/bin/sh", "-c", "echo $BAR"},
+ stdout: "output.txt",
+ env: map[string]string{"BAR": "$(task.keep)"}}}}},
+ Task{sequence: 0})
+ c.Check(err, IsNil)
+ checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestEnvReplace(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ err := runner(ArvTestClient{c, ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+ KeepTestClient{},
+ "zzzz-8i9sb-111111111111111",
+ "zzzz-ot0gb-111111111111111",
+ tmpdir,
+ "",
+ Job{script_parameters: Tasks{[]TaskDef{TaskDef{
+ command: []string{"/bin/sh", "-c", "echo $PATH"},
+ stdout: "output.txt",
+ env: map[string]string{"PATH": "foo"}}}}},
+ Task{sequence: 0})
+ c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
@@ -167,22 +204,10 @@ func (t *SubtaskTestClient) Create(resourceType string, parameters arvadosclient
return nil
}
-func (t SubtaskTestClient) Delete(resource string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
return nil
}
-func (t SubtaskTestClient) Get(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
-func (t SubtaskTestClient) List(resource string, parameters arvadosclient.Dict, output interface{}) (err error) {
- return nil
-}
-
func (s *TestSuite) TestScheduleSubtask(c *C) {
api := SubtaskTestClient{c, []Task{
@@ -190,12 +215,12 @@ func (s *TestSuite) TestScheduleSubtask(c *C) {
created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
sequence: 1,
parameters: TaskDef{
- commands: []string{"echo", "bar"}}},
+ command: []string{"echo", "bar"}}},
Task{job_uuid: "zzzz-8i9sb-111111111111111",
created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
sequence: 1,
parameters: TaskDef{
- commands: []string{"echo", "foo"}}}},
+ command: []string{"echo", "foo"}}}},
0}
tmpdir, _ := ioutil.TempDir("", "")
@@ -209,8 +234,8 @@ func (s *TestSuite) TestScheduleSubtask(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{
- TaskDef{commands: []string{"echo", "bar"}},
- TaskDef{commands: []string{"echo", "foo"}}}}},
+ TaskDef{command: []string{"echo", "bar"}},
+ TaskDef{command: []string{"echo", "foo"}}}}},
Task{sequence: 0})
c.Check(err, IsNil)
@@ -228,7 +253,7 @@ func (s *TestSuite) TestRunFail(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"/bin/sh", "-c", "exit 1"}}}}},
+ command: []string{"/bin/sh", "-c", "exit 1"}}}}},
Task{sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
}
@@ -245,7 +270,7 @@ func (s *TestSuite) TestRunSuccessCode(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"/bin/sh", "-c", "exit 1"},
+ command: []string{"/bin/sh", "-c", "exit 1"},
successCodes: []int{0, 1}}}}},
Task{sequence: 0})
c.Check(err, IsNil)
@@ -263,7 +288,7 @@ func (s *TestSuite) TestRunFailCode(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"/bin/sh", "-c", "exit 0"},
+ command: []string{"/bin/sh", "-c", "exit 0"},
permanentFailCodes: []int{0, 1}}}}},
Task{sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
@@ -281,7 +306,7 @@ func (s *TestSuite) TestRunTempFailCode(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"/bin/sh", "-c", "exit 1"},
+ command: []string{"/bin/sh", "-c", "exit 1"},
temporaryFailCodes: []int{1}}}}},
Task{sequence: 0})
c.Check(err, FitsTypeOf, TempFail{})
@@ -305,10 +330,116 @@ func (s *TestSuite) TestVwd(c *C) {
tmpdir,
"",
Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- commands: []string{"ls", "output.txt"},
+ command: []string{"ls", "output.txt"},
vwd: map[string]string{
"output.txt": tmpfile.Name()}}}}},
Task{sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
+
+func (s *TestSuite) TestSubstitutionStdin(c *C) {
+ keepmount, _ := ioutil.TempDir("", "")
+ ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
+ defer func() {
+ os.RemoveAll(keepmount)
+ }()
+
+ log.Print("Keepmount is ", keepmount)
+
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ log.Print("tmpdir is ", tmpdir)
+
+ err := runner(ArvTestClient{c,
+ ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+ KeepTestClient{},
+ "zzzz-8i9sb-111111111111111",
+ "zzzz-ot0gb-111111111111111",
+ tmpdir,
+ keepmount,
+ Job{script_parameters: Tasks{[]TaskDef{TaskDef{
+ command: []string{"cat"},
+ stdout: "output.txt",
+ stdin: "$(task.keep)/file1.txt"}}}},
+ Task{sequence: 0})
+ c.Check(err, IsNil)
+ checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
+ keepmount, _ := ioutil.TempDir("", "")
+ ioutil.WriteFile(keepmount+"/"+"file1.txt", []byte("foo\n"), 0600)
+ defer func() {
+ os.RemoveAll(keepmount)
+ }()
+
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ err := runner(ArvTestClient{c,
+ ". d3b07384d113edec49eaa6238ad5ff00+4 0:4:output.txt\n", true},
+ KeepTestClient{},
+ "zzzz-8i9sb-111111111111111",
+ "zzzz-ot0gb-111111111111111",
+ tmpdir,
+ keepmount,
+ Job{script_parameters: Tasks{[]TaskDef{TaskDef{
+ command: []string{"cat", "$(task.keep)/file1.txt"},
+ stdout: "output.txt"}}}},
+ Task{sequence: 0})
+ c.Check(err, IsNil)
+
+ checkOutput(c, tmpdir)
+}
+
+func (s *TestSuite) TestSignal(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ go func() {
+ time.Sleep(1 * time.Second)
+ self, _ := os.FindProcess(os.Getpid())
+ self.Signal(syscall.SIGINT)
+ }()
+
+ err := runner(ArvTestClient{c,
+ "", false},
+ KeepTestClient{},
+ "zzzz-8i9sb-111111111111111",
+ "zzzz-ot0gb-111111111111111",
+ tmpdir,
+ "",
+ Job{script_parameters: Tasks{[]TaskDef{TaskDef{
+ command: []string{"sleep", "4"}}}}},
+ Task{sequence: 0})
+ c.Check(err, FitsTypeOf, PermFail{})
+
+}
+
+func (s *TestSuite) TestQuoting(c *C) {
+ tmpdir, _ := ioutil.TempDir("", "")
+ defer func() {
+ os.RemoveAll(tmpdir)
+ }()
+
+ err := runner(ArvTestClient{c,
+ "./s\\040ub:dir d3b07384d113edec49eaa6238ad5ff00+4 0:4::e\\040vil\n", true},
+ KeepTestClient{},
+ "zzzz-8i9sb-111111111111111",
+ "zzzz-ot0gb-111111111111111",
+ tmpdir,
+ "",
+ Job{script_parameters: Tasks{[]TaskDef{TaskDef{
+ command: []string{"echo", "foo"},
+ stdout: "s ub:dir/:e vi\nl"}}}},
+ Task{sequence: 0})
+ c.Check(err, IsNil)
+}
diff --git a/sdk/go/crunchrunner/upload.go b/sdk/go/crunchrunner/upload.go
index 4feb142..ac3f065 100644
--- a/sdk/go/crunchrunner/upload.go
+++ b/sdk/go/crunchrunner/upload.go
@@ -11,6 +11,8 @@ import (
"log"
"os"
"path/filepath"
+ "sort"
+ "strings"
)
type Block struct {
@@ -169,10 +171,23 @@ func (m *ManifestWriter) Finish() error {
func (m *ManifestWriter) ManifestText() string {
m.Finish()
var buf bytes.Buffer
- for k, v := range m.Streams {
+
+ dirs := make([]string, len(m.Streams))
+ i := 0
+ for k := range m.Streams {
+ dirs[i] = k
+ i++
+ }
+ sort.Strings(dirs)
+
+ for _, k := range dirs {
+ v := m.Streams[k]
+
if k == "." {
buf.WriteString(".")
} else {
+ k = strings.Replace(k, " ", "\\040", -1)
+ k = strings.Replace(k, "\n", "", -1)
buf.WriteString("./" + k)
}
for _, b := range v.Blocks {
@@ -181,6 +196,8 @@ func (m *ManifestWriter) ManifestText() string {
}
for _, f := range v.Files {
buf.WriteString(" ")
+ f = strings.Replace(f, " ", "\\040", -1)
+ f = strings.Replace(f, "\n", "", -1)
buf.WriteString(f)
}
buf.WriteString("\n")
diff --git a/sdk/go/crunchrunner/upload_test.go b/sdk/go/crunchrunner/upload_test.go
index e337b76..a2bf0ac 100644
--- a/sdk/go/crunchrunner/upload_test.go
+++ b/sdk/go/crunchrunner/upload_test.go
@@ -6,7 +6,6 @@ import (
"fmt"
. "gopkg.in/check.v1"
"io/ioutil"
- "log"
"os"
)
@@ -23,8 +22,6 @@ func (k KeepTestClient) PutHB(hash string, buf []byte) (string, int, error) {
}
func (s *TestSuite) TestSimpleUpload(c *C) {
- log.Print("--TestSimpleUpload--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
@@ -38,8 +35,6 @@ func (s *TestSuite) TestSimpleUpload(c *C) {
}
func (s *TestSuite) TestSimpleUploadTwofiles(c *C) {
- log.Print("--TestSimpleUploadTwofiles--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
@@ -54,8 +49,6 @@ func (s *TestSuite) TestSimpleUploadTwofiles(c *C) {
}
func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
- log.Print("--TestSimpleUploadSubdir--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
@@ -74,8 +67,6 @@ func (s *TestSuite) TestSimpleUploadSubdir(c *C) {
}
func (s *TestSuite) TestSimpleUploadLarge(c *C) {
- log.Print("--TestSimpleUploadLarge--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
@@ -83,7 +74,7 @@ func (s *TestSuite) TestSimpleUploadLarge(c *C) {
file, _ := os.Create(tmpdir + "/" + "file1.txt")
data := make([]byte, 1024*1024-1)
- for i := 0; i < 1024*1024-1; i++ {
+ for i := range data {
data[i] = byte(i % 10)
}
for i := 0; i < 65; i++ {
@@ -99,8 +90,6 @@ func (s *TestSuite) TestSimpleUploadLarge(c *C) {
}
func (s *TestSuite) TestUploadEmptySubdir(c *C) {
- log.Print("--TestUploadEmptySubdir--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
@@ -117,8 +106,6 @@ func (s *TestSuite) TestUploadEmptySubdir(c *C) {
}
func (s *TestSuite) TestUploadEmptyFile(c *C) {
- log.Print("--TestUploadEmptyFile--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
@@ -140,8 +127,6 @@ func (k KeepErrorTestClient) PutHB(hash string, buf []byte) (string, int, error)
}
func (s *TestSuite) TestUploadError(c *C) {
- log.Print("--TestSimpleUpload--")
-
tmpdir, _ := ioutil.TempDir("", "")
defer func() {
os.RemoveAll(tmpdir)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list