[ARVADOS] updated: 5d3de212473f9fbf0e797c1fc03ec1dbf8f532ef
git at public.curoverse.com
git at public.curoverse.com
Thu Oct 22 14:55:51 EDT 2015
Summary of changes:
sdk/go/crunchrunner/crunchrunner.go | 98 +++++++++---------
sdk/go/crunchrunner/crunchrunner_test.go | 164 +++++++++++++++----------------
2 files changed, 133 insertions(+), 129 deletions(-)
via 5d3de212473f9fbf0e797c1fc03ec1dbf8f532ef (commit)
from 53b1b5bbe6d6ae007e8ce546ba2539e0c061e25a (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 5d3de212473f9fbf0e797c1fc03ec1dbf8f532ef
Author: Peter Amstutz <peter.amstutz at curoverse.com>
Date: Thu Oct 22 14:55:44 2015 -0400
7582: Make fields in Job, Task, TaskDefs public so that json loading reflection works.
diff --git a/sdk/go/crunchrunner/crunchrunner.go b/sdk/go/crunchrunner/crunchrunner.go
index d3f6fcb..ea29be8 100644
--- a/sdk/go/crunchrunner/crunchrunner.go
+++ b/sdk/go/crunchrunner/crunchrunner.go
@@ -13,32 +13,32 @@ import (
)
type TaskDef struct {
- command []string `json:"command"`
- env map[string]string `json:"task.env"`
- stdin string `json:"task.stdin"`
- stdout string `json:"task.stdout"`
- vwd map[string]string `json:"task.vwd"`
- successCodes []int `json:"task.successCodes"`
- permanentFailCodes []int `json:"task.permanentFailCodes"`
- temporaryFailCodes []int `json:"task.temporaryFailCodes"`
+ Command []string `json:"command"`
+ Env map[string]string `json:"task.env"`
+ Stdin string `json:"task.stdin"`
+ Stdout string `json:"task.stdout"`
+ Vwd map[string]string `json:"task.vwd"`
+ SuccessCodes []int `json:"task.successCodes"`
+ PermanentFailCodes []int `json:"task.permanentFailCodes"`
+ TemporaryFailCodes []int `json:"task.temporaryFailCodes"`
}
type Tasks struct {
- tasks []TaskDef `json:"script_parameters"`
+ Tasks []TaskDef `json:"tasks"`
}
type Job struct {
- script_parameters Tasks `json:"script_parameters"`
+ Script_parameters Tasks `json:"script_parameters"`
}
type Task struct {
- job_uuid string `json:"job_uuid"`
- created_by_job_task_uuid string `json:"created_by_job_task_uuid"`
- parameters TaskDef `json:"parameters"`
- sequence int `json:"sequence"`
- output string `json:"output"`
- success bool `json:"success"`
- progress float32 `json:"sequence"`
+ Job_uuid string `json:"job_uuid"`
+ Created_by_job_task_uuid string `json:"created_by_job_task_uuid"`
+ Parameters TaskDef `json:"parameters"`
+ Sequence int `json:"sequence"`
+ Output string `json:"output"`
+ Success bool `json:"success"`
+ Progress float32 `json:"sequence"`
}
type IArvadosClient interface {
@@ -78,8 +78,8 @@ func checkOutputFilename(outdir, fn string) error {
}
func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[string]string) (stdin, stdout string, err error) {
- if taskp.vwd != nil {
- for k, v := range taskp.vwd {
+ if taskp.Vwd != nil {
+ for k, v := range taskp.Vwd {
v = substitute(v, replacements)
err = checkOutputFilename(outdir, k)
if err != nil {
@@ -89,22 +89,22 @@ func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[
}
}
- if taskp.stdin != "" {
+ if taskp.Stdin != "" {
// Set up stdin redirection
- stdin = substitute(taskp.stdin, replacements)
+ stdin = substitute(taskp.Stdin, replacements)
cmd.Stdin, err = os.Open(stdin)
if err != nil {
return "", "", err
}
}
- if taskp.stdout != "" {
- err = checkOutputFilename(outdir, taskp.stdout)
+ if taskp.Stdout != "" {
+ err = checkOutputFilename(outdir, taskp.Stdout)
if err != nil {
return "", "", err
}
// Set up stdout redirection
- stdout = outdir + "/" + taskp.stdout
+ stdout = outdir + "/" + taskp.Stdout
cmd.Stdout, err = os.Create(stdout)
if err != nil {
return "", "", err
@@ -113,10 +113,10 @@ func setupCommand(cmd *exec.Cmd, taskp TaskDef, outdir string, replacements map[
cmd.Stdout = os.Stdout
}
- if taskp.env != nil {
+ if taskp.Env != nil {
// Set up subprocess environment
cmd.Env = os.Environ()
- for k, v := range taskp.env {
+ for k, v := range taskp.Env {
v = substitute(v, replacements)
cmd.Env = append(cmd.Env, k+"="+v)
}
@@ -171,21 +171,21 @@ func runner(api IArvadosClient,
jobStruct Job, taskStruct Task) error {
var err error
- taskp := taskStruct.parameters
+ taskp := taskStruct.Parameters
// If this is task 0 and there are multiple tasks, dispatch subtasks
// and exit.
- if taskStruct.sequence == 0 {
- if len(jobStruct.script_parameters.tasks) == 1 {
- taskp = jobStruct.script_parameters.tasks[0]
+ if taskStruct.Sequence == 0 {
+ if len(jobStruct.Script_parameters.Tasks) == 1 {
+ taskp = jobStruct.Script_parameters.Tasks[0]
} else {
- for _, task := range jobStruct.script_parameters.tasks {
+ for _, task := range jobStruct.Script_parameters.Tasks {
err := api.Create("job_tasks",
map[string]interface{}{
- "job_task": Task{job_uuid: jobUuid,
- created_by_job_task_uuid: taskUuid,
- sequence: 1,
- parameters: task}},
+ "job_task": Task{Job_uuid: jobUuid,
+ Created_by_job_task_uuid: taskUuid,
+ Sequence: 1,
+ Parameters: task}},
nil)
if err != nil {
return TempFail{err}
@@ -194,9 +194,9 @@ func runner(api IArvadosClient,
err = api.Update("job_tasks", taskUuid,
map[string]interface{}{
"job_task": Task{
- output: "",
- success: true,
- progress: 1.0}},
+ Output: "",
+ Success: true,
+ Progress: 1.0}},
nil)
return nil
}
@@ -214,11 +214,11 @@ func runner(api IArvadosClient,
"$(task.keep)": keepmount}
// Set up subprocess
- for k, v := range taskp.command {
- taskp.command[k] = substitute(v, replacements)
+ for k, v := range taskp.Command {
+ taskp.Command[k] = substitute(v, replacements)
}
- cmd := exec.Command(taskp.command[0], taskp.command[1:]...)
+ cmd := exec.Command(taskp.Command[0], taskp.Command[1:]...)
cmd.Dir = outdir
@@ -257,11 +257,11 @@ func runner(api IArvadosClient,
log.Printf("Completed with exit code %v", exitCode)
- if inCodes(exitCode, taskp.permanentFailCodes) {
+ if inCodes(exitCode, taskp.PermanentFailCodes) {
success = false
- } else if inCodes(exitCode, taskp.temporaryFailCodes) {
+ } else if inCodes(exitCode, taskp.TemporaryFailCodes) {
return TempFail{fmt.Errorf("Process tempfail with exit code %v", exitCode)}
- } else if inCodes(exitCode, taskp.successCodes) || cmd.ProcessState.Success() {
+ } else if inCodes(exitCode, taskp.SuccessCodes) || cmd.ProcessState.Success() {
success = true
} else {
success = false
@@ -277,9 +277,9 @@ func runner(api IArvadosClient,
err = api.Update("job_tasks", taskUuid,
map[string]interface{}{
"job_task": Task{
- output: manifest,
- success: success,
- progress: 1}},
+ Output: manifest,
+ Success: success,
+ Progress: 1}},
nil)
if err != nil {
return TempFail{err}
@@ -317,6 +317,10 @@ func main() {
var kc IKeepClient
kc, err = keepclient.MakeKeepClient(&api)
+ if err != nil {
+ log.Fatal(err)
+ }
+
err = runner(api, kc, jobUuid, taskUuid, tmpdir, keepmount, jobStruct, taskStruct)
if err == nil {
diff --git a/sdk/go/crunchrunner/crunchrunner_test.go b/sdk/go/crunchrunner/crunchrunner_test.go
index 5c309b5..52d5c1a 100644
--- a/sdk/go/crunchrunner/crunchrunner_test.go
+++ b/sdk/go/crunchrunner/crunchrunner_test.go
@@ -35,9 +35,9 @@ func (t ArvTestClient) Create(resourceType string, parameters arvadosclient.Dict
func (t ArvTestClient) Update(resourceType string, uuid string, parameters arvadosclient.Dict, output interface{}) (err error) {
t.c.Check(resourceType, Equals, "job_tasks")
t.c.Check(parameters, DeepEquals, arvadosclient.Dict{"job_task": Task{
- output: t.manifest,
- success: t.success,
- progress: 1}})
+ Output: t.manifest,
+ Success: t.success,
+ Progress: 1}})
return nil
}
@@ -53,9 +53,9 @@ func (s *TestSuite) TestSimpleRun(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"echo", "foo"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"echo", "foo"}}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
}
@@ -88,13 +88,13 @@ func (s *TestSuite) TestSimpleRunSubtask(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{
- TaskDef{command: []string{"echo", "bar"}},
- TaskDef{command: []string{"echo", "foo"}}}}},
- Task{parameters: TaskDef{
- command: []string{"echo", "foo"},
- stdout: "output.txt"},
- sequence: 1})
+ Job{Script_parameters: Tasks{[]TaskDef{
+ TaskDef{Command: []string{"echo", "bar"}},
+ TaskDef{Command: []string{"echo", "foo"}}}}},
+ Task{Parameters: TaskDef{
+ Command: []string{"echo", "foo"},
+ Stdout: "output.txt"},
+ Sequence: 1})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
@@ -118,11 +118,11 @@ func (s *TestSuite) TestRedirect(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"cat"},
- stdout: "output.txt",
- stdin: tmpfile.Name()}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"cat"},
+ Stdout: "output.txt",
+ Stdin: tmpfile.Name()}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
@@ -140,11 +140,11 @@ func (s *TestSuite) TestEnv(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "echo $BAR"},
- stdout: "output.txt",
- env: map[string]string{"BAR": "foo"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "echo $BAR"},
+ Stdout: "output.txt",
+ Env: map[string]string{"BAR": "foo"}}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
@@ -161,11 +161,11 @@ func (s *TestSuite) TestEnvSubstitute(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"foo\n",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "echo $BAR"},
- stdout: "output.txt",
- env: map[string]string{"BAR": "$(task.keep)"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "echo $BAR"},
+ Stdout: "output.txt",
+ Env: map[string]string{"BAR": "$(task.keep)"}}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
@@ -182,11 +182,11 @@ func (s *TestSuite) TestEnvReplace(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "echo $PATH"},
- stdout: "output.txt",
- env: map[string]string{"PATH": "foo"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "echo $PATH"},
+ Stdout: "output.txt",
+ Env: map[string]string{"PATH": "foo"}}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
@@ -211,16 +211,16 @@ func (t SubtaskTestClient) Update(resourceType string, uuid string, parameters a
func (s *TestSuite) TestScheduleSubtask(c *C) {
api := SubtaskTestClient{c, []Task{
- Task{job_uuid: "zzzz-8i9sb-111111111111111",
- created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
- sequence: 1,
- parameters: TaskDef{
- command: []string{"echo", "bar"}}},
- Task{job_uuid: "zzzz-8i9sb-111111111111111",
- created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
- sequence: 1,
- parameters: TaskDef{
- command: []string{"echo", "foo"}}}},
+ Task{Job_uuid: "zzzz-8i9sb-111111111111111",
+ Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
+ Sequence: 1,
+ Parameters: TaskDef{
+ Command: []string{"echo", "bar"}}},
+ Task{Job_uuid: "zzzz-8i9sb-111111111111111",
+ Created_by_job_task_uuid: "zzzz-ot0gb-111111111111111",
+ Sequence: 1,
+ Parameters: TaskDef{
+ Command: []string{"echo", "foo"}}}},
0}
tmpdir, _ := ioutil.TempDir("", "")
@@ -233,10 +233,10 @@ func (s *TestSuite) TestScheduleSubtask(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{
- TaskDef{command: []string{"echo", "bar"}},
- TaskDef{command: []string{"echo", "foo"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{
+ TaskDef{Command: []string{"echo", "bar"}},
+ TaskDef{Command: []string{"echo", "foo"}}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
}
@@ -252,9 +252,9 @@ func (s *TestSuite) TestRunFail(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "exit 1"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "exit 1"}}}}},
+ Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
}
@@ -269,10 +269,10 @@ func (s *TestSuite) TestRunSuccessCode(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "exit 1"},
- successCodes: []int{0, 1}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "exit 1"},
+ SuccessCodes: []int{0, 1}}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
}
@@ -287,10 +287,10 @@ func (s *TestSuite) TestRunFailCode(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "exit 0"},
- permanentFailCodes: []int{0, 1}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "exit 0"},
+ PermanentFailCodes: []int{0, 1}}}}},
+ Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
}
@@ -305,10 +305,10 @@ func (s *TestSuite) TestRunTempFailCode(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"/bin/sh", "-c", "exit 1"},
- temporaryFailCodes: []int{1}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"/bin/sh", "-c", "exit 1"},
+ TemporaryFailCodes: []int{1}}}}},
+ Task{Sequence: 0})
c.Check(err, FitsTypeOf, TempFail{})
}
@@ -329,11 +329,11 @@ func (s *TestSuite) TestVwd(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"ls", "output.txt"},
- vwd: map[string]string{
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"ls", "output.txt"},
+ Vwd: map[string]string{
"output.txt": tmpfile.Name()}}}}},
- Task{sequence: 0})
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
@@ -361,11 +361,11 @@ func (s *TestSuite) TestSubstitutionStdin(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"cat"},
- stdout: "output.txt",
- stdin: "$(task.keep)/file1.txt"}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"cat"},
+ Stdout: "output.txt",
+ Stdin: "$(task.keep)/file1.txt"}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
}
@@ -389,10 +389,10 @@ func (s *TestSuite) TestSubstitutionCommandLine(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
keepmount,
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"cat", "$(task.keep)/file1.txt"},
- stdout: "output.txt"}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"cat", "$(task.keep)/file1.txt"},
+ Stdout: "output.txt"}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
checkOutput(c, tmpdir)
@@ -417,9 +417,9 @@ func (s *TestSuite) TestSignal(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"sleep", "4"}}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"sleep", "4"}}}}},
+ Task{Sequence: 0})
c.Check(err, FitsTypeOf, PermFail{})
}
@@ -437,9 +437,9 @@ func (s *TestSuite) TestQuoting(c *C) {
"zzzz-ot0gb-111111111111111",
tmpdir,
"",
- Job{script_parameters: Tasks{[]TaskDef{TaskDef{
- command: []string{"echo", "foo"},
- stdout: "s ub:dir/:e vi\nl"}}}},
- Task{sequence: 0})
+ Job{Script_parameters: Tasks{[]TaskDef{TaskDef{
+ Command: []string{"echo", "foo"},
+ Stdout: "s ub:dir/:e vi\nl"}}}},
+ Task{Sequence: 0})
c.Check(err, IsNil)
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list