[ARVADOS] updated: 48c0b2f90d232d1508f1a6c8214c1b8e33c78795
Git user
git at public.curoverse.com
Tue May 10 11:59:39 EDT 2016
Summary of changes:
services/crunch-dispatch-slurm/crunch-dispatch-slurm.go | 10 +++++-----
services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go | 2 +-
2 files changed, 6 insertions(+), 6 deletions(-)
via 48c0b2f90d232d1508f1a6c8214c1b8e33c78795 (commit)
via 37a00e62fa74ce162ee13d16c731dc10218c5df3 (commit)
via b5e2bbb94cd7e121d4b8d83af86ef0ab0c449dcf (commit)
via 1695ccb6279de083eeed80d36d38f607a5c7098d (commit)
from 505c7f34a713906581329417a4e21ac932859926 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 48c0b2f90d232d1508f1a6c8214c1b8e33c78795
Author: radhika <radhika at curoverse.com>
Date: Tue May 10 11:59:16 2016 -0400
8017: RuntimeConstraints uses int64
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 096420f..8c3f5c9 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -108,10 +108,10 @@ func runQueuedContainers(pollInterval, priorityPollInterval int, crunchRunComman
// Container data
type Container struct {
- UUID string `json:"uuid"`
- State string `json:"state"`
- Priority int `json:"priority"`
- RuntimeConstraints map[string]int `json:"runtime_constraints"`
+ UUID string `json:"uuid"`
+ State string `json:"state"`
+ Priority int `json:"priority"`
+ RuntimeConstraints map[string]int64 `json:"runtime_constraints"`
}
// ContainerList is a list of the containers from api
@@ -145,7 +145,7 @@ func sbatchFunc(container Container) *exec.Cmd {
return exec.Command("sbatch", "--share", "--parsable",
"--job-name="+container.UUID,
"--mem-per-cpu="+strconv.Itoa(int(memPerCPU)),
- "--cpus-per-task="+strconv.Itoa(container.RuntimeConstraints["vcpus"]))
+ "--cpus-per-task="+strconv.Itoa(int(container.RuntimeConstraints["vcpus"])))
}
var sbatchCmd = sbatchFunc
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 5994cdc..e58b9e4 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -118,7 +118,7 @@ func (s *TestSuite) Test_doMain(c *C) {
sbatchCmdComps := []string{"sbatch", "--share", "--parsable",
fmt.Sprintf("--job-name=%s", item.UUID),
fmt.Sprintf("--mem-per-cpu=%s", strconv.Itoa(int(math.Ceil(float64(item.RuntimeConstraints["ram"])/float64(item.RuntimeConstraints["vcpus"]*1048576))))),
- fmt.Sprintf("--cpus-per-task=%s", strconv.Itoa(item.RuntimeConstraints["vcpus"]))}
+ fmt.Sprintf("--cpus-per-task=%s", strconv.Itoa(int(item.RuntimeConstraints["vcpus"])))}
c.Check(sbatchCmdLine, DeepEquals, sbatchCmdComps)
c.Check(striggerCmdLine, DeepEquals, []string{"strigger", "--set", "--jobid=zzzzz-dz642-queuedcontainer\n", "--fini",
commit 37a00e62fa74ce162ee13d16c731dc10218c5df3
Merge: b5e2bbb 505c7f3
Author: radhika <radhika at curoverse.com>
Date: Tue May 10 11:28:54 2016 -0400
Merge branch '8017-slurm-runtime-constraints' of git.curoverse.com:arvados into 8017-slurm-runtime-constraints
commit b5e2bbb94cd7e121d4b8d83af86ef0ab0c449dcf
Author: radhika <radhika at curoverse.com>
Date: Mon May 9 12:08:15 2016 -0400
8017: mem-per-cpu
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 07653b1..096420f 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -6,6 +6,7 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"io/ioutil"
"log"
+ "math"
"os"
"os/exec"
"os/signal"
@@ -140,9 +141,10 @@ func dispatchSlurm(priorityPollInterval int, crunchRunCommand, finishCommand str
// sbatchCmd
func sbatchFunc(container Container) *exec.Cmd {
+ memPerCPU := math.Ceil(float64(container.RuntimeConstraints["ram"]) / float64(container.RuntimeConstraints["vcpus"]*1048576))
return exec.Command("sbatch", "--share", "--parsable",
"--job-name="+container.UUID,
- "--mem="+strconv.Itoa(container.RuntimeConstraints["ram"]),
+ "--mem-per-cpu="+strconv.Itoa(int(memPerCPU)),
"--cpus-per-task="+strconv.Itoa(container.RuntimeConstraints["vcpus"]))
}
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 82675b2..5994cdc 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -7,6 +7,7 @@ import (
"fmt"
"io/ioutil"
"log"
+ "math"
"net/http"
"net/http/httptest"
"os"
@@ -113,10 +114,11 @@ func (s *TestSuite) Test_doMain(c *C) {
err = doMain()
c.Check(err, IsNil)
+ item := containers.Items[0]
sbatchCmdComps := []string{"sbatch", "--share", "--parsable",
- fmt.Sprintf("--job-name=%s", containers.Items[0].UUID),
- fmt.Sprintf("--mem=%s", strconv.Itoa(containers.Items[0].RuntimeConstraints["ram"])),
- fmt.Sprintf("--cpus-per-task=%s", strconv.Itoa(containers.Items[0].RuntimeConstraints["vcpus"]))}
+ fmt.Sprintf("--job-name=%s", item.UUID),
+ fmt.Sprintf("--mem-per-cpu=%s", strconv.Itoa(int(math.Ceil(float64(item.RuntimeConstraints["ram"])/float64(item.RuntimeConstraints["vcpus"]*1048576))))),
+ fmt.Sprintf("--cpus-per-task=%s", strconv.Itoa(item.RuntimeConstraints["vcpus"]))}
c.Check(sbatchCmdLine, DeepEquals, sbatchCmdComps)
c.Check(striggerCmdLine, DeepEquals, []string{"strigger", "--set", "--jobid=zzzzz-dz642-queuedcontainer\n", "--fini",
commit 1695ccb6279de083eeed80d36d38f607a5c7098d
Author: radhika <radhika at curoverse.com>
Date: Tue May 3 15:09:54 2016 -0400
8017: pass ram and vcpus runtime_constraints from Container to sbatch command.
diff --git a/services/api/test/fixtures/containers.yml b/services/api/test/fixtures/containers.yml
index 22004b4..aa7ad31 100644
--- a/services/api/test/fixtures/containers.yml
+++ b/services/api/test/fixtures/containers.yml
@@ -10,6 +10,9 @@ queued:
output: test
output_path: test
command: ["echo", "hello"]
+ runtime_constraints:
+ ram: 12000000000
+ vcpus: 4
completed:
uuid: zzzzz-dz642-compltcontainer
@@ -23,3 +26,6 @@ completed:
output: test
output_path: test
command: ["echo", "hello"]
+ runtime_constraints:
+ ram: 12000000000
+ vcpus: 4
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
index 8fbc0fa..07653b1 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm.go
@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"os/signal"
+ "strconv"
"sync"
"syscall"
"time"
@@ -106,9 +107,10 @@ func runQueuedContainers(pollInterval, priorityPollInterval int, crunchRunComman
// Container data
type Container struct {
- UUID string `json:"uuid"`
- State string `json:"state"`
- Priority int `json:"priority"`
+ UUID string `json:"uuid"`
+ State string `json:"state"`
+ Priority int `json:"priority"`
+ RuntimeConstraints map[string]int `json:"runtime_constraints"`
}
// ContainerList is a list of the containers from api
@@ -137,8 +139,11 @@ func dispatchSlurm(priorityPollInterval int, crunchRunCommand, finishCommand str
}
// sbatchCmd
-func sbatchFunc(uuid string) *exec.Cmd {
- return exec.Command("sbatch", "--job-name="+uuid, "--share", "--parsable")
+func sbatchFunc(container Container) *exec.Cmd {
+ return exec.Command("sbatch", "--share", "--parsable",
+ "--job-name="+container.UUID,
+ "--mem="+strconv.Itoa(container.RuntimeConstraints["ram"]),
+ "--cpus-per-task="+strconv.Itoa(container.RuntimeConstraints["vcpus"]))
}
var sbatchCmd = sbatchFunc
@@ -170,7 +175,7 @@ func submit(container Container, crunchRunCommand string) (jobid string, submitE
}()
// Create the command and attach to stdin/stdout
- cmd := sbatchCmd(container.UUID)
+ cmd := sbatchCmd(container)
stdinWriter, stdinerr := cmd.StdinPipe()
if stdinerr != nil {
submitErr = fmt.Errorf("Error creating stdin pipe %v: %q", container.UUID, stdinerr)
diff --git a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
index 7355cff..82675b2 100644
--- a/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
+++ b/services/crunch-dispatch-slurm/crunch-dispatch-slurm_test.go
@@ -4,12 +4,14 @@ import (
"git.curoverse.com/arvados.git/sdk/go/arvadosclient"
"git.curoverse.com/arvados.git/sdk/go/arvadostest"
+ "fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"os"
"os/exec"
+ "strconv"
"strings"
"syscall"
"testing"
@@ -68,12 +70,12 @@ func (s *TestSuite) Test_doMain(c *C) {
var striggerCmdLine []string
// Override sbatchCmd
- defer func(orig func(string) *exec.Cmd) {
+ defer func(orig func(Container) *exec.Cmd) {
sbatchCmd = orig
}(sbatchCmd)
- sbatchCmd = func(uuid string) *exec.Cmd {
- sbatchCmdLine = sbatchFunc(uuid).Args
- return exec.Command("echo", uuid)
+ sbatchCmd = func(container Container) *exec.Cmd {
+ sbatchCmdLine = sbatchFunc(container).Args
+ return exec.Command("echo", container.UUID)
}
// Override striggerCmd
@@ -111,7 +113,12 @@ func (s *TestSuite) Test_doMain(c *C) {
err = doMain()
c.Check(err, IsNil)
- c.Check(sbatchCmdLine, DeepEquals, []string{"sbatch", "--job-name=zzzzz-dz642-queuedcontainer", "--share", "--parsable"})
+ sbatchCmdComps := []string{"sbatch", "--share", "--parsable",
+ fmt.Sprintf("--job-name=%s", containers.Items[0].UUID),
+ fmt.Sprintf("--mem=%s", strconv.Itoa(containers.Items[0].RuntimeConstraints["ram"])),
+ fmt.Sprintf("--cpus-per-task=%s", strconv.Itoa(containers.Items[0].RuntimeConstraints["vcpus"]))}
+ c.Check(sbatchCmdLine, DeepEquals, sbatchCmdComps)
+
c.Check(striggerCmdLine, DeepEquals, []string{"strigger", "--set", "--jobid=zzzzz-dz642-queuedcontainer\n", "--fini",
"--program=/usr/bin/crunch-finish-slurm.sh " + os.Getenv("ARVADOS_API_HOST") + " 4axaw8zxe0qm22wa6urpp5nskcne8z88cvbupv653y1njyi05h 1 zzzzz-dz642-queuedcontainer"})
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list