[arvados] created: 2.6.0-195-gcefb56587
git repository hosting
git at public.arvados.org
Mon May 22 15:51:54 UTC 2023
at cefb56587ce9343e035e9a3db1d67ad7b3f092ce (commit)
commit cefb56587ce9343e035e9a3db1d67ad7b3f092ce
Author: Tom Clegg <tom at curii.com>
Date: Mon May 22 11:50:57 2023 -0400
19860: Fix incomplete test state reset.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go
index 32192aad7..9d0c378b7 100644
--- a/lib/crunchrun/integration_test.go
+++ b/lib/crunchrun/integration_test.go
@@ -107,8 +107,6 @@ func (s *integrationSuite) SetUpTest(c *C) {
s.engine = "docker"
s.args = nil
s.stdin = bytes.Buffer{}
- s.stdout = bytes.Buffer{}
- s.stderr = bytes.Buffer{}
s.logCollection = arvados.Collection{}
s.outputCollection = arvados.Collection{}
s.logFiles = map[string]string{}
@@ -144,6 +142,8 @@ func (s *integrationSuite) SetUpTest(c *C) {
}
func (s *integrationSuite) setup(c *C) {
+ s.stdout = bytes.Buffer{}
+ s.stderr = bytes.Buffer{}
err := s.client.RequestAndDecode(&s.cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
"priority": s.cr.Priority,
"state": s.cr.State,
commit b8fa738c7a04e0d5138f5b2d56766d6801fcd7a3
Author: Tom Clegg <tom at curii.com>
Date: Mon May 22 10:31:08 2023 -0400
19860: Document "docker pull" builtin command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/api/methods/container_requests.html.textile.liquid b/doc/api/methods/container_requests.html.textile.liquid
index c108c3280..d1d8f89e5 100644
--- a/doc/api/methods/container_requests.html.textile.liquid
+++ b/doc/api/methods/container_requests.html.textile.liquid
@@ -44,7 +44,7 @@ table(table table-bordered table-condensed).
|scheduling_parameters|hash|Parameters to be passed to the container scheduler when running this container.|e.g.,<pre><code>{
"partitions":["fastcpu","vfastcpu"]
}</code></pre>See "Scheduling parameters":#scheduling_parameters for more details.|
-|container_image|string|Portable data hash of a collection containing the docker image to run the container.|Required.|
+|container_image|string|Name (@repo@ or @repo:tag@) or portable data hash of a collection containing the docker image to run the container. If the image is specified by name, the image must have been stored in Arvados using @arv keep docker@ or an "image pull request":#pull_image.|Required. e.g., @arvados/jobs@, @alpine:latest@|
|environment|hash|Environment variables and values that should be set in the container environment (@docker run --env@). This augments and (when conflicts exist) overrides environment variables given in the image's Dockerfile.||
|cwd|string|Initial working directory, given as an absolute path (in the container) or a path relative to the WORKDIR given in the image's Dockerfile.|Required.|
|command|array of strings|Command to execute in the container.|Required. e.g., @["echo","hello"]@|
@@ -159,6 +159,33 @@ A container request may be canceled by setting its priority to 0, using an updat
When a container request is canceled, it will still reflect the state of the Container it is associated with via the container_uuid attribute. If that Container is being reused by any other container_requests that are still active, i.e., not yet canceled, that Container may continue to run or be scheduled to run by the system in future. However, if no other container_requests are using that Container, then the Container will get canceled as well.
+h2(#pull_image). Pulling a docker image
+
+To download an existing Docker image from Docker Hub to Arvados, submit a container request with a @docker pull@ command, using the special container image name @arvados/builtin@:
+
+<pre>
+{
+ "container_image": "arvados/builtin",
+ "command": ["docker", "pull", "alpine:latest"],
+ "mounts": {},
+ "output_path": "/",
+ "runtime_constraints": {
+ "API": true
+ },
+ ...
+}
+</pre>
+
+The downloaded image can then be used as a container_image in subsequent container requests.
+
+<pre>
+{
+ "container_image": "alpine:latest",
+ "command": ["echo", "ok"],
+ ...
+}
+</pre>
+
h2. Methods
See "Common resource methods":{{site.baseurl}}/api/methods.html for more information about @create@, @delete@, @get@, @list@, and @update at .
commit 60b713410fbab57932bc58da5a3f1a8bf9047c8b
Author: Tom Clegg <tom at curii.com>
Date: Mon May 22 10:03:45 2023 -0400
19860: Fix singularity-version-sensitive test case.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/executor_test.go b/lib/crunchrun/executor_test.go
index f1a873ae8..91a957d2d 100644
--- a/lib/crunchrun/executor_test.go
+++ b/lib/crunchrun/executor_test.go
@@ -134,6 +134,8 @@ func (s *executorSuite) TestExecCleanEnv(c *C) {
// singularity also sets this by itself (v3.5.2, but not v3.7.4)
case "PROMPT_COMMAND", "PS1", "SINGULARITY_BIND", "SINGULARITY_COMMAND", "SINGULARITY_ENVIRONMENT":
// singularity also sets these by itself (v3.7.4)
+ case "SINGULARITY_NO_EVAL":
+ // singularity redacts this (v3.10) or not (earlier)
default:
got[kv[0]] = kv[1]
}
commit 91c418b19bf12a7f0801afa6d24f0050d8f76b8e
Author: Tom Clegg <tom at curii.com>
Date: Fri May 19 14:48:04 2023 -0400
19860: Add arvados/builtin pseudo-image with "docker pull" command.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/crunchrun/crunchrun.go b/lib/crunchrun/crunchrun.go
index 4a514f3d8..d5ead6aa7 100644
--- a/lib/crunchrun/crunchrun.go
+++ b/lib/crunchrun/crunchrun.go
@@ -1721,6 +1721,24 @@ func (runner *ContainerRunner) Run() (err error) {
runner.hoststatReporter.ReportPID("keepstore", runner.keepstore.Process.Pid)
}
+ err = runner.LogHostInfo()
+ if err != nil {
+ return
+ }
+ err = runner.LogNodeRecord()
+ if err != nil {
+ return
+ }
+ err = runner.LogContainerRecord()
+ if err != nil {
+ return
+ }
+
+ if runner.Container.ContainerImage == "arvados/builtin" {
+ err = runner.runBuiltinCommand()
+ return
+ }
+
// set up FUSE mount and binds
bindmounts, err = runner.SetupMounts()
if err != nil {
@@ -1745,18 +1763,6 @@ func (runner *ContainerRunner) Run() (err error) {
if err != nil {
return
}
- err = runner.LogHostInfo()
- if err != nil {
- return
- }
- err = runner.LogNodeRecord()
- if err != nil {
- return
- }
- err = runner.LogContainerRecord()
- if err != nil {
- return
- }
if runner.IsCancelled() {
return
diff --git a/lib/crunchrun/crunchrun_test.go b/lib/crunchrun/crunchrun_test.go
index 9c4fe20bc..17d24c48f 100644
--- a/lib/crunchrun/crunchrun_test.go
+++ b/lib/crunchrun/crunchrun_test.go
@@ -118,6 +118,7 @@ type KeepTestClient struct {
type stubExecutor struct {
imageLoaded bool
+ pullErr error
loaded string
loadErr error
exitCode int
@@ -156,6 +157,9 @@ func (e *stubExecutor) InjectCommand(ctx context.Context, _, _ string, _ bool, _
return nil, errors.New("unimplemented")
}
func (e *stubExecutor) IPAddress() (string, error) { return "", errors.New("unimplemented") }
+func (e *stubExecutor) PullImage(context.Context, string) (io.ReadCloser, string, error) {
+ return nil, "", e.pullErr
+}
const fakeInputCollectionPDH = "ffffffffaaaaaaaa88888888eeeeeeee+1234"
diff --git a/lib/crunchrun/docker.go b/lib/crunchrun/docker.go
index 8d8cdfc8b..16ceef4a1 100644
--- a/lib/crunchrun/docker.go
+++ b/lib/crunchrun/docker.go
@@ -77,6 +77,29 @@ func (e *dockerExecutor) Runtime() string {
return "docker " + info
}
+func (e *dockerExecutor) PullImage(ctx context.Context, repotag string) (imageData io.ReadCloser, imageID string, err error) {
+ out, err := e.dockerclient.ImagePull(ctx, repotag, dockertypes.ImagePullOptions{})
+ if err != nil {
+ return nil, "", fmt.Errorf("ImagePull: %w", err)
+ }
+ defer out.Close()
+ buf, err := ioutil.ReadAll(out)
+ if err != nil {
+ return nil, "", fmt.Errorf("reading response: %w", err)
+ }
+ e.logf("%s", buf)
+
+ inspect, _, err := e.dockerclient.ImageInspectWithRaw(context.TODO(), repotag)
+ if err != nil {
+ return nil, "", err
+ }
+ imagedata, err := e.dockerclient.ImageSave(ctx, []string{inspect.ID})
+ if err != nil {
+ return nil, "", err
+ }
+ return imagedata, inspect.ID, nil
+}
+
func (e *dockerExecutor) LoadImage(imageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
containerClient *arvados.Client) error {
_, _, err := e.dockerclient.ImageInspectWithRaw(context.TODO(), imageID)
diff --git a/lib/crunchrun/executor.go b/lib/crunchrun/executor.go
index 6ec5b838f..685138bee 100644
--- a/lib/crunchrun/executor.go
+++ b/lib/crunchrun/executor.go
@@ -35,6 +35,11 @@ type containerSpec struct {
// containerExecutor is an interface to a container runtime
// (docker/singularity).
type containerExecutor interface {
+ // Pull the specified image (repo:tag) from docker hub. Return
+ // a reader that reads the image tarball, and the expected
+ // image hash.
+ PullImage(context.Context, string) (io.ReadCloser, string, error)
+
// ImageLoad loads the image from the given tarball such that
// it can be used to create/start a container.
LoadImage(imageID string, imageTarballPath string, container arvados.Container, keepMount string,
diff --git a/lib/crunchrun/integration_test.go b/lib/crunchrun/integration_test.go
index d56902082..32192aad7 100644
--- a/lib/crunchrun/integration_test.go
+++ b/lib/crunchrun/integration_test.go
@@ -351,3 +351,105 @@ func (s *integrationSuite) testRunTrivialContainer(c *C) {
}
s.outputCollection = output
}
+
+func (s *integrationSuite) TestBuiltinPullImage(c *C) {
+ s.engine = "docker"
+ if err := exec.Command("which", s.engine).Run(); err != nil {
+ c.Skip(fmt.Sprintf("%s: %s", s.engine, err))
+ }
+ s.cr.Command = []string{"docker", "pull", "alpine:latest"}
+ s.cr.Mounts = nil
+ s.cr.ContainerImage = "arvados/builtin"
+ s.cr.OutputPath = "/"
+ s.setup(c)
+ args := []string{
+ "-runtime-engine=" + s.engine,
+ "-enable-memory-limit=false",
+ s.cr.ContainerUUID,
+ }
+ code := command{}.RunCommand("crunch-run", args, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
+ c.Logf("\n===== stdout =====\n%s", s.stdout.String())
+ c.Logf("\n===== stderr =====\n%s", s.stderr.String())
+ c.Check(code, Equals, 0)
+ err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Check(s.cr.State, Equals, arvados.ContainerRequestStateFinal)
+ var ctr arvados.Container
+ err = s.client.RequestAndDecode(&ctr, "GET", "arvados/v1/containers/"+s.cr.ContainerUUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Check(ctr.State, Equals, arvados.ContainerStateComplete)
+ c.Check(ctr.ExitCode, Equals, 0)
+ var outcoll arvados.Collection
+ err = s.client.RequestAndDecode(&outcoll, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Check(outcoll.Properties["docker-image-repo-tag"], Equals, "alpine:latest")
+ c.Check(outcoll.Properties["docker-image-hash"], Matches, `sha256:[a-f0-9]{64}`)
+ c.Check(outcoll.IsTrashed, Equals, false)
+ c.Check(outcoll.TrashAt, IsNil)
+
+ var links arvados.LinkList
+ err = s.client.RequestAndDecode(&links, "GET", "arvados/v1/links", nil, map[string]interface{}{
+ "filters": []arvados.Filter{{"head_uuid", "=", outcoll.UUID}},
+ "order": "link_class",
+ })
+ c.Assert(err, IsNil)
+ c.Assert(links.Items, HasLen, 2)
+ c.Check(links.Items[0].LinkClass, Equals, "docker_image_hash")
+ c.Check(links.Items[0].HeadUUID, Equals, outcoll.UUID)
+ c.Check(links.Items[0].Name, Matches, `sha256:[a-f0-9]{64}`)
+ c.Check(links.Items[1].LinkClass, Equals, "docker_image_repo+tag")
+ c.Check(links.Items[1].HeadUUID, Equals, outcoll.UUID)
+ c.Check(links.Items[1].Name, Equals, "alpine:latest")
+
+ c.Logf("Pull succeeded, docker image hash is %s", links.Items[0].Properties["docker_image_hash"])
+
+ // Use the output to run a container. We can reference it by
+ // either PDH (output of the pull CR) or repo:tag (tag links
+ // added by railsapi).
+
+ for _, image := range []string{
+ outcoll.PortableDataHash,
+ "alpine:latest",
+ } {
+ c.Logf("===== running container, specifying pulled image as %q =====", image)
+ s.cr = arvados.ContainerRequest{
+ Priority: 1,
+ State: "Committed",
+ OutputPath: "/mnt/out",
+ ContainerImage: image,
+ Command: []string{"sh", "-c", "echo ok >/mnt/out/ok"},
+ Mounts: map[string]arvados.Mount{
+ "/mnt/out": {
+ Kind: "tmp",
+ Capacity: 1000,
+ },
+ },
+ RuntimeConstraints: arvados.RuntimeConstraints{
+ RAM: 128000000,
+ VCPUs: 1,
+ API: true,
+ },
+ }
+ s.setup(c)
+ args := []string{
+ "-runtime-engine=" + s.engine,
+ "-enable-memory-limit=false",
+ s.cr.ContainerUUID,
+ }
+ code := command{}.RunCommand("crunch-run", args, &s.stdin, io.MultiWriter(&s.stdout, os.Stderr), io.MultiWriter(&s.stderr, os.Stderr))
+ c.Logf("\n===== stdout =====\n%s", s.stdout.String())
+ c.Logf("\n===== stderr =====\n%s", s.stderr.String())
+ c.Check(code, Equals, 0)
+ err := s.client.RequestAndDecode(&s.cr, "GET", "arvados/v1/container_requests/"+s.cr.UUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Check(s.cr.State, Equals, arvados.ContainerRequestStateFinal)
+ var ctr2 arvados.Container
+ err = s.client.RequestAndDecode(&ctr2, "GET", "arvados/v1/containers/"+s.cr.ContainerUUID, nil, nil)
+ c.Assert(err, IsNil)
+ c.Check(ctr2.State, Equals, arvados.ContainerStateComplete)
+ c.Check(ctr2.ExitCode, Equals, 0)
+ var outcoll2 arvados.Collection
+ err = s.client.RequestAndDecode(&outcoll2, "GET", "arvados/v1/collections/"+s.cr.OutputUUID, nil, nil)
+ c.Check(err, IsNil)
+ }
+}
diff --git a/lib/crunchrun/pull.go b/lib/crunchrun/pull.go
new file mode 100644
index 000000000..250610895
--- /dev/null
+++ b/lib/crunchrun/pull.go
@@ -0,0 +1,93 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+package crunchrun
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "os"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+func (runner *ContainerRunner) runBuiltinCommand() error {
+ err := runner.UpdateContainerRunning("")
+ if err != nil {
+ return err
+ }
+ exitCode := 1
+ runner.ExitCode = &exitCode
+ runner.finalState = string(arvados.ContainerStateComplete)
+ if len(runner.Container.Command) == 3 && runner.Container.Command[0] == "docker" && runner.Container.Command[1] == "pull" {
+ repotag := runner.Container.Command[2]
+ outcoll, err := pullImageAndSaveCollection(runner.Container.UUID, runner.executor, repotag, runner.containerClient, runner.ContainerKeepClient)
+ if err != nil {
+ return err
+ }
+ runner.OutputPDH = &outcoll.PortableDataHash
+ exitCode = 0
+ return nil
+ }
+ return fmt.Errorf("unsupported builtin command %v", runner.Container.Command)
+}
+
+func pullImageAndSaveCollection(ctrUUID string, executor containerExecutor, repotag string, arvClient *arvados.Client, keepClient IKeepClient) (outcoll arvados.Collection, err error) {
+ outfs, err := outcoll.FileSystem(arvClient, keepClient)
+ if err != nil {
+ return outcoll, fmt.Errorf("error creating filesystem: %w", err)
+ }
+
+ imagedata, imagehash, err := executor.PullImage(context.TODO(), repotag)
+ if err != nil {
+ return outcoll, fmt.Errorf("error pulling image: %w", err)
+ }
+ tarfile, err := outfs.OpenFile(imagehash+".tar", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0777)
+ if err != nil {
+ return outcoll, fmt.Errorf("error opening file to save image: %w", err)
+ }
+ defer tarfile.Close()
+
+ _, err = io.Copy(tarfile, imagedata)
+ if err != nil {
+ return outcoll, fmt.Errorf("error saving image data: %w", err)
+ }
+ err = imagedata.Close()
+ if err != nil {
+ return outcoll, fmt.Errorf("error closing image data reader: %w", err)
+ }
+ err = tarfile.Close()
+ if err != nil {
+ return outcoll, fmt.Errorf("error closing image file: %w", err)
+ }
+ outcoll.ManifestText, err = outfs.MarshalManifest(".")
+ if err != nil {
+ return outcoll, fmt.Errorf("error saving image collection manifest: %w", err)
+ }
+ err = arvClient.RequestAndDecode(&outcoll, "POST", "arvados/v1/collections", nil, map[string]interface{}{
+ "collection": map[string]interface{}{
+ "manifest_text": outcoll.ManifestText,
+ "is_trashed": true,
+ }})
+ if err != nil {
+ return outcoll, fmt.Errorf("error saving image collection: %w", err)
+ }
+ // Now we update the container properties with the repo:tag
+ // and hash. RailsAPI will use these values when creating the
+ // container request output collection during finalize.
+ //
+ // Additionally, RailsAPI has "arvados/builtin"-specific code
+ // to create a tag link with these values, pointing to the
+ // CR's output collection.
+ err = arvClient.RequestAndDecode(nil, "PATCH", "arvados/v1/containers/"+ctrUUID, nil, map[string]interface{}{
+ "container": map[string]interface{}{
+ "output_properties": map[string]interface{}{
+ "docker-image-repo-tag": repotag,
+ "docker-image-hash": imagehash,
+ }}})
+ if err != nil {
+ return outcoll, err
+ }
+ return outcoll, nil
+}
diff --git a/lib/crunchrun/singularity.go b/lib/crunchrun/singularity.go
index 8c0d8f5bc..565188ebe 100644
--- a/lib/crunchrun/singularity.go
+++ b/lib/crunchrun/singularity.go
@@ -9,6 +9,7 @@ import (
"context"
"errors"
"fmt"
+ "io"
"io/ioutil"
"net"
"os"
@@ -139,7 +140,11 @@ func (e *singularityExecutor) checkImageCache(dockerImageID string, container ar
return &imageCollection, nil
}
-// LoadImage will satisfy ContainerExecuter interface transforming
+func (e *singularityExecutor) PullImage(context.Context, string) (io.ReadCloser, string, error) {
+ return nil, "", errors.New("image pull is not supported by the singularity executor")
+}
+
+// LoadImage will satisfy ContainerExecutor interface transforming
// containerImage into a sif file for later use.
func (e *singularityExecutor) LoadImage(dockerImageID string, imageTarballPath string, container arvados.Container, arvMountPoint string,
containerClient *arvados.Client) error {
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index d897ff7af..b3a7011fe 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -233,8 +233,11 @@ class Container < ArvadosModel
return c_mounts
end
- # Return a container_image PDH suitable for a Container.
+ # Resolve an image specification found in a container request (UUID,
+ # PDH, repo:tag, or "arvados/builtin") to an image specification
+ # suitable for a Container (PDH or "arvados/builtin").
def self.resolve_container_image(container_image)
+ return "arvados/builtin" if container_image == "arvados/builtin"
coll = Collection.for_latest_docker_image(container_image)
if !coll
raise ArvadosModel::UnresolvableContainerError.new "docker image #{container_image.inspect} not found"
diff --git a/services/api/app/models/container_request.rb b/services/api/app/models/container_request.rb
index 3c3896771..b07a4c53e 100644
--- a/services/api/app/models/container_request.rb
+++ b/services/api/app/models/container_request.rb
@@ -39,6 +39,7 @@ class ContainerRequest < ArvadosModel
validates :command, :container_image, :output_path, :cwd, :presence => true
validates :output_ttl, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
validates :priority, numericality: { only_integer: true, greater_than_or_equal_to: 0, less_than_or_equal_to: 1000 }
+ validate :validate_builtin_command
validate :validate_datatypes
validate :validate_runtime_constraints
validate :validate_scheduling_parameters
@@ -293,6 +294,22 @@ class ContainerRequest < ArvadosModel
properties: merged_properties)
coll.save_with_unique_name!
self.send(out_type + '_uuid=', coll.uuid)
+
+ if out_type == 'output' &&
+ container_image == 'arvados/builtin' &&
+ command[0..1] == ['docker', 'pull'] &&
+ container.exit_code == 0
+ Link.create!(
+ head_uuid: coll.uuid,
+ link_class: 'docker_image_repo+tag',
+ name: container.output_properties['docker-image-repo-tag'],
+ )
+ Link.create!(
+ head_uuid: coll.uuid,
+ link_class: 'docker_image_hash',
+ name: container.output_properties['docker-image-hash'],
+ )
+ end
end
end
@@ -392,6 +409,23 @@ class ContainerRequest < ArvadosModel
end
end
+ def validate_builtin_command
+ return if container_image != "arvados/builtin"
+ if command.length == 3 && command[0..1] == ["docker", "pull"]
+ if !mounts.empty?
+ errors.add(:mounts, "must be empty for builtin docker pull command")
+ end
+ if !runtime_constraints['API']
+ errors.add(:runtime_constraints, "API flag must be set for builtin docker pull command")
+ end
+ if output_path != "/"
+ errors.add(:output_path, "must be '/' for builtin docker pull command")
+ end
+ else
+ errors.add(:command, "is not a valid builtin command")
+ end
+ end
+
def validate_runtime_constraints
case self.state
when Committed
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list