[ARVADOS] created: 1.3.0-595-g4d9a5c4f0
Git user
git at public.curoverse.com
Thu Mar 21 20:55:45 UTC 2019
at 4d9a5c4f0f19c2e6d394dca6a1de903dc09c43e5 (commit)
commit 4d9a5c4f0f19c2e6d394dca6a1de903dc09c43e5
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Mar 21 14:47:53 2019 -0400
14807: Refuse to create instance with AddedScratch>0.
The resulting VM would be the wrong size because dynamically attached
scratch space is not implemented.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/cloud/azure/azure.go b/lib/cloud/azure/azure.go
index be7980ae7..d37183fbd 100644
--- a/lib/cloud/azure/azure.go
+++ b/lib/cloud/azure/azure.go
@@ -341,6 +341,10 @@ func (az *azureInstanceSet) Create(
az.stopWg.Add(1)
defer az.stopWg.Done()
+ if instanceType.AddedScratch > 0 {
+ return nil, fmt.Errorf("cannot create instance type %q: driver does not implement non-zero AddedScratch (%d)", instanceType.Name, instanceType.AddedScratch)
+ }
+
name, err := randutil.String(15, "abcdefghijklmnopqrstuvwxyz0123456789")
if err != nil {
return nil, err
commit 115cbd6482632c47fdcbbbe4abc9543e7e8e30ec
Author: Tom Clegg <tclegg at veritasgenetics.com>
Date: Thu Mar 21 14:36:29 2019 -0400
14807: Load API host/token from cluster config if present.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tclegg at veritasgenetics.com>
diff --git a/lib/dispatchcloud/cmd.go b/lib/dispatchcloud/cmd.go
index 7231e8394..82205c742 100644
--- a/lib/dispatchcloud/cmd.go
+++ b/lib/dispatchcloud/cmd.go
@@ -15,7 +15,11 @@ import (
var Command cmd.Handler = service.Command(arvados.ServiceNameDispatchCloud, newHandler)
func newHandler(ctx context.Context, cluster *arvados.Cluster, _ *arvados.NodeProfile) service.Handler {
- d := &dispatcher{Cluster: cluster, Context: ctx}
+ d := &dispatcher{
+ Cluster: cluster,
+ Context: ctx,
+ AuthToken: service.Token(ctx),
+ }
go d.Start()
return d
}
diff --git a/lib/dispatchcloud/dispatcher.go b/lib/dispatchcloud/dispatcher.go
index 9245d5de3..147b0c015 100644
--- a/lib/dispatchcloud/dispatcher.go
+++ b/lib/dispatchcloud/dispatcher.go
@@ -46,6 +46,7 @@ type pool interface {
type dispatcher struct {
Cluster *arvados.Cluster
Context context.Context
+ AuthToken string
InstanceSetID cloud.InstanceSetID
logger logrus.FieldLogger
@@ -108,7 +109,15 @@ func (disp *dispatcher) setup() {
}
func (disp *dispatcher) initialize() {
- arvClient := arvados.NewClientFromEnv()
+ disp.logger = ctxlog.FromContext(disp.Context)
+
+ arvClient, err := arvados.NewClientFromConfig(disp.Cluster)
+ if err != nil {
+ disp.logger.WithError(err).Warn("error initializing client from cluster config, falling back to ARVADOS_API_HOST(_INSECURE) environment variables")
+ arvClient = arvados.NewClientFromEnv()
+ }
+ arvClient.AuthToken = disp.AuthToken
+
if disp.InstanceSetID == "" {
if strings.HasPrefix(arvClient.AuthToken, "v2/") {
disp.InstanceSetID = cloud.InstanceSetID(strings.Split(arvClient.AuthToken, "/")[1])
@@ -120,7 +129,6 @@ func (disp *dispatcher) initialize() {
}
disp.stop = make(chan struct{}, 1)
disp.stopped = make(chan struct{})
- disp.logger = ctxlog.FromContext(disp.Context)
if key, err := ssh.ParsePrivateKey([]byte(disp.Cluster.Dispatch.PrivateKey)); err != nil {
disp.logger.Fatalf("error parsing configured Dispatch.PrivateKey: %s", err)
diff --git a/lib/dispatchcloud/dispatcher_test.go b/lib/dispatchcloud/dispatcher_test.go
index b0033353c..d7e841e73 100644
--- a/lib/dispatchcloud/dispatcher_test.go
+++ b/lib/dispatchcloud/dispatcher_test.go
@@ -17,6 +17,7 @@ import (
"git.curoverse.com/arvados.git/lib/dispatchcloud/test"
"git.curoverse.com/arvados.git/sdk/go/arvados"
+ "git.curoverse.com/arvados.git/sdk/go/arvadostest"
"git.curoverse.com/arvados.git/sdk/go/ctxlog"
"golang.org/x/crypto/ssh"
check "gopkg.in/check.v1"
@@ -81,10 +82,16 @@ func (s *DispatcherSuite) SetUpTest(c *check.C) {
DispatchCloud: arvados.SystemServiceInstance{Listen: ":"},
},
},
+ Services: arvados.Services{
+ DispatchCloud: arvados.Service{InternalURLs: map[arvados.URL]arvados.ServiceInstance{
+ arvados.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}: {},
+ }},
+ },
}
s.disp = &dispatcher{
- Cluster: s.cluster,
- Context: s.ctx,
+ Cluster: s.cluster,
+ Context: s.ctx,
+ AuthToken: arvadostest.AdminToken,
}
// Test cases can modify s.cluster before calling
// initialize(), and then modify private state before calling
diff --git a/lib/dispatchcloud/test/stub_driver.go b/lib/dispatchcloud/test/stub_driver.go
index 02346a970..873d98732 100644
--- a/lib/dispatchcloud/test/stub_driver.go
+++ b/lib/dispatchcloud/test/stub_driver.go
@@ -245,7 +245,7 @@ func (svm *StubVM) Exec(env map[string]string, command string, stdin io.Reader,
}
for _, name := range []string{"ARVADOS_API_HOST", "ARVADOS_API_TOKEN"} {
if stdinKV[name] == "" {
- fmt.Fprintf(stderr, "%s env var missing from stdin %q\n", name, stdin)
+ fmt.Fprintf(stderr, "%s env var missing from stdin %q\n", name, stdinData)
return 1
}
}
diff --git a/lib/service/cmd.go b/lib/service/cmd.go
index d99af0eea..e56f52eec 100644
--- a/lib/service/cmd.go
+++ b/lib/service/cmd.go
@@ -78,6 +78,17 @@ func (c *command) RunCommand(prog string, args []string, stdin io.Reader, stdout
"PID": os.Getpid(),
})
ctx := ctxlog.Context(context.Background(), log)
+
+ // Currently all components use SystemRootToken if configured,
+ // otherwise ARVADOS_API_TOKEN. In future, per-process tokens
+ // will be generated/obtained here.
+ token := cluster.SystemRootToken
+ if token == "" {
+ log.Warn("SystemRootToken missing from cluster config, falling back to ARVADOS_API_TOKEN environment variable")
+ token = os.Getenv("ARVADOS_API_TOKEN")
+ }
+ ctx = tokenContext(ctx, token)
+
profileName := *nodeProfile
if profileName == "" {
profileName = os.Getenv("ARVADOS_NODE_PROFILE")
diff --git a/lib/service/token.go b/lib/service/token.go
new file mode 100644
index 000000000..5070ae564
--- /dev/null
+++ b/lib/service/token.go
@@ -0,0 +1,30 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package service
+
+import (
+ "context"
+)
+
+type contextKey string
+
+var contextKeyServiceToken contextKey = "serviceToken"
+
+// Token returns the privileged system token suitable for the given
+// service context.
+//
+// It only works on contexts that were generated by Command() and
+// passed to a Handler. For other contexts it returns the empty
+// string.
+func Token(ctx context.Context) string {
+ t, _ := ctx.Value(contextKeyServiceToken).(string)
+ return t
+}
+
+// tokenContext returns a child context with the given token attached
+// so it can be retrieved by Token().
+func tokenContext(ctx context.Context, t string) context.Context {
+ return context.WithValue(ctx, contextKeyServiceToken, t)
+}
diff --git a/sdk/go/arvados/client.go b/sdk/go/arvados/client.go
index 787e01ab8..2b82df9b8 100644
--- a/sdk/go/arvados/client.go
+++ b/sdk/go/arvados/client.go
@@ -69,6 +69,21 @@ var InsecureHTTPClient = &http.Client{
var DefaultSecureClient = &http.Client{
Timeout: 5 * time.Minute}
+// NewClientFromConfig creates a new Client that uses the endpoints in
+// the given cluster.
+//
+// AuthToken is left empty for the caller to populate.
+func NewClientFromConfig(cluster *Cluster) (*Client, error) {
+ ctrlURL := cluster.Services.Controller.ExternalURL
+ if ctrlURL.Host == "" {
+ return nil, fmt.Errorf("no host in config Services.Controller.ExternalURL: %s", ctrlURL)
+ }
+ return &Client{
+ APIHost: fmt.Sprintf("%s", ctrlURL),
+ Insecure: cluster.TLS.Insecure,
+ }, nil
+}
+
// NewClientFromEnv creates a new Client that uses the default HTTP
// client with the API endpoint and credentials given by the
// ARVADOS_API_* environment variables.
diff --git a/sdk/go/arvados/config.go b/sdk/go/arvados/config.go
index 7c87ff029..2965d5ecb 100644
--- a/sdk/go/arvados/config.go
+++ b/sdk/go/arvados/config.go
@@ -8,6 +8,7 @@ import (
"encoding/json"
"errors"
"fmt"
+ "net/url"
"os"
"git.curoverse.com/arvados.git/sdk/go/config"
@@ -58,6 +59,8 @@ type RequestLimits struct {
type Cluster struct {
ClusterID string `json:"-"`
ManagementToken string
+ SystemRootToken string
+ Services Services
NodeProfiles map[string]NodeProfile
InstanceTypes InstanceTypeMap
CloudVMs CloudVMs
@@ -67,8 +70,43 @@ type Cluster struct {
PostgreSQL PostgreSQL
RequestLimits RequestLimits
Logging Logging
+ TLS TLS
}
+type Services struct {
+ Controller Service
+ DispatchCloud Service
+ Health Service
+ Keepbalance Service
+ Keepproxy Service
+ Keepstore Service
+ Keepweb Service
+ Nodemanager Service
+ RailsAPI Service
+ Websocket Service
+ Workbench Service
+}
+
+type Service struct {
+ InternalURLs map[URL]ServiceInstance
+ ExternalURL URL
+}
+
+// URL is a url.URL that is also usable as a JSON key/value.
+type URL url.URL
+
+// UnmarshalText implements encoding.TextUnmarshaler so URL can be
+// used as a JSON key/value.
+func (su *URL) UnmarshalText(text []byte) error {
+ u, err := url.Parse(string(text))
+ if err == nil {
+ *su = URL(*u)
+ }
+ return err
+}
+
+type ServiceInstance struct{}
+
type Logging struct {
Level string
Format string
@@ -309,3 +347,9 @@ type SystemServiceInstance struct {
TLS bool
Insecure bool
}
+
+type TLS struct {
+ Certificate string
+ Key string
+ Insecure bool
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list