[arvados] created: 2.5.0-222-gab3837094
git repository hosting
git at public.arvados.org
Sat Mar 4 09:19:58 UTC 2023
at ab38370942e87b2b1a744ab63e8957a6a474fd7e (commit)
commit ab38370942e87b2b1a744ab63e8957a6a474fd7e
Author: Tom Clegg <tom at curii.com>
Date: Sat Mar 4 04:15:00 2023 -0500
20183: Deduplicate test suite setup.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index d4b60baa0..241dc246f 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -14,14 +14,10 @@ import (
"strings"
"time"
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/lib/controller/rpc"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/keepclient"
check "gopkg.in/check.v1"
)
@@ -29,53 +25,7 @@ import (
var _ = check.Suite(&CollectionSuite{})
type CollectionSuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- railsSpy *arvadostest.Proxy
-}
-
-func (s *CollectionSuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
-
-func (s *CollectionSuite) SetUpTest(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
- s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
-}
-
-func (s *CollectionSuite) TearDownTest(c *check.C) {
- s.railsSpy.Close()
-}
-
-func (s *CollectionSuite) setUpVocabulary(c *check.C, testVocabulary string) {
- if testVocabulary == "" {
- testVocabulary = `{
- "strict_tags": false,
- "tags": {
- "IDTAGIMPORTANCES": {
- "strict": true,
- "labels": [{"label": "Importance"}, {"label": "Priority"}],
- "values": {
- "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
- "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
- "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
- }
- }
- }
- }`
- }
- voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
- c.Assert(err, check.IsNil)
- s.cluster.API.VocabularyPath = "foo"
- s.localdb.vocabularyCache = voc
+ localdbSuite
}
func (s *CollectionSuite) TestCollectionCreateAndUpdateWithProperties(c *check.C) {
diff --git a/lib/controller/localdb/container_gateway_test.go b/lib/controller/localdb/container_gateway_test.go
index 92065c9ed..ad3136fbc 100644
--- a/lib/controller/localdb/container_gateway_test.go
+++ b/lib/controller/localdb/container_gateway_test.go
@@ -17,11 +17,9 @@ import (
"strings"
"time"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/router"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/lib/crunchrun"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -33,27 +31,14 @@ import (
var _ = check.Suite(&ContainerGatewaySuite{})
type ContainerGatewaySuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- ctx context.Context
+ localdbSuite
ctrUUID string
gw *crunchrun.Gateway
}
-func (s *ContainerGatewaySuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
-
-func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
- s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
- s.ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
+ s.localdbSuite.SetUpTest(c)
+ s.ctx = auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
s.ctrUUID = arvadostest.QueuedContainerUUID
@@ -83,21 +68,14 @@ func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
ArvadosClient: ac,
}
c.Assert(s.gw.Start(), check.IsNil)
- rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
- _, err = s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
+ rootctx := auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
+ // OK if this line fails (because state is already Running
+ // from a previous test case) as long as the following line
+ // succeeds:
+ s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
UUID: s.ctrUUID,
Attrs: map[string]interface{}{
"state": arvados.ContainerStateLocked}})
- c.Assert(err, check.IsNil)
-}
-
-func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
- // clear any tunnel sessions started by previous test cases
- s.localdb.gwTunnelsLock.Lock()
- s.localdb.gwTunnels = nil
- s.localdb.gwTunnelsLock.Unlock()
-
- rootctx := auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{s.cluster.SystemRootToken}})
_, err := s.localdb.ContainerUpdate(rootctx, arvados.UpdateOptions{
UUID: s.ctrUUID,
Attrs: map[string]interface{}{
@@ -107,7 +85,7 @@ func (s *ContainerGatewaySuite) SetUpTest(c *check.C) {
s.cluster.Containers.ShellAccess.Admin = true
s.cluster.Containers.ShellAccess.User = true
- _, err = arvadostest.DB(c, s.cluster).Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
+ _, err = s.db.Exec(`update containers set interactive_session_started=$1 where uuid=$2`, false, s.ctrUUID)
c.Check(err, check.IsNil)
}
diff --git a/lib/controller/localdb/container_request_test.go b/lib/controller/localdb/container_request_test.go
index 2d89f58ab..45b6de453 100644
--- a/lib/controller/localdb/container_request_test.go
+++ b/lib/controller/localdb/container_request_test.go
@@ -7,66 +7,16 @@ package localdb
import (
"context"
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/lib/controller/rpc"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&ContainerRequestSuite{})
type ContainerRequestSuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- railsSpy *arvadostest.Proxy
-}
-
-func (s *ContainerRequestSuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
-
-func (s *ContainerRequestSuite) SetUpTest(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
- s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
-}
-
-func (s *ContainerRequestSuite) TearDownTest(c *check.C) {
- s.railsSpy.Close()
-}
-
-func (s *ContainerRequestSuite) setUpVocabulary(c *check.C, testVocabulary string) {
- if testVocabulary == "" {
- testVocabulary = `{
- "strict_tags": false,
- "tags": {
- "IDTAGIMPORTANCES": {
- "strict": true,
- "labels": [{"label": "Importance"}, {"label": "Priority"}],
- "values": {
- "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
- "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
- "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
- }
- }
- }
- }`
- }
- voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
- c.Assert(err, check.IsNil)
- s.localdb.vocabularyCache = voc
- s.cluster.API.VocabularyPath = "foo"
+ localdbSuite
}
func (s *ContainerRequestSuite) TestCRCreateWithProperties(c *check.C) {
diff --git a/lib/controller/localdb/group_test.go b/lib/controller/localdb/group_test.go
index ff80dd500..aa44486ff 100644
--- a/lib/controller/localdb/group_test.go
+++ b/lib/controller/localdb/group_test.go
@@ -7,63 +7,16 @@ package localdb
import (
"context"
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/lib/controller/rpc"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&GroupSuite{})
type GroupSuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- railsSpy *arvadostest.Proxy
-}
-
-func (s *GroupSuite) SetUpSuite(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
- s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
-}
-
-func (s *GroupSuite) TearDownSuite(c *check.C) {
- s.railsSpy.Close()
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
-
-func (s *GroupSuite) setUpVocabulary(c *check.C, testVocabulary string) {
- if testVocabulary == "" {
- testVocabulary = `{
- "strict_tags": false,
- "tags": {
- "IDTAGIMPORTANCES": {
- "strict": true,
- "labels": [{"label": "Importance"}, {"label": "Priority"}],
- "values": {
- "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
- "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
- "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
- }
- }
- }
- }`
- }
- voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
- c.Assert(err, check.IsNil)
- s.localdb.vocabularyCache = voc
- s.cluster.API.VocabularyPath = "foo"
+ localdbSuite
}
func (s *GroupSuite) TestGroupCreateWithProperties(c *check.C) {
diff --git a/lib/controller/localdb/link_test.go b/lib/controller/localdb/link_test.go
index f28b32eb0..7f0a30af6 100644
--- a/lib/controller/localdb/link_test.go
+++ b/lib/controller/localdb/link_test.go
@@ -7,66 +7,16 @@ package localdb
import (
"context"
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/lib/controller/rpc"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&LinkSuite{})
type LinkSuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- railsSpy *arvadostest.Proxy
-}
-
-func (s *LinkSuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
-
-func (s *LinkSuite) SetUpTest(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
- s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
-}
-
-func (s *LinkSuite) TearDownTest(c *check.C) {
- s.railsSpy.Close()
-}
-
-func (s *LinkSuite) setUpVocabulary(c *check.C, testVocabulary string) {
- if testVocabulary == "" {
- testVocabulary = `{
- "strict_tags": false,
- "tags": {
- "IDTAGIMPORTANCES": {
- "strict": true,
- "labels": [{"label": "Importance"}, {"label": "Priority"}],
- "values": {
- "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
- "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
- "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
- }
- }
- }
- }`
- }
- voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
- c.Assert(err, check.IsNil)
- s.localdb.vocabularyCache = voc
- s.cluster.API.VocabularyPath = "foo"
+ localdbSuite
}
func (s *LinkSuite) TestLinkCreateWithProperties(c *check.C) {
diff --git a/lib/controller/localdb/localdb_test.go b/lib/controller/localdb/localdb_test.go
new file mode 100644
index 000000000..3e7c7421c
--- /dev/null
+++ b/lib/controller/localdb/localdb_test.go
@@ -0,0 +1,91 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/jmoiron/sqlx"
+ check "gopkg.in/check.v1"
+)
+
+type localdbSuite struct {
+ ctx context.Context
+ cancel context.CancelFunc
+ cluster *arvados.Cluster
+ db *sqlx.DB
+ dbConnector *ctrlctx.DBConnector
+ tx *sqlx.Tx
+ localdb *Conn
+ railsSpy *arvadostest.Proxy
+}
+
+func (s *localdbSuite) TearDownSuite(c *check.C) {
+ // Undo any changes/additions to the user database so they
+ // don't affect subsequent tests.
+ arvadostest.ResetEnv()
+ c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+}
+
+func (s *localdbSuite) SetUpTest(c *check.C) {
+ *s = localdbSuite{}
+ s.ctx, s.cancel = context.WithCancel(context.Background())
+ cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
+ c.Assert(err, check.IsNil)
+ s.cluster, err = cfg.GetCluster("")
+ c.Assert(err, check.IsNil)
+ s.dbConnector = &ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}
+ s.db, err = s.dbConnector.GetDB(s.ctx)
+ c.Assert(err, check.IsNil)
+ s.localdb = NewConn(s.ctx, s.cluster, s.dbConnector.GetDB)
+ s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
+ *s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
+
+ s.tx, err = s.db.Beginx()
+ c.Assert(err, check.IsNil)
+ s.ctx = ctrlctx.NewWithTransaction(s.ctx, s.tx)
+}
+
+func (s *localdbSuite) TearDownTest(c *check.C) {
+ if s.tx != nil {
+ s.tx.Rollback()
+ }
+ if s.railsSpy != nil {
+ s.railsSpy.Close()
+ }
+ if s.dbConnector != nil {
+ s.dbConnector.Close()
+ }
+ s.cancel()
+}
+
+func (s *localdbSuite) setUpVocabulary(c *check.C, testVocabulary string) {
+ if testVocabulary == "" {
+ testVocabulary = `{
+ "strict_tags": false,
+ "tags": {
+ "IDTAGIMPORTANCES": {
+ "strict": true,
+ "labels": [{"label": "Importance"}, {"label": "Priority"}],
+ "values": {
+ "IDVALIMPORTANCES1": { "labels": [{"label": "Critical"}, {"label": "Urgent"}, {"label": "High"}] },
+ "IDVALIMPORTANCES2": { "labels": [{"label": "Normal"}, {"label": "Moderate"}] },
+ "IDVALIMPORTANCES3": { "labels": [{"label": "Low"}] }
+ }
+ }
+ }
+ }`
+ }
+ voc, err := arvados.NewVocabulary([]byte(testVocabulary), []string{})
+ c.Assert(err, check.IsNil)
+ s.localdb.vocabularyCache = voc
+ s.cluster.API.VocabularyPath = "foo"
+}
diff --git a/lib/controller/localdb/log_activity_test.go b/lib/controller/localdb/log_activity_test.go
index ea7f234cc..b52bb162e 100644
--- a/lib/controller/localdb/log_activity_test.go
+++ b/lib/controller/localdb/log_activity_test.go
@@ -58,9 +58,8 @@ func (s *CollectionSuite) TestLogActivity(c *check.C) {
s.localdb.activeUsersLock.Lock()
s.localdb.activeUsersReset = starttime
s.localdb.activeUsersLock.Unlock()
- db := arvadostest.DB(c, s.cluster)
wrap := api.ComposeWrappers(
- ctrlctx.WrapCallsInTransactions(func(ctx context.Context) (*sqlx.DB, error) { return db, nil }),
+ ctrlctx.WrapCallsInTransactions(func(ctx context.Context) (*sqlx.DB, error) { return s.db, nil }),
ctrlctx.WrapCallsWithAuth(s.cluster))
collectionCreate := wrap(func(ctx context.Context, opts interface{}) (interface{}, error) {
return s.localdb.CollectionCreate(ctx, opts.(arvados.CreateOptions))
@@ -76,7 +75,7 @@ func (s *CollectionSuite) TestLogActivity(c *check.C) {
})
c.Assert(err, check.IsNil)
var uuid string
- err = db.QueryRowContext(ctx, `select uuid from logs where object_uuid = $1 and event_at > $2`, arvadostest.ActiveUserUUID, logthreshold.UTC()).Scan(&uuid)
+ err = s.db.QueryRowContext(ctx, `select uuid from logs where object_uuid = $1 and event_at > $2`, arvadostest.ActiveUserUUID, logthreshold.UTC()).Scan(&uuid)
if i == 0 {
c.Check(err, check.IsNil)
c.Check(uuid, check.HasLen, 27)
diff --git a/lib/controller/localdb/login_ldap_test.go b/lib/controller/localdb/login_ldap_test.go
index 1487d46f2..69b7f5780 100644
--- a/lib/controller/localdb/login_ldap_test.go
+++ b/lib/controller/localdb/login_ldap_test.go
@@ -5,48 +5,27 @@
package localdb
import (
- "context"
"encoding/json"
"net"
"net/http"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/railsproxy"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/bradleypeabody/godap"
- "github.com/jmoiron/sqlx"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&LDAPSuite{})
type LDAPSuite struct {
- cluster *arvados.Cluster
- ctrl *ldapLoginController
- ldap *godap.LDAPServer // fake ldap server that accepts auth goodusername/goodpassword
- db *sqlx.DB
-
- // transaction context
- ctx context.Context
- rollback func() error
-}
-
-func (s *LDAPSuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
+ localdbSuite
+ ldap *godap.LDAPServer // fake ldap server that accepts auth goodusername/goodpassword
}
-func (s *LDAPSuite) SetUpSuite(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
+func (s *LDAPSuite) SetUpTest(c *check.C) {
+ s.localdbSuite.SetUpTest(c)
ln, err := net.Listen("tcp", "127.0.0.1:0")
c.Assert(err, check.IsNil)
@@ -84,35 +63,19 @@ func (s *LDAPSuite) SetUpSuite(c *check.C) {
s.cluster.Login.LDAP.Enable = true
err = json.Unmarshal([]byte(`"ldap://`+ln.Addr().String()+`"`), &s.cluster.Login.LDAP.URL)
+ c.Assert(err, check.IsNil)
s.cluster.Login.LDAP.StartTLS = false
s.cluster.Login.LDAP.SearchBindUser = "cn=goodusername,dc=example,dc=com"
s.cluster.Login.LDAP.SearchBindPassword = "goodpassword"
s.cluster.Login.LDAP.SearchBase = "dc=example,dc=com"
- c.Assert(err, check.IsNil)
- s.ctrl = &ldapLoginController{
+ s.localdb.loginController = &ldapLoginController{
Cluster: s.cluster,
- Parent: &Conn{railsProxy: railsproxy.NewConn(s.cluster)},
- }
- s.db = arvadostest.DB(c, s.cluster)
-}
-
-func (s *LDAPSuite) SetUpTest(c *check.C) {
- tx, err := s.db.Beginx()
- c.Assert(err, check.IsNil)
- s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx)
- s.rollback = tx.Rollback
-}
-
-func (s *LDAPSuite) TearDownTest(c *check.C) {
- if s.rollback != nil {
- s.rollback()
+ Parent: s.localdb,
}
}
func (s *LDAPSuite) TestLoginSuccess(c *check.C) {
- conn := NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
- conn.loginController = s.ctrl
- resp, err := conn.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ resp, err := s.localdb.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: "goodusername",
Password: "goodpassword",
})
@@ -131,7 +94,7 @@ func (s *LDAPSuite) TestLoginSuccess(c *check.C) {
func (s *LDAPSuite) TestLoginFailure(c *check.C) {
// search returns no results
s.cluster.Login.LDAP.SearchBase = "dc=example,dc=invalid"
- resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ resp, err := s.localdb.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: "goodusername",
Password: "goodpassword",
})
@@ -144,7 +107,7 @@ func (s *LDAPSuite) TestLoginFailure(c *check.C) {
// search returns result, but auth fails
s.cluster.Login.LDAP.SearchBase = "dc=example,dc=com"
- resp, err = s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ resp, err = s.localdb.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: "badusername",
Password: "badpassword",
})
diff --git a/lib/controller/localdb/login_oidc_test.go b/lib/controller/localdb/login_oidc_test.go
index 582c0ac2a..5088de6ba 100644
--- a/lib/controller/localdb/login_oidc_test.go
+++ b/lib/controller/localdb/login_oidc_test.go
@@ -21,13 +21,11 @@ import (
"testing"
"time"
- "git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/jmoiron/sqlx"
check "gopkg.in/check.v1"
)
@@ -40,20 +38,11 @@ func Test(t *testing.T) {
var _ = check.Suite(&OIDCLoginSuite{})
type OIDCLoginSuite struct {
- cluster *arvados.Cluster
- localdb *Conn
- railsSpy *arvadostest.Proxy
+ localdbSuite
trustedURL *arvados.URL
fakeProvider *arvadostest.OIDCProvider
}
-func (s *OIDCLoginSuite) TearDownSuite(c *check.C) {
- // Undo any changes/additions to the user database so they
- // don't affect subsequent tests.
- arvadostest.ResetEnv()
- c.Check(arvados.NewClientFromEnv().RequestAndDecode(nil, "POST", "database/reset", nil, nil), check.IsNil)
-}
-
func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
s.trustedURL = &arvados.URL{Scheme: "https", Host: "app.example.com", Path: "/"}
@@ -66,10 +55,8 @@ func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
s.fakeProvider.ValidCode = fmt.Sprintf("abcdefgh-%d", time.Now().Unix())
s.fakeProvider.PeopleAPIResponse = map[string]interface{}{}
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
+ s.localdbSuite.SetUpTest(c)
+
s.cluster.Login.Test.Enable = false
s.cluster.Login.Google.Enable = true
s.cluster.Login.Google.ClientID = "test%client$id"
@@ -79,19 +66,14 @@ func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
s.fakeProvider.ValidClientID = "test%client$id"
s.fakeProvider.ValidClientSecret = "test#client/secret"
- s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
+ s.localdb = NewConn(s.ctx, s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeProvider.Issuer.URL
s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
-func (s *OIDCLoginSuite) TearDownTest(c *check.C) {
- s.railsSpy.Close()
-}
-
func (s *OIDCLoginSuite) TestGoogleLogout(c *check.C) {
s.cluster.Login.TrustedClients[arvados.URL{Scheme: "https", Host: "foo.example", Path: "/"}] = struct{}{}
s.cluster.Login.TrustPrivateNetworks = false
diff --git a/lib/controller/localdb/login_pam_test.go b/lib/controller/localdb/login_pam_test.go
index 0282b566f..2c3fa4d0f 100644
--- a/lib/controller/localdb/login_pam_test.go
+++ b/lib/controller/localdb/login_pam_test.go
@@ -5,63 +5,33 @@
package localdb
import (
- "context"
"io/ioutil"
"net/http"
"os"
"strings"
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/lib/controller/rpc"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
- "git.arvados.org/arvados.git/sdk/go/arvadostest"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/jmoiron/sqlx"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&PamSuite{})
type PamSuite struct {
- cluster *arvados.Cluster
- ctrl *pamLoginController
- railsSpy *arvadostest.Proxy
- db *sqlx.DB
- ctx context.Context
- rollback func() error
+ localdbSuite
}
-func (s *PamSuite) SetUpSuite(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
+func (s *PamSuite) SetUpTest(c *check.C) {
+ s.localdbSuite.SetUpTest(c)
s.cluster.Login.PAM.Enable = true
s.cluster.Login.PAM.DefaultEmailDomain = "example.com"
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- s.ctrl = &pamLoginController{
+ s.localdb.loginController = &pamLoginController{
Cluster: s.cluster,
- Parent: &Conn{railsProxy: rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)},
- }
- s.db = arvadostest.DB(c, s.cluster)
-}
-
-func (s *PamSuite) SetUpTest(c *check.C) {
- tx, err := s.db.Beginx()
- c.Assert(err, check.IsNil)
- s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx)
- s.rollback = tx.Rollback
-}
-
-func (s *PamSuite) TearDownTest(c *check.C) {
- if s.rollback != nil {
- s.rollback()
+ Parent: s.localdb,
}
}
func (s *PamSuite) TestLoginFailure(c *check.C) {
- resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ resp, err := s.localdb.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: "bogususername",
Password: "boguspassword",
})
@@ -91,7 +61,7 @@ func (s *PamSuite) TestLoginSuccess(c *check.C) {
c.Assert(len(lines), check.Equals, 2, check.Commentf("credentials file %s should contain \"username\\npassword\"", testCredsFile))
u, p := lines[0], lines[1]
- resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ resp, err := s.localdb.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: u,
Password: p,
})
diff --git a/lib/controller/localdb/login_testuser_test.go b/lib/controller/localdb/login_testuser_test.go
index 871761788..d5d3c2ff1 100644
--- a/lib/controller/localdb/login_testuser_test.go
+++ b/lib/controller/localdb/login_testuser_test.go
@@ -5,59 +5,30 @@
package localdb
import (
- "context"
"database/sql"
- "git.arvados.org/arvados.git/lib/config"
- "git.arvados.org/arvados.git/lib/controller/rpc"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
- "git.arvados.org/arvados.git/sdk/go/ctxlog"
- "github.com/jmoiron/sqlx"
check "gopkg.in/check.v1"
)
var _ = check.Suite(&TestUserSuite{})
type TestUserSuite struct {
- cluster *arvados.Cluster
- ctrl *testLoginController
- railsSpy *arvadostest.Proxy
- db *sqlx.DB
-
- // transaction context
- ctx context.Context
- tx *sqlx.Tx
+ localdbSuite
}
-func (s *TestUserSuite) SetUpSuite(c *check.C) {
- cfg, err := config.NewLoader(nil, ctxlog.TestLogger(c)).Load()
- c.Assert(err, check.IsNil)
- s.cluster, err = cfg.GetCluster("")
- c.Assert(err, check.IsNil)
+func (s *TestUserSuite) SetUpTest(c *check.C) {
+ s.localdbSuite.SetUpTest(c)
s.cluster.Login.Test.Enable = true
s.cluster.Login.Test.Users = map[string]arvados.TestUser{
"valid": {Email: "valid at example.com", Password: "v at l1d"},
}
- s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- s.ctrl = &testLoginController{
+ s.localdb.loginController = &testLoginController{
Cluster: s.cluster,
- Parent: &Conn{railsProxy: rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)},
+ Parent: s.localdb,
}
- s.db = arvadostest.DB(c, s.cluster)
-}
-
-func (s *TestUserSuite) SetUpTest(c *check.C) {
- tx, err := s.db.Beginx()
- c.Assert(err, check.IsNil)
- s.ctx = ctrlctx.NewWithTransaction(context.Background(), tx)
- s.tx = tx
-}
-
-func (s *TestUserSuite) TearDownTest(c *check.C) {
- s.tx.Rollback()
}
func (s *TestUserSuite) TestLogin(c *check.C) {
@@ -74,7 +45,7 @@ func (s *TestUserSuite) TestLogin(c *check.C) {
{true, "valid at example.com", "v at l1d"},
} {
c.Logf("=== %#v", trial)
- resp, err := s.ctrl.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
+ resp, err := s.localdb.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: trial.username,
Password: trial.password,
})
@@ -94,7 +65,7 @@ func (s *TestUserSuite) TestLogin(c *check.C) {
}
func (s *TestUserSuite) TestLoginForm(c *check.C) {
- resp, err := s.ctrl.Login(s.ctx, arvados.LoginOptions{
+ resp, err := s.localdb.Login(s.ctx, arvados.LoginOptions{
ReturnTo: "https://localhost:12345/example",
})
c.Check(err, check.IsNil)
@@ -133,7 +104,7 @@ func (s *TestUserSuite) TestExpireTokenOnLogout(c *check.C) {
c.Check(err, check.IsNil)
}
- resp, err := s.ctrl.Logout(ctx, arvados.LogoutOptions{
+ resp, err := s.localdb.Logout(ctx, arvados.LogoutOptions{
ReturnTo: returnTo,
})
c.Check(err, check.IsNil)
commit 5a3cde3245f7b3240fdef72486853ef0e1a81a05
Author: Tom Clegg <tom at curii.com>
Date: Sat Mar 4 03:49:55 2023 -0500
20183: Move priority update thread from rails to controller.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index 03690af02..3a232d29b 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -23,16 +23,18 @@ import (
"git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/health"
+ "github.com/jmoiron/sqlx"
)
type Conn struct {
+ bgCtx context.Context
cluster *arvados.Cluster
local backend
remotes map[string]backend
}
-func New(cluster *arvados.Cluster, healthFuncs *map[string]health.Func) *Conn {
- local := localdb.NewConn(cluster)
+func New(bgCtx context.Context, cluster *arvados.Cluster, healthFuncs *map[string]health.Func, getdb func(context.Context) (*sqlx.DB, error)) *Conn {
+ local := localdb.NewConn(bgCtx, cluster, getdb)
remotes := map[string]backend{}
for id, remote := range cluster.RemoteClusters {
if !remote.Proxy || id == cluster.ClusterID {
@@ -51,6 +53,7 @@ func New(cluster *arvados.Cluster, healthFuncs *map[string]health.Func) *Conn {
}
return &Conn{
+ bgCtx: bgCtx,
cluster: cluster,
local: local,
remotes: remotes,
@@ -362,6 +365,10 @@ func (conn *Conn) ContainerUpdate(ctx context.Context, options arvados.UpdateOpt
return conn.chooseBackend(options.UUID).ContainerUpdate(ctx, options)
}
+func (conn *Conn) ContainerPriorityUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) {
+ return conn.chooseBackend(options.UUID).ContainerPriorityUpdate(ctx, options)
+}
+
func (conn *Conn) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
return conn.chooseBackend(options.UUID).ContainerGet(ctx, options)
}
diff --git a/lib/controller/federation/federation_test.go b/lib/controller/federation/federation_test.go
index 5460e938a..6e85dfdba 100644
--- a/lib/controller/federation/federation_test.go
+++ b/lib/controller/federation/federation_test.go
@@ -70,7 +70,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
ctx = ctrlctx.NewWithTransaction(ctx, s.tx)
s.ctx = ctx
- s.fed = New(s.cluster, nil)
+ s.fed = New(ctx, s.cluster, nil, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
}
func (s *FederationSuite) TearDownTest(c *check.C) {
diff --git a/lib/controller/federation/group_test.go b/lib/controller/federation/group_test.go
index 1ee6f5876..a62120c58 100644
--- a/lib/controller/federation/group_test.go
+++ b/lib/controller/federation/group_test.go
@@ -5,6 +5,7 @@
package federation
import (
+ "context"
"errors"
"git.arvados.org/arvados.git/sdk/go/arvados"
@@ -21,7 +22,7 @@ type GroupSuite struct {
func makeConn() (*Conn, *arvadostest.APIStub, *arvadostest.APIStub) {
localAPIstub := &arvadostest.APIStub{Error: errors.New("No result")}
remoteAPIstub := &arvadostest.APIStub{Error: errors.New("No result")}
- return &Conn{&arvados.Cluster{ClusterID: "local"}, localAPIstub, map[string]backend{"zzzzz": remoteAPIstub}}, localAPIstub, remoteAPIstub
+ return &Conn{context.Background(), &arvados.Cluster{ClusterID: "local"}, localAPIstub, map[string]backend{"zzzzz": remoteAPIstub}}, localAPIstub, remoteAPIstub
}
func (s *UserSuite) TestGroupContents(c *check.C) {
diff --git a/lib/controller/federation/login_test.go b/lib/controller/federation/login_test.go
index e1114bf7e..a6743b320 100644
--- a/lib/controller/federation/login_test.go
+++ b/lib/controller/federation/login_test.go
@@ -8,6 +8,7 @@ import (
"context"
"net/url"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -50,7 +51,7 @@ func (s *LoginSuite) TestLogout(c *check.C) {
s.cluster.Login.LoginCluster = "zhome"
// s.fed is already set by SetUpTest, but we need to
// reinitialize with the above config changes.
- s.fed = New(s.cluster, nil)
+ s.fed = New(s.ctx, s.cluster, nil, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
for _, trial := range []struct {
token string
diff --git a/lib/controller/federation/user_test.go b/lib/controller/federation/user_test.go
index 064f8ce5d..1bd1bd2f1 100644
--- a/lib/controller/federation/user_test.go
+++ b/lib/controller/federation/user_test.go
@@ -14,6 +14,7 @@ import (
"strings"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -30,7 +31,7 @@ type UserSuite struct {
func (s *UserSuite) TestLoginClusterUserList(c *check.C) {
s.cluster.ClusterID = "local"
s.cluster.Login.LoginCluster = "zzzzz"
- s.fed = New(s.cluster, nil)
+ s.fed = New(s.ctx, s.cluster, nil, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.addDirectRemote(c, "zzzzz", rpc.NewConn("zzzzz", &url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}, true, rpc.PassthroughTokenProvider))
for _, updateFail := range []bool{false, true} {
@@ -120,7 +121,7 @@ func (s *UserSuite) TestLoginClusterUserList(c *check.C) {
func (s *UserSuite) TestLoginClusterUserGet(c *check.C) {
s.cluster.ClusterID = "local"
s.cluster.Login.LoginCluster = "zzzzz"
- s.fed = New(s.cluster, nil)
+ s.fed = New(s.ctx, s.cluster, nil, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.addDirectRemote(c, "zzzzz", rpc.NewConn("zzzzz", &url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")}, true, rpc.PassthroughTokenProvider))
opts := arvados.GetOptions{UUID: "zzzzz-tpzed-xurymjxw79nv3jz", Select: []string{"uuid", "email"}}
@@ -174,7 +175,7 @@ func (s *UserSuite) TestLoginClusterUserGet(c *check.C) {
func (s *UserSuite) TestLoginClusterUserListBypassFederation(c *check.C) {
s.cluster.ClusterID = "local"
s.cluster.Login.LoginCluster = "zzzzz"
- s.fed = New(s.cluster, nil)
+ s.fed = New(s.ctx, s.cluster, nil, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.addDirectRemote(c, "zzzzz", rpc.NewConn("zzzzz", &url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_API_HOST")},
true, rpc.PassthroughTokenProvider))
diff --git a/lib/controller/federation_test.go b/lib/controller/federation_test.go
index a3b198ffc..4fbb3440e 100644
--- a/lib/controller/federation_test.go
+++ b/lib/controller/federation_test.go
@@ -32,6 +32,9 @@ import (
var _ = check.Suite(&FederationSuite{})
type FederationSuite struct {
+ ctx context.Context
+ cancel context.CancelFunc
+
log logrus.FieldLogger
// testServer and testHandler are the controller being tested,
// "zhome".
@@ -48,6 +51,7 @@ type FederationSuite struct {
}
func (s *FederationSuite) SetUpTest(c *check.C) {
+ s.ctx, s.cancel = context.WithCancel(context.Background())
s.log = ctxlog.TestLogger(c)
s.remoteServer = newServerFromIntegrationTestEnv(c)
@@ -70,7 +74,7 @@ func (s *FederationSuite) SetUpTest(c *check.C) {
cluster.Collections.BlobSigningTTL = arvados.Duration(time.Hour * 24 * 14)
arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "http://localhost:1/")
arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
- s.testHandler = &Handler{Cluster: cluster, BackgroundContext: ctxlog.Context(context.Background(), s.log)}
+ s.testHandler = &Handler{Cluster: cluster, BackgroundContext: ctxlog.Context(s.ctx, s.log)}
s.testServer = newServerFromIntegrationTestEnv(c)
s.testServer.Server.BaseContext = func(net.Listener) context.Context {
return ctxlog.Context(context.Background(), s.log)
@@ -115,6 +119,7 @@ func (s *FederationSuite) TearDownTest(c *check.C) {
if s.testServer != nil {
s.testServer.Close()
}
+ s.cancel()
}
func (s *FederationSuite) testRequest(req *http.Request) *httptest.ResponseRecorder {
diff --git a/lib/controller/handler.go b/lib/controller/handler.go
index 4c6fca7f7..e40a087c1 100644
--- a/lib/controller/handler.go
+++ b/lib/controller/handler.go
@@ -94,8 +94,12 @@ func (h *Handler) setup() {
healthFuncs := make(map[string]health.Func)
h.dbConnector = ctrlctx.DBConnector{PostgreSQL: h.Cluster.PostgreSQL}
+ go func() {
+ <-h.BackgroundContext.Done()
+ h.dbConnector.Close()
+ }()
oidcAuthorizer := localdb.OIDCAccessTokenAuthorizer(h.Cluster, h.dbConnector.GetDB)
- h.federation = federation.New(h.Cluster, &healthFuncs)
+ h.federation = federation.New(h.BackgroundContext, h.Cluster, &healthFuncs, h.dbConnector.GetDB)
rtr := router.New(h.federation, router.Config{
MaxRequestSize: h.Cluster.API.MaxRequestSize,
WrapCalls: api.ComposeWrappers(
diff --git a/lib/controller/localdb/collection_test.go b/lib/controller/localdb/collection_test.go
index dac8b769f..d4b60baa0 100644
--- a/lib/controller/localdb/collection_test.go
+++ b/lib/controller/localdb/collection_test.go
@@ -16,6 +16,7 @@ import (
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadosclient"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
@@ -45,7 +46,7 @@ func (s *CollectionSuite) SetUpTest(c *check.C) {
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index 5b6964de0..6ab9e1450 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -20,6 +20,7 @@ import (
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"git.arvados.org/arvados.git/sdk/go/httpserver"
"github.com/hashicorp/yamux"
+ "github.com/jmoiron/sqlx"
"github.com/sirupsen/logrus"
)
@@ -28,6 +29,7 @@ type railsProxy = rpc.Conn
type Conn struct {
cluster *arvados.Cluster
*railsProxy // handles API methods that aren't defined on Conn itself
+ getdb func(context.Context) (*sqlx.DB, error)
vocabularyCache *arvados.Vocabulary
vocabularyFileModTime time.Time
lastVocabularyRefreshCheck time.Time
@@ -38,16 +40,21 @@ type Conn struct {
activeUsers map[string]bool
activeUsersLock sync.Mutex
activeUsersReset time.Time
+
+ wantContainerPriorityUpdate chan struct{}
}
-func NewConn(cluster *arvados.Cluster) *Conn {
+func NewConn(bgCtx context.Context, cluster *arvados.Cluster, getdb func(context.Context) (*sqlx.DB, error)) *Conn {
railsProxy := railsproxy.NewConn(cluster)
railsProxy.RedactHostInErrors = true
conn := Conn{
- cluster: cluster,
- railsProxy: railsProxy,
+ cluster: cluster,
+ railsProxy: railsProxy,
+ getdb: getdb,
+ wantContainerPriorityUpdate: make(chan struct{}, 1),
}
conn.loginController = chooseLoginController(cluster, &conn)
+ go conn.runContainerPriorityUpdateThread(bgCtx)
return &conn
}
diff --git a/lib/controller/localdb/container.go b/lib/controller/localdb/container.go
new file mode 100644
index 000000000..82f3c3b0a
--- /dev/null
+++ b/lib/controller/localdb/container.go
@@ -0,0 +1,110 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package localdb
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+ "github.com/sirupsen/logrus"
+)
+
+// ContainerUpdate defers to railsProxy and then notifies the
+// container priority updater thread.
+func (conn *Conn) ContainerUpdate(ctx context.Context, opts arvados.UpdateOptions) (arvados.Container, error) {
+ resp, err := conn.railsProxy.ContainerUpdate(ctx, opts)
+ if err == nil {
+ select {
+ case conn.wantContainerPriorityUpdate <- struct{}{}:
+ default:
+ // update already pending
+ }
+ }
+ return resp, err
+}
+
+// runContainerPriorityUpdateThread periodically (and immediately
+// after each container update request) corrects any inconsistent
+// container priorities caused by races.
+func (conn *Conn) runContainerPriorityUpdateThread(ctx context.Context) {
+ log := ctxlog.FromContext(ctx).WithField("worker", "runContainerPriorityUpdateThread")
+ ticker := time.NewTicker(5 * time.Minute)
+ for {
+ err := conn.containerPriorityUpdate(ctx, log)
+ if err != nil {
+ log.WithError(err).Warn("error updating container priorities")
+ }
+ select {
+ case <-ticker.C:
+ case <-conn.wantContainerPriorityUpdate:
+ }
+ }
+}
+
+func (conn *Conn) containerPriorityUpdate(ctx context.Context, log logrus.FieldLogger) error {
+ db, err := conn.getdb(ctx)
+ if err != nil {
+ return fmt.Errorf("getdb: %w", err)
+ }
+ res, err := db.ExecContext(ctx, `
+ UPDATE containers AS c
+ SET priority=0
+ WHERE state IN ('Queued', 'Locked', 'Running')
+ AND priority>0
+ AND uuid NOT IN (
+ SELECT container_uuid
+ FROM container_requests
+ WHERE priority > 0
+ AND state = 'Committed')`)
+ if err != nil {
+ return fmt.Errorf("update: %w", err)
+ } else if rows, err := res.RowsAffected(); err != nil {
+ return fmt.Errorf("update: %w", err)
+ } else if rows > 0 {
+ log.Infof("found %d containers with no active requests but priority>0, updated to priority=0", rows)
+ }
+ // In this loop we look for a single container that needs
+ // fixing, call out to Rails to fix it, and repeat until we
+ // don't find any more.
+ //
+ // We could get a batch of UUIDs that need attention by
+ // increasing LIMIT 1, however, updating priority on one
+ // container typically cascades to other containers, so we
+ // would often end up repeating work.
+ for lastUUID := ""; ; {
+ var uuid string
+ err := db.QueryRowxContext(ctx, `
+ SELECT containers.uuid from containers
+ JOIN container_requests
+ ON container_requests.container_uuid=containers.uuid
+ AND container_requests.state = 'Committed' AND container_requests.priority > 0
+ WHERE containers.state IN ('Queued', 'Locked', 'Running')
+ AND containers.priority = 0
+ AND container_requests.uuid IS NOT NULL
+ LIMIT 1`).Scan(&uuid)
+ if err == sql.ErrNoRows {
+ break
+ }
+ if err != nil {
+ return fmt.Errorf("join: %w", err)
+ }
+ if uuid == lastUUID {
+ // We don't want to keep hammering this
+ // forever if the ContainerPriorityUpdate call
+ // didn't achieve anything.
+ return fmt.Errorf("possible lack of progress: container %s still has priority=0 after updating", uuid)
+ }
+ lastUUID = uuid
+ _, err = conn.railsProxy.ContainerPriorityUpdate(ctx, arvados.UpdateOptions{UUID: uuid, Select: []string{"uuid"}})
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/lib/controller/localdb/container_gateway_test.go b/lib/controller/localdb/container_gateway_test.go
index 3f63e7aa8..92065c9ed 100644
--- a/lib/controller/localdb/container_gateway_test.go
+++ b/lib/controller/localdb/container_gateway_test.go
@@ -21,6 +21,7 @@ import (
"git.arvados.org/arvados.git/lib/controller/router"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/lib/crunchrun"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -51,7 +52,7 @@ func (s *ContainerGatewaySuite) SetUpSuite(c *check.C) {
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.ctx = auth.NewContext(context.Background(), &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
s.ctrUUID = arvadostest.QueuedContainerUUID
diff --git a/lib/controller/localdb/container_request_test.go b/lib/controller/localdb/container_request_test.go
index cca541a40..2d89f58ab 100644
--- a/lib/controller/localdb/container_request_test.go
+++ b/lib/controller/localdb/container_request_test.go
@@ -9,6 +9,7 @@ import (
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -36,7 +37,7 @@ func (s *ContainerRequestSuite) SetUpTest(c *check.C) {
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
diff --git a/lib/controller/localdb/group_test.go b/lib/controller/localdb/group_test.go
index 78150c955..ff80dd500 100644
--- a/lib/controller/localdb/group_test.go
+++ b/lib/controller/localdb/group_test.go
@@ -9,6 +9,7 @@ import (
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -29,7 +30,7 @@ func (s *GroupSuite) SetUpSuite(c *check.C) {
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
diff --git a/lib/controller/localdb/link_test.go b/lib/controller/localdb/link_test.go
index 2f07fb459..f28b32eb0 100644
--- a/lib/controller/localdb/link_test.go
+++ b/lib/controller/localdb/link_test.go
@@ -9,6 +9,7 @@ import (
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -36,7 +37,7 @@ func (s *LinkSuite) SetUpTest(c *check.C) {
c.Assert(err, check.IsNil)
s.cluster, err = cfg.GetCluster("")
c.Assert(err, check.IsNil)
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
}
diff --git a/lib/controller/localdb/login_ldap_test.go b/lib/controller/localdb/login_ldap_test.go
index b8ba6b467..1487d46f2 100644
--- a/lib/controller/localdb/login_ldap_test.go
+++ b/lib/controller/localdb/login_ldap_test.go
@@ -110,7 +110,7 @@ func (s *LDAPSuite) TearDownTest(c *check.C) {
}
func (s *LDAPSuite) TestLoginSuccess(c *check.C) {
- conn := NewConn(s.cluster)
+ conn := NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
conn.loginController = s.ctrl
resp, err := conn.UserAuthenticate(s.ctx, arvados.UserAuthenticateOptions{
Username: "goodusername",
diff --git a/lib/controller/localdb/login_oidc.go b/lib/controller/localdb/login_oidc.go
index 8a1b8fd82..65e2e250e 100644
--- a/lib/controller/localdb/login_oidc.go
+++ b/lib/controller/localdb/login_oidc.go
@@ -340,7 +340,7 @@ func OIDCAccessTokenAuthorizer(cluster *arvados.Cluster, getdb func(context.Cont
// We want ctrl to be nil if the chosen controller is not a
// *oidcLoginController, so we can ignore the 2nd return value
// of this type cast.
- ctrl, _ := NewConn(cluster).loginController.(*oidcLoginController)
+ ctrl, _ := NewConn(context.Background(), cluster, getdb).loginController.(*oidcLoginController)
cache, err := lru.New2Q(tokenCacheSize)
if err != nil {
panic(err)
diff --git a/lib/controller/localdb/login_oidc_test.go b/lib/controller/localdb/login_oidc_test.go
index 40cdde76f..582c0ac2a 100644
--- a/lib/controller/localdb/login_oidc_test.go
+++ b/lib/controller/localdb/login_oidc_test.go
@@ -23,6 +23,7 @@ import (
"git.arvados.org/arvados.git/lib/config"
"git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
"git.arvados.org/arvados.git/sdk/go/arvadostest"
"git.arvados.org/arvados.git/sdk/go/auth"
@@ -78,7 +79,7 @@ func (s *OIDCLoginSuite) SetUpTest(c *check.C) {
s.fakeProvider.ValidClientID = "test%client$id"
s.fakeProvider.ValidClientSecret = "test#client/secret"
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
c.Assert(s.localdb.loginController, check.FitsTypeOf, (*oidcLoginController)(nil))
s.localdb.loginController.(*oidcLoginController).Issuer = s.fakeProvider.Issuer.URL
s.localdb.loginController.(*oidcLoginController).peopleAPIBasePath = s.fakeProvider.PeopleAPI.URL
@@ -197,7 +198,7 @@ func (s *OIDCLoginSuite) TestConfig(c *check.C) {
s.cluster.Login.OpenIDConnect.ClientID = "oidc-client-id"
s.cluster.Login.OpenIDConnect.ClientSecret = "oidc-client-secret"
s.cluster.Login.OpenIDConnect.AuthenticationRequestParameters = map[string]string{"testkey": "testvalue"}
- localdb := NewConn(s.cluster)
+ localdb := NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
ctrl := localdb.loginController.(*oidcLoginController)
c.Check(ctrl.Issuer, check.Equals, "https://accounts.example.com/")
c.Check(ctrl.ClientID, check.Equals, "oidc-client-id")
@@ -212,7 +213,7 @@ func (s *OIDCLoginSuite) TestConfig(c *check.C) {
s.cluster.Login.Google.ClientSecret = "google-client-secret"
s.cluster.Login.Google.AlternateEmailAddresses = enableAltEmails
s.cluster.Login.Google.AuthenticationRequestParameters = map[string]string{"testkey": "testvalue"}
- localdb = NewConn(s.cluster)
+ localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
ctrl = localdb.loginController.(*oidcLoginController)
c.Check(ctrl.Issuer, check.Equals, "https://accounts.google.com")
c.Check(ctrl.ClientID, check.Equals, "google-client-id")
@@ -477,7 +478,7 @@ func (s *OIDCLoginSuite) TestGenericOIDCLogin(c *check.C) {
s.railsSpy.Close()
}
s.railsSpy = arvadostest.NewProxy(c, s.cluster.Services.RailsAPI)
- s.localdb = NewConn(s.cluster)
+ s.localdb = NewConn(context.Background(), s.cluster, (&ctrlctx.DBConnector{PostgreSQL: s.cluster.PostgreSQL}).GetDB)
*s.localdb.railsProxy = *rpc.NewConn(s.cluster.ClusterID, s.railsSpy.URL, true, rpc.PassthroughTokenProvider)
state := s.startLogin(c, func(form url.Values) {
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 4d8a82ce4..d5763d9ef 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -294,6 +294,13 @@ func (conn *Conn) ContainerUpdate(ctx context.Context, options arvados.UpdateOpt
return resp, err
}
+func (conn *Conn) ContainerPriorityUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) {
+ ep := arvados.EndpointContainerPriorityUpdate
+ var resp arvados.Container
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
func (conn *Conn) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
ep := arvados.EndpointContainerGet
var resp arvados.Container
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index bec387e85..1a4d61b42 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -43,6 +43,7 @@ var (
EndpointSpecimenDelete = APIEndpoint{"DELETE", "arvados/v1/specimens/{uuid}", ""}
EndpointContainerCreate = APIEndpoint{"POST", "arvados/v1/containers", "container"}
EndpointContainerUpdate = APIEndpoint{"PATCH", "arvados/v1/containers/{uuid}", "container"}
+ EndpointContainerPriorityUpdate = APIEndpoint{"POST", "arvados/v1/containers/{uuid}/update_priority", "container"}
EndpointContainerGet = APIEndpoint{"GET", "arvados/v1/containers/{uuid}", ""}
EndpointContainerList = APIEndpoint{"GET", "arvados/v1/containers", ""}
EndpointContainerDelete = APIEndpoint{"DELETE", "arvados/v1/containers/{uuid}", ""}
@@ -263,6 +264,7 @@ type API interface {
CollectionUntrash(ctx context.Context, options UntrashOptions) (Collection, error)
ContainerCreate(ctx context.Context, options CreateOptions) (Container, error)
ContainerUpdate(ctx context.Context, options UpdateOptions) (Container, error)
+ ContainerPriorityUpdate(ctx context.Context, options UpdateOptions) (Container, error)
ContainerGet(ctx context.Context, options GetOptions) (Container, error)
ContainerList(ctx context.Context, options ListOptions) (ContainerList, error)
ContainerDelete(ctx context.Context, options DeleteOptions) (Container, error)
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index 83efd8892..9b51e5ce2 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -89,6 +89,10 @@ func (as *APIStub) ContainerUpdate(ctx context.Context, options arvados.UpdateOp
as.appendCall(ctx, as.ContainerUpdate, options)
return arvados.Container{}, as.Error
}
+func (as *APIStub) ContainerPriorityUpdate(ctx context.Context, options arvados.UpdateOptions) (arvados.Container, error) {
+ as.appendCall(ctx, as.ContainerPriorityUpdate, options)
+ return arvados.Container{}, as.Error
+}
func (as *APIStub) ContainerGet(ctx context.Context, options arvados.GetOptions) (arvados.Container, error) {
as.appendCall(ctx, as.ContainerGet, options)
return arvados.Container{}, as.Error
diff --git a/services/api/app/controllers/arvados/v1/containers_controller.rb b/services/api/app/controllers/arvados/v1/containers_controller.rb
index 041f55947..c87aa8c79 100644
--- a/services/api/app/controllers/arvados/v1/containers_controller.rb
+++ b/services/api/app/controllers/arvados/v1/containers_controller.rb
@@ -53,6 +53,11 @@ class Arvados::V1::ContainersController < ApplicationController
show
end
+ def update_priority
+ @object.update_priority!
+ show
+ end
+
def current
if Thread.current[:api_client_authorization].nil?
send_error("Not logged in", status: 401)
diff --git a/services/api/app/models/container.rb b/services/api/app/models/container.rb
index 42d0ed49b..28cdd5795 100644
--- a/services/api/app/models/container.rb
+++ b/services/api/app/models/container.rb
@@ -5,7 +5,6 @@
require 'log_reuse_info'
require 'whitelist_update'
require 'safe_json'
-require 'update_priority'
class Container < ArvadosModel
include ArvadosModelUpdates
@@ -51,7 +50,6 @@ class Container < ArvadosModel
after_save :update_cr_logs
after_save :handle_completed
after_save :propagate_priority
- after_commit { UpdatePriority.run_update_thread }
has_many :container_requests, :foreign_key => :container_uuid, :class_name => 'ContainerRequest', :primary_key => :uuid
belongs_to :auth, :class_name => 'ApiClientAuthorization', :foreign_key => :auth_uuid, :primary_key => :uuid
diff --git a/services/api/config/routes.rb b/services/api/config/routes.rb
index 9c7bfc3a7..87e273757 100644
--- a/services/api/config/routes.rb
+++ b/services/api/config/routes.rb
@@ -40,6 +40,7 @@ Rails.application.routes.draw do
get 'auth', on: :member
post 'lock', on: :member
post 'unlock', on: :member
+ post 'update_priority', on: :member
get 'secret_mounts', on: :member
get 'current', on: :collection
end
diff --git a/services/api/lib/update_priority.rb b/services/api/lib/update_priority.rb
deleted file mode 100644
index 6c17f1bd0..000000000
--- a/services/api/lib/update_priority.rb
+++ /dev/null
@@ -1,67 +0,0 @@
-# Copyright (C) The Arvados Authors. All rights reserved.
-#
-# SPDX-License-Identifier: AGPL-3.0
-
-module UpdatePriority
- extend CurrentApiClient
-
- # Clean up after races.
- #
- # If container priority>0 but there are no committed container
- # requests for it, reset priority to 0.
- #
- # If container priority=0 but there are committed container requests
- # for it with priority>0, update priority.
- #
- # Normally, update_priority is a no-op if another thread/process is
- # already updating. Test cases that need to check priorities after
- # updating can force a (possibly overlapping) update in the current
- # thread/transaction by setting the "nolock" flag. See #14878.
- def self.update_priority(nolock: false)
- if !File.owned?(Rails.root.join('tmp'))
- Rails.logger.warn("UpdatePriority: not owner of #{Rails.root}/tmp, skipping")
- return
- end
- lockfile = Rails.root.join('tmp', 'update_priority.lock')
- File.open(lockfile, File::RDWR|File::CREAT, 0600) do |f|
- return unless nolock || f.flock(File::LOCK_NB|File::LOCK_EX)
-
- # priority>0 but should be 0:
- ActiveRecord::Base.connection.
- exec_query("UPDATE containers AS c SET priority=0 WHERE state IN ('Queued', 'Locked', 'Running') AND priority>0 AND uuid NOT IN (SELECT container_uuid FROM container_requests WHERE priority>0 AND state='Committed');", 'UpdatePriority')
-
- # priority==0 but should be >0:
- act_as_system_user do
- Container.
- joins("JOIN container_requests ON container_requests.container_uuid=containers.uuid AND container_requests.state=#{ActiveRecord::Base.connection.quote(ContainerRequest::Committed)} AND container_requests.priority>0").
- where('containers.state IN (?) AND containers.priority=0 AND container_requests.uuid IS NOT NULL',
- [Container::Queued, Container::Locked, Container::Running]).
- map(&:update_priority!)
- end
- end
- end
-
- def self.run_update_thread
- need = false
- Rails.cache.fetch('UpdatePriority', expires_in: 5.seconds) do
- need = true
- end
- return if !need
-
- Thread.new do
- Thread.current.abort_on_exception = false
- begin
- update_priority
- rescue => e
- Rails.logger.error "#{e.class}: #{e}\n#{e.backtrace.join("\n\t")}"
- ensure
- # Rails 5.1+ makes test threads share a database connection, so we can't
- # close a connection shared with other threads.
- # https://github.com/rails/rails/commit/deba47799ff905f778e0c98a015789a1327d5087
- if Rails.env != "test"
- ActiveRecord::Base.connection.close
- end
- end
- end
- end
-end
diff --git a/services/api/test/unit/update_priority_test.rb b/services/api/test/unit/update_priority_test.rb
index c1f60d91d..054fe4003 100644
--- a/services/api/test/unit/update_priority_test.rb
+++ b/services/api/test/unit/update_priority_test.rb
@@ -3,7 +3,6 @@
# SPDX-License-Identifier: AGPL-3.0
require 'test_helper'
-require 'update_priority'
class UpdatePriorityTest < ActiveSupport::TestCase
test 'priority 0 but should be >0' do
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list