[arvados] created: 2.1.0-3028-gdd19fce98
git repository hosting
git at public.arvados.org
Mon Nov 14 20:33:02 UTC 2022
at dd19fce98b082c54d968ecb9e74b50657dd6601d (commit)
commit dd19fce98b082c54d968ecb9e74b50657dd6601d
Author: Tom Clegg <tom at curii.com>
Date: Mon Nov 14 15:31:42 2022 -0500
19709: Apply pending rails migrations at service start/restart.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/boot/rails_db.go b/lib/boot/rails_db.go
new file mode 100644
index 000000000..3f8151144
--- /dev/null
+++ b/lib/boot/rails_db.go
@@ -0,0 +1,89 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package boot
+
+import (
+ "context"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+
+ "git.arvados.org/arvados.git/lib/controller/dblock"
+ "git.arvados.org/arvados.git/lib/ctrlctx"
+)
+
+type railsDatabase struct{}
+
+func (runner railsDatabase) String() string {
+ return "railsDatabase"
+}
+
+func (runner railsDatabase) Run(ctx context.Context, fail func(error), super *Supervisor) error {
+ err := super.wait(ctx, runPostgreSQL{}, installPassenger{src: "services/api"})
+ if err != nil {
+ return err
+ }
+
+ // determine path to installed rails app or source tree
+ var appdir string
+ if super.ClusterType == "production" {
+ appdir = "/var/lib/arvados/railsapi"
+ } else {
+ appdir = filepath.Join(super.SourcePath, "services/api")
+ }
+
+ // list versions in db/migrate/{version}_{name}.rb
+ todo := map[string]bool{}
+ fs.WalkDir(os.DirFS(appdir), "db/migrate", func(path string, d fs.DirEntry, err error) error {
+ if cut := strings.Index(d.Name(), "_"); cut > 0 && strings.HasSuffix(d.Name(), ".rb") {
+ todo[d.Name()[:cut]] = true
+ }
+ return nil
+ })
+
+ // read schema_migrations table (list of migrations already
+ // applied) and remove those entries from todo
+ dbconnector := ctrlctx.DBConnector{PostgreSQL: super.cluster.PostgreSQL}
+ defer dbconnector.Close()
+ db, err := dbconnector.GetDB(ctx)
+ if err != nil {
+ return err
+ }
+ rows, err := db.QueryContext(ctx, `SELECT version FROM schema_migrations`)
+ if err != nil {
+ if super.ClusterType == "production" {
+ return err
+ }
+ super.logger.WithError(err).Info("schema_migrations query failed, trying db:setup")
+ return super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
+ }
+ for rows.Next() {
+ var v string
+ err = rows.Scan(&v)
+ if err != nil {
+ return err
+ }
+ delete(todo, v)
+ }
+ err = rows.Close()
+ if err != nil {
+ return err
+ }
+
+ // if nothing remains in todo, all available migrations are
+ // done, so return without running any [relatively slow]
+ // ruby/rake commands
+ if len(todo) == 0 {
+ return nil
+ }
+
+ super.logger.Infof("%d migrations pending", len(todo))
+ if !dblock.RailsMigrations.Lock(ctx, dbconnector.GetDB) {
+ return context.Canceled
+ }
+ defer dblock.RailsMigrations.Unlock()
+ return super.RunProgram(ctx, appdir, runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:migrate")
+}
diff --git a/lib/boot/seed.go b/lib/boot/seed.go
deleted file mode 100644
index b43d90720..000000000
--- a/lib/boot/seed.go
+++ /dev/null
@@ -1,31 +0,0 @@
-// Copyright (C) The Arvados Authors. All rights reserved.
-//
-// SPDX-License-Identifier: AGPL-3.0
-
-package boot
-
-import (
- "context"
-)
-
-// Populate a blank database with arvados tables and seed rows.
-type seedDatabase struct{}
-
-func (seedDatabase) String() string {
- return "seedDatabase"
-}
-
-func (seedDatabase) Run(ctx context.Context, fail func(error), super *Supervisor) error {
- err := super.wait(ctx, runPostgreSQL{}, installPassenger{src: "services/api"})
- if err != nil {
- return err
- }
- if super.ClusterType == "production" {
- return nil
- }
- err = super.RunProgram(ctx, "services/api", runOptions{env: railsEnv}, "bundle", "exec", "rake", "db:setup")
- if err != nil {
- return err
- }
- return nil
-}
diff --git a/lib/boot/supervisor.go b/lib/boot/supervisor.go
index ca88653fa..0f0600f18 100644
--- a/lib/boot/supervisor.go
+++ b/lib/boot/supervisor.go
@@ -356,20 +356,24 @@ func (super *Supervisor) runCluster() error {
createCertificates{},
runPostgreSQL{},
runNginx{},
- runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{seedDatabase{}}},
+ railsDatabase{},
+ runServiceCommand{name: "controller", svc: super.cluster.Services.Controller, depends: []supervisedTask{railsDatabase{}}},
runServiceCommand{name: "git-httpd", svc: super.cluster.Services.GitHTTP},
runServiceCommand{name: "health", svc: super.cluster.Services.Health},
runServiceCommand{name: "keepproxy", svc: super.cluster.Services.Keepproxy, depends: []supervisedTask{runPassenger{src: "services/api"}}},
runServiceCommand{name: "keepstore", svc: super.cluster.Services.Keepstore},
runServiceCommand{name: "keep-web", svc: super.cluster.Services.WebDAV},
- runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{seedDatabase{}}},
+ runServiceCommand{name: "ws", svc: super.cluster.Services.Websocket, depends: []supervisedTask{railsDatabase{}}},
installPassenger{src: "services/api", varlibdir: "railsapi"},
- runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{createCertificates{}, seedDatabase{}, installPassenger{src: "services/api", varlibdir: "railsapi"}}},
- seedDatabase{},
+ runPassenger{src: "services/api", varlibdir: "railsapi", svc: super.cluster.Services.RailsAPI, depends: []supervisedTask{
+ createCertificates{},
+ installPassenger{src: "services/api", varlibdir: "railsapi"},
+ railsDatabase{},
+ }},
}
if !super.NoWorkbench1 {
tasks = append(tasks,
- installPassenger{src: "apps/workbench", varlibdir: "workbench1", depends: []supervisedTask{seedDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
+ installPassenger{src: "apps/workbench", varlibdir: "workbench1", depends: []supervisedTask{railsDatabase{}}}, // dependency ensures workbench doesn't delay api install/startup
runPassenger{src: "apps/workbench", varlibdir: "workbench1", svc: super.cluster.Services.Workbench1, depends: []supervisedTask{installPassenger{src: "apps/workbench", varlibdir: "workbench1"}}},
)
}
diff --git a/lib/controller/dblock/dblock.go b/lib/controller/dblock/dblock.go
index ad2733abf..c59bcef0b 100644
--- a/lib/controller/dblock/dblock.go
+++ b/lib/controller/dblock/dblock.go
@@ -22,6 +22,7 @@ var (
KeepBalanceService = &DBLocker{key: 10003} // keep-balance service in periodic-sweep loop
KeepBalanceActive = &DBLocker{key: 10004} // keep-balance sweep in progress (either -once=true or service loop)
Dispatch = &DBLocker{key: 10005} // any dispatcher running
+ RailsMigrations = &DBLocker{key: 10006}
retryDelay = 5 * time.Second
)
diff --git a/lib/ctrlctx/db.go b/lib/ctrlctx/db.go
index 2a05096ce..b711b3e65 100644
--- a/lib/ctrlctx/db.go
+++ b/lib/ctrlctx/db.go
@@ -168,8 +168,20 @@ func (dbc *DBConnector) GetDB(ctx context.Context) (*sqlx.DB, error) {
}
if err := db.Ping(); err != nil {
ctxlog.FromContext(ctx).WithError(err).Error("postgresql connect succeeded but ping failed")
+ db.Close()
return nil, errDBConnection
}
dbc.pgdb = db
return db, nil
}
+
+func (dbc *DBConnector) Close() error {
+ dbc.mtx.Lock()
+ defer dbc.mtx.Unlock()
+ var err error
+ if dbc.pgdb != nil {
+ err = dbc.pgdb.Close()
+ dbc.pgdb = nil
+ }
+ return err
+}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list