[arvados] created: 2.1.0-2733-g2897e5d5b
git repository hosting
git at public.arvados.org
Wed Jul 13 19:41:54 UTC 2022
at 2897e5d5ba1e051986bab9cd225e9062e7d72fa2 (commit)
commit 2897e5d5ba1e051986bab9cd225e9062e7d72fa2
Author: Tom Clegg <tom at curii.com>
Date: Wed Jul 13 15:38:26 2022 -0400
17344: Diagnostics warning, not error, if there are no VMs listed.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 6dc5f679b..ed8d440b8 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -458,9 +458,10 @@ func (diag *diagnoser) runtests() {
return err
}
if len(vmlist.Items) < 1 {
- return fmt.Errorf("no VMs found")
+ diag.warnf("no VMs found")
+ } else {
+ vm = vmlist.Items[0]
}
- vm = vmlist.Items[0]
return nil
})
@@ -468,7 +469,8 @@ func (diag *diagnoser) runtests() {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if vm.UUID == "" {
- return fmt.Errorf("skipping, no vm available")
+ diag.warnf("skipping, no vm available")
+ return nil
}
webshelltermurl := cluster.Services.Workbench1.ExternalURL.String() + "virtual_machines/" + vm.UUID + "/webshell/testusername"
diag.debugf("url %s", webshelltermurl)
@@ -496,7 +498,8 @@ func (diag *diagnoser) runtests() {
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if vm.UUID == "" {
- return fmt.Errorf("skipping, no vm available")
+ diag.warnf("skipping, no vm available")
+ return nil
}
u := cluster.Services.WebShell.ExternalURL
webshellurl := u.String() + vm.Hostname + "?"
commit 2cf6d939cf14761d64baad8fb9adffa0f09fce4a
Author: Tom Clegg <tom at curii.com>
Date: Wed Jul 13 15:34:04 2022 -0400
17344: Rename "root" subcommand to "sudo".
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index 783fb0ca9..c10783c97 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -61,7 +61,7 @@ var (
"shell": shellCommand{},
"connect-ssh": connectSSHCommand{},
"diagnostics": diagnostics.Command{},
- "root": rootCommand{},
+ "sudo": sudoCommand{},
})
)
diff --git a/cmd/arvados-client/root.go b/cmd/arvados-client/sudo.go
similarity index 88%
rename from cmd/arvados-client/root.go
rename to cmd/arvados-client/sudo.go
index 5d9845ce4..94c2d275d 100644
--- a/cmd/arvados-client/root.go
+++ b/cmd/arvados-client/sudo.go
@@ -15,12 +15,12 @@ import (
"git.arvados.org/arvados.git/sdk/go/ctxlog"
)
-// rootCommand runs another command using API connection info and
+// sudoCommand runs another command using API connection info and
// SystemRootToken from the system config file instead of the caller's
// environment vars.
-type rootCommand struct{}
+type sudoCommand struct{}
-func (rootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+func (sudoCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
ldr := config.NewLoader(stdin, ctxlog.New(stderr, "text", "info"))
flags := flag.NewFlagSet(prog, flag.ContinueOnError)
ldr.SetupFlags(flags)
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
index 199518ffc..9923767dd 100644
--- a/doc/install/automatic.html.textile.liquid
+++ b/doc/install/automatic.html.textile.liquid
@@ -61,13 +61,13 @@ When the "init" command is finished, navigate to the link shown in the terminal
Activate your new Arvados user account.
<pre>
-# arv root user setup exampleUserName
+# arv sudo user setup exampleUserName
</pre>
Run the diagnostics tool to ensure everything is working.
<pre>
-# arv root diagnostics
+# arv sudo diagnostics
</pre>
h2. Customize the cluster
diff --git a/lib/install/init.go b/lib/install/init.go
index 80fa35cc4..a341155ab 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -374,7 +374,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
err = fmt.Errorf("%v: %w", cmd.Args, err)
return 1
}
- cmd = exec.CommandContext(ctx, "arv", "root", "keep", "docker", "alpine")
+ cmd = exec.CommandContext(ctx, "arv", "sudo", "keep", "docker", "alpine")
cmd.Stdout = stderr
cmd.Stderr = stderr
err = cmd.Run()
commit b1471b47676ca0220351cb02151137b696bc40eb
Author: Tom Clegg <tom at curii.com>
Date: Wed Jul 13 15:27:04 2022 -0400
17344: Add firewall info to easy install doc page.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
index 19ea3fd12..199518ffc 100644
--- a/doc/install/automatic.html.textile.liquid
+++ b/doc/install/automatic.html.textile.liquid
@@ -22,7 +22,10 @@ h2. Prerequisites
You will need:
* a server host running Debian 10 (buster) or Debian 11 (bullseye).
* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
-* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
+* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP requests on port 80[1] and HTTPS requests on ports 443 and 4440-4460 through to the same port on your server host).
+* a firewall setup that allows incoming connections to ports 80[1], 443, and 4440-4460.
+
+fn1. Port 80 is only used to obtain TLS certificates automatically from Let's Encrypt. It is not needed if you have another way to provision certificates.
h2. Options
@@ -47,7 +50,7 @@ Arvados needs a login backend. To get started quickly, add a user account on you
h2. Initialize the cluster
<pre>
-# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
+# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/$(lsb_release -sc) $(lsb_release -sc) main"
# apt update
# apt install arvados-server-easy
# arvados-server init -cluster-id x9999 -domain x9999.example.com -tls auto -login pam
commit b14adeba625ea7cad637e17cd7af2e9fe525380a
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 4 10:55:38 2022 -0400
19235: Don't try bogus URL if WebDAV ExternalURL has no wildcard.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 467c2f0b5..6dc5f679b 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -390,29 +390,35 @@ func (diag *diagnoser) runtests() {
})
davurl := cluster.Services.WebDAV.ExternalURL
+ davWildcard := strings.HasPrefix(davurl.Host, "*--") || strings.HasPrefix(davurl.Host, "*.")
diag.dotest(110, fmt.Sprintf("checking WebDAV ExternalURL wildcard (%s)", davurl), func() error {
if davurl.Host == "" {
return fmt.Errorf("host missing - content previews will not work")
}
- if !strings.HasPrefix(davurl.Host, "*--") && !strings.HasPrefix(davurl.Host, "*.") && !cluster.Collections.TrustAllContent {
+ if !davWildcard && !cluster.Collections.TrustAllContent {
diag.warnf("WebDAV ExternalURL has no leading wildcard and TrustAllContent==false - content previews will not work")
}
return nil
})
for i, trial := range []struct {
- needcoll bool
- status int
- fileurl string
+ needcoll bool
+ needWildcard bool
+ status int
+ fileurl string
}{
- {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
- {false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
- {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
- {false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
- {true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
- {true, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
+ {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "foo"},
+ {false, false, http.StatusNotFound, strings.Replace(davurl.String(), "*", "d41d8cd98f00b204e9800998ecf8427e-0", 1) + "testfile"},
+ {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/foo"},
+ {false, false, http.StatusNotFound, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=d41d8cd98f00b204e9800998ecf8427e+0/_/testfile"},
+ {true, true, http.StatusOK, strings.Replace(davurl.String(), "*", strings.Replace(collection.PortableDataHash, "+", "-", -1), 1) + "testfile"},
+ {true, false, http.StatusOK, cluster.Services.WebDAVDownload.ExternalURL.String() + "c=" + collection.UUID + "/_/testfile"},
} {
diag.dotest(120+i, fmt.Sprintf("downloading from webdav (%s)", trial.fileurl), func() error {
+ if trial.needWildcard && !davWildcard {
+ diag.warnf("skipping collection-id-in-vhost test because WebDAV ExternalURL has no leading wildcard")
+ return nil
+ }
ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(diag.timeout))
defer cancel()
if trial.needcoll && collection.UUID == "" {
commit b3a15352fc47a951bb8329d0c421551e04c13341
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 22:32:40 2022 -0400
17344: Check that needed ports are usable before doing any init.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 3eeac5c54..80fa35cc4 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -13,6 +13,8 @@ import (
"flag"
"fmt"
"io"
+ "net"
+ "net/http"
"net/url"
"os"
"os/exec"
@@ -20,7 +22,9 @@ import (
"regexp"
"strconv"
"strings"
+ "sync/atomic"
"text/template"
+ "time"
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
@@ -111,6 +115,15 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
+ err = initcmd.checkPort(ctx, "4440")
+ err = initcmd.checkPort(ctx, "443")
+ if initcmd.TLS == "auto" {
+ err = initcmd.checkPort(ctx, "80")
+ if err != nil {
+ return 1
+ }
+ }
+
// Do the "create extension" thing early. This way, if there's
// no local postgresql server (a likely failure mode), we can
// bail out without any side effects, and the user can start
@@ -417,3 +430,82 @@ func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.Postgre
}
return nil
}
+
+// Confirm that http://{initcmd.Domain}:{port} reaches a server that
+// we run on {port}.
+//
+// If port is "80", listening fails, and Nginx appears to be using the
+// debian-packaged default configuration that listens on port 80,
+// disable that Nginx config and try again.
+//
+// (Typically, the reason Nginx is installed is so that Arvados can
+// run an Nginx child process; the default Nginx service using config
+// from /etc/nginx is just an unfortunate side effect of installing
+// Nginx by way of the Debian package.)
+func (initcmd *initCommand) checkPort(ctx context.Context, port string) error {
+ err := initcmd.checkPortOnce(ctx, port)
+ if err == nil || port != "80" {
+ // success, or poking Nginx in the eye won't help
+ return err
+ }
+ d, err2 := os.Open("/etc/nginx/sites-enabled/.")
+ if err2 != nil {
+ return err
+ }
+ fis, err2 := d.Readdir(-1)
+ if err2 != nil || len(fis) != 1 {
+ return err
+ }
+ if target, err2 := os.Readlink("/etc/nginx/sites-enabled/default"); err2 != nil || target != "/etc/nginx/sites-available/default" {
+ return err
+ }
+ err2 = os.Remove("/etc/nginx/sites-enabled/default")
+ if err2 != nil {
+ return err
+ }
+ exec.CommandContext(ctx, "nginx", "-s", "reload").Run()
+ time.Sleep(time.Second)
+ return initcmd.checkPortOnce(ctx, port)
+}
+
+// Start an http server on 0.0.0.0:{port} and confirm that
+// http://{initcmd.Domain}:{port} reaches that server.
+func (initcmd *initCommand) checkPortOnce(ctx context.Context, port string) error {
+ b := make([]byte, 128)
+ _, err := rand.Read(b)
+ if err != nil {
+ return err
+ }
+ token := fmt.Sprintf("%x", b)
+
+ srv := http.Server{
+ Addr: net.JoinHostPort("", port),
+ Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ fmt.Fprint(w, token)
+ })}
+ var errServe atomic.Value
+ go func() {
+ errServe.Store(srv.ListenAndServe())
+ }()
+ defer srv.Close()
+ url := "http://" + net.JoinHostPort(initcmd.Domain, port) + "/probe"
+ req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
+ if err != nil {
+ return err
+ }
+ resp, err := http.DefaultClient.Do(req)
+ if errServe, _ := errServe.Load().(error); errServe != nil {
+ // If server already exited, return that error
+ // (probably "can't listen"), not the request error.
+ return errServe
+ }
+ if err != nil {
+ return err
+ }
+ buf := make([]byte, len(token))
+ n, err := io.ReadFull(resp.Body, buf)
+ if string(buf[:n]) != token {
+ return fmt.Errorf("listened on port %s but %s connected to something else, returned %q, err %v", port, url, buf[:n], err)
+ }
+ return nil
+}
commit 3dbb53f39fa7b6528b1cbc010097e084e222735f
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 15:31:41 2022 -0400
17344: Load alpine docker image for diagnostics.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/diagnostics/cmd.go b/lib/diagnostics/cmd.go
index 71fe1c5dc..467c2f0b5 100644
--- a/lib/diagnostics/cmd.go
+++ b/lib/diagnostics/cmd.go
@@ -30,6 +30,7 @@ func (Command) RunCommand(prog string, args []string, stdin io.Reader, stdout, s
f := flag.NewFlagSet(prog, flag.ContinueOnError)
f.StringVar(&diag.projectName, "project-name", "scratch area for diagnostics", "name of project to find/create in home project and use for temporary/test objects")
f.StringVar(&diag.logLevel, "log-level", "info", "logging level (debug, info, warning, error)")
+ f.StringVar(&diag.dockerImage, "docker-image", "alpine:latest", "image to use when running a test container")
f.BoolVar(&diag.checkInternal, "internal-client", false, "check that this host is considered an \"internal\" client")
f.BoolVar(&diag.checkExternal, "external-client", false, "check that this host is considered an \"external\" client")
f.IntVar(&diag.priority, "priority", 500, "priority for test container (1..1000, or 0 to skip)")
@@ -60,6 +61,7 @@ type diagnoser struct {
logLevel string
priority int
projectName string
+ dockerImage string
checkInternal bool
checkExternal bool
timeout time.Duration
@@ -545,7 +547,7 @@ func (diag *diagnoser) runtests() {
err := client.RequestAndDecodeContext(ctx, &cr, "POST", "arvados/v1/container_requests", nil, map[string]interface{}{"container_request": map[string]interface{}{
"owner_uuid": project.UUID,
"name": fmt.Sprintf("diagnostics container request %s", timestamp),
- "container_image": "arvados/jobs",
+ "container_image": diag.dockerImage,
"command": []string{"echo", timestamp},
"use_existing": false,
"output_path": "/mnt/output",
diff --git a/lib/install/init.go b/lib/install/init.go
index 11a62f18e..3eeac5c54 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -351,6 +351,29 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
fmt.Fprintln(stderr, "...looks good")
}
+ if out, err := exec.CommandContext(ctx, "docker", "version").CombinedOutput(); err == nil && strings.Contains(string(out), "\nServer:\n") {
+ fmt.Fprintln(stderr, "loading alpine docker image for diagnostics...")
+ cmd := exec.CommandContext(ctx, "docker", "pull", "alpine")
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ cmd = exec.CommandContext(ctx, "arv", "root", "keep", "docker", "alpine")
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("%v: %w", cmd.Args, err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+ } else {
+ fmt.Fprintln(stderr, "docker is not installed -- skipping step of downloading 'alpine' image")
+ }
+
fmt.Fprintln(stderr, "Setup complete. You should now be able to log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String())
return 0
commit ca379001ccb7fe9be0ad6ef89b6c8dfff8cd2380
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 14:41:33 2022 -0400
17344: Add docker and postgresql as suggested deb packages.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index 465901fa4..64d0adabe 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -104,6 +104,8 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
"--deb-systemd-enable",
"--no-deb-systemd-auto-start",
"--no-deb-systemd-restart-after-upgrade",
+ "--deb-suggests", "postgresql",
+ "--deb-suggests", "docker.io",
"/usr/bin/arvados-client",
"/usr/bin/arvados-server",
"/usr/bin/arv",
commit 54ea229e384bddaab334073ad61fcd54f97593c0
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 14:41:23 2022 -0400
17344: Update automatic install docs.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/doc/install/automatic.html.textile.liquid b/doc/install/automatic.html.textile.liquid
index f520ffb50..19ea3fd12 100644
--- a/doc/install/automatic.html.textile.liquid
+++ b/doc/install/automatic.html.textile.liquid
@@ -23,25 +23,70 @@ You will need:
* a server host running Debian 10 (buster) or Debian 11 (bullseye).
* a unique 5-character ID like @x9999@ for your cluster (first character should be @[a-w]@ for a long-lived / production cluster; all characters are @[a-z0-9]@).
* a DNS name like @x9999.example.com@ that resolves to your server host (or a load balancer / proxy that passes HTTP and HTTPS requests through to your server host).
-* a Google account (use it in place of <code>example at gmail.com.example</code> in the instructions below).
+
+h2. Options
+
+Arvados needs a PostgreSQL database. To get started quickly, install the postgresql-server package on your server host.
+
+<pre>
+# apt install postgresql
+</pre>
+
+Arvados normally uses cloud VMs or a Slurm/LSF cluster to run containers. To get started quickly, install Docker on your system host. The @arvados-server init@ command, as shown below, will configure Arvados to run containers on the system host.
+
+<pre>
+# apt install docker.io
+</pre>
+
+Arvados needs a login backend. To get started quickly, add a user account on your server host and assign a password. The @arvados-server init ... -login pam@ option, as shown below, will configure Arvados so you can log in with this username and password.
+
+<pre>
+# adduser exampleUserName
+</pre>
h2. Initialize the cluster
<pre>
# echo > /etc/apt/sources.list.d/arvados.list "deb http://apt.arvados.org/buster buster main"
-# apt-get update
-# apt-get install arvados-server-easy
-# arvados-server init -cluster-id x9999 -domain x9999.example.com -tls auto -admin-email example at gmail.com.example
+# apt update
+# apt install arvados-server-easy
+# arvados-server init -cluster-id x9999 -domain x9999.example.com -tls auto -login pam
</pre>
-When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/token?api_token=zzzzzzzzzzzzzzzzzzzzzz@). This will log you in to your admin account.
+When the "init" command is finished, navigate to the link shown in the terminal (e.g., @https://x9999.example.com/@) and log in with the account you created above.
-h2. Enable login
+Activate your new Arvados user account.
-Follow the instructions to "set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+<pre>
+# arv root user setup exampleUserName
+</pre>
+
+Run the diagnostics tool to ensure everything is working.
+
+<pre>
+# arv root diagnostics
+</pre>
+
+h2. Customize the cluster
+
+Things you should plan to update before using your cluster in production:
+* "Set up Google login":{{site.baseurl}}/install/setup-login.html or another authentication option.
+* Set up a wildcard TLS certificate and DNS name, or enable @TrustAllContent@ mode.
+* Update storage configuration to use a cloud storage bucket instead of the local filesystem.
+* Update CloudVMs configuration to use a cloud provider to bring up VMs on demand instead of running containers on the server host.
+
+h2. Updating configuration
+
+After updating your configuration file (@/etc/arvados/config.yml@), notify the server:
+
+<pre>
+# systemctl reload arvados-server
+</pre>
-After updating your configuration file (@/etc/arvados/config.yml@), restart the server to make your changes take effect:
+Optionally, add "AutoReloadConfig: true" at the top of @/etc/arvados/config.yml at . Arvados will automatically reload the config file when it changes.
<pre>
-# systemctl restart arvados-server
+AutoReloadConfig: true
+Clusters:
+ [...]
</pre>
commit a97a2a679505262e347991cae9233be5b448e860
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 10:25:28 2022 -0400
17344: Add arv-mount and python tools to package.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index c26cbe887..465901fa4 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -110,6 +110,16 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
"/usr/bin/arv-ruby",
"/usr/bin/arv-tag",
"/var/lib/arvados",
+ "/usr/bin/arv-copy",
+ "/usr/bin/arv-federation-migrate",
+ "/usr/bin/arv-get",
+ "/usr/bin/arv-keepdocker",
+ "/usr/bin/arv-ls",
+ "/usr/bin/arv-migrate-docker19",
+ "/usr/bin/arv-normalize",
+ "/usr/bin/arv-put",
+ "/usr/bin/arv-ws",
+ "/usr/bin/arv-mount",
"/var/www/.gem",
"/var/www/.passenger",
"/var/www/.bundle",
diff --git a/lib/install/deps.go b/lib/install/deps.go
index a9fb874e2..a5c428d0a 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -586,6 +586,33 @@ yarn install
return 1
}
+ // Install python SDK and arv-mount in
+ // /var/lib/arvados/lib/python.
+ //
+ // setup.py writes a file in the source directory in
+ // order to include the version number in the package
+ // itself. We don't want to write to the source tree
+ // (in "arvados-package" context it's mounted
+ // readonly) so we run setup.py in a temporary copy of
+ // the source dir.
+ if err = inst.runBash(`
+v=/var/lib/arvados/lib/python
+tmp=/var/lib/arvados/tmp/python
+python3 -m venv "$v"
+. "$v/bin/activate"
+pip3 install --no-cache-dir 'setuptools>=18.5' 'pip>=7'
+export ARVADOS_BUILDING_VERSION="`+inst.PackageVersion+`"
+for src in "`+inst.SourcePath+`/sdk/python" "`+inst.SourcePath+`/services/fuse"; do
+ rsync -a --delete-after "$src/" "$tmp/"
+ cd "$tmp"
+ python3 setup.py install
+ cd ..
+ rm -rf "$tmp"
+done
+`, stdout, stderr); err != nil {
+ return 1
+ }
+
// Install Rails apps to /var/lib/arvados/{railsapi,workbench1}/
for dstdir, srcdir := range map[string]string{
"railsapi": "services/api",
@@ -693,22 +720,40 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/
}
}
- // Symlink user-facing programs /usr/bin/x ->
- // /var/lib/arvados/bin/x
+ // Add symlinks in /usr/bin for user-facing programs
for _, srcdst := range [][]string{
- {"arvados-client", "arvados-client"},
- {"arvados-client", "arv"},
- {"arvados-server", "arvados-server"},
- {"arv", "arv-ruby"},
- {"arv-tag", "arv-tag"},
+ // go
+ {"bin/arvados-client"},
+ {"bin/arvados-client", "arv"},
+ {"bin/arvados-server"},
+ // sdk/cli
+ {"bin/arv", "arv-ruby"},
+ {"bin/arv-tag"},
+ // sdk/python
+ {"lib/python/bin/arv-copy"},
+ {"lib/python/bin/arv-federation-migrate"},
+ {"lib/python/bin/arv-get"},
+ {"lib/python/bin/arv-keepdocker"},
+ {"lib/python/bin/arv-ls"},
+ {"lib/python/bin/arv-migrate-docker19"},
+ {"lib/python/bin/arv-normalize"},
+ {"lib/python/bin/arv-put"},
+ {"lib/python/bin/arv-ws"},
+ // services/fuse
+ {"lib/python/bin/arv-mount"},
} {
- src := srcdst[0]
- dst := srcdst[1]
- err = os.Remove("/usr/bin/" + dst)
+ src := "/var/lib/arvados/" + srcdst[0]
+ if _, err = os.Stat(src); err != nil {
+ return 1
+ }
+ dst := srcdst[len(srcdst)-1]
+ _, dst = filepath.Split(dst)
+ dst = "/usr/bin/" + dst
+ err = os.Remove(dst)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return 1
}
- err = os.Symlink("/var/lib/arvados/bin/"+src, "/usr/bin/"+dst)
+ err = os.Symlink(src, dst)
if err != nil {
return 1
}
commit 1f851a7e22ff7ce006b389f7b0fb469722a562ec
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 09:32:24 2022 -0400
17344: Note "systemd enable" step is irrelevant in pkg case.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index fb2de8a72..a9fb874e2 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -677,17 +677,20 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/
if err != nil {
return 1
}
- // This is equivalent to "systemd enable", but does
- // not depend on the systemctl program being
- // available.
- symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service"
- err = os.Remove(symlink)
- if err != nil && !errors.Is(err, os.ErrNotExist) {
- return 1
- }
- err = os.Symlink("/lib/systemd/system/arvados.service", symlink)
- if err != nil {
- return 1
+ if prod {
+ // (fpm will do this for us in the pkg case)
+ // This is equivalent to "systemd enable", but
+ // does not depend on the systemctl program
+ // being available:
+ symlink := "/etc/systemd/system/multi-user.target.wants/arvados.service"
+ err = os.Remove(symlink)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ return 1
+ }
+ err = os.Symlink("/lib/systemd/system/arvados.service", symlink)
+ if err != nil {
+ return 1
+ }
}
// Symlink user-facing programs /usr/bin/x ->
commit 3db7310f35339f9605cb75f51d02969170ef4a98
Author: Tom Clegg <tom at curii.com>
Date: Tue Jul 12 00:10:41 2022 -0400
17344: Install ruby arv as arv-ruby, and arvados-client as arv.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index c337d8563..c26cbe887 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -107,6 +107,7 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
"/usr/bin/arvados-client",
"/usr/bin/arvados-server",
"/usr/bin/arv",
+ "/usr/bin/arv-ruby",
"/usr/bin/arv-tag",
"/var/lib/arvados",
"/var/www/.gem",
diff --git a/lib/cli/external.go b/lib/cli/external.go
index 7d9bb6f20..54dfd9a91 100644
--- a/lib/cli/external.go
+++ b/lib/cli/external.go
@@ -57,7 +57,7 @@ func (cmd apiCallCmd) RunCommand(prog string, args []string, stdin io.Reader, st
return 2
}
model := split[len(split)-1]
- return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(model, args), stdin, stdout, stderr)
+ return rubyArvCmd{model}.RunCommand(prog, args, stdin, stdout, stderr)
}
type rubyArvCmd struct {
@@ -65,7 +65,18 @@ type rubyArvCmd struct {
}
func (rc rubyArvCmd) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
- return externalCmd{"arv"}.RunCommand("arv", legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr)
+ wrapprog := "arv-ruby"
+ if _, err := exec.LookPath(wrapprog); err != nil && !strings.Contains(prog, "arv ") {
+ // arv-ruby isn't in PATH (i.e., installation method
+ // wasn't a recent "arvados-server install", which
+ // symlinks /usr/bin/arv-ruby ->
+ // /var/lib/arvados/bin/arv), so fall back to looking
+ // for the arvados-cli program as "arv". (But don't do
+ // this if we are being run as "arv" -- that would
+ // probably cause a recursive fork bomb.)
+ wrapprog = "arv"
+ }
+ return externalCmd{wrapprog}.RunCommand(wrapprog, legacyFlagsToFront(rc.subcommand, args), stdin, stdout, stderr)
}
type externalCmd struct {
@@ -90,7 +101,7 @@ func (ec externalCmd) RunCommand(prog string, args []string, stdin io.Reader, st
return 1
case *exec.Error:
fmt.Fprintln(stderr, err)
- if ec.prog == "arv" {
+ if ec.prog == "arv" || ec.prog == "arv-ruby" {
fmt.Fprint(stderr, rubyInstallHints)
} else if strings.HasPrefix(ec.prog, "arv-") {
fmt.Fprint(stderr, pythonInstallHints)
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 57bb9af51..fb2de8a72 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -692,12 +692,20 @@ rsync -a --delete-after build/ /var/lib/arvados/workbench2/
// Symlink user-facing programs /usr/bin/x ->
// /var/lib/arvados/bin/x
- for _, prog := range []string{"arvados-client", "arvados-server", "arv", "arv-tag"} {
- err = os.Remove("/usr/bin/" + prog)
+ for _, srcdst := range [][]string{
+ {"arvados-client", "arvados-client"},
+ {"arvados-client", "arv"},
+ {"arvados-server", "arvados-server"},
+ {"arv", "arv-ruby"},
+ {"arv-tag", "arv-tag"},
+ } {
+ src := srcdst[0]
+ dst := srcdst[1]
+ err = os.Remove("/usr/bin/" + dst)
if err != nil && !errors.Is(err, os.ErrNotExist) {
return 1
}
- err = os.Symlink("/var/lib/arvados/bin/"+prog, "/usr/bin/"+prog)
+ err = os.Symlink("/var/lib/arvados/bin/"+src, "/usr/bin/"+dst)
if err != nil {
return 1
}
commit 5e7ba70c779986fb28baff847973ca9f62d9d170
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 14:33:45 2022 -0400
17344: 'arvados-client root cmd...' uses root token from config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-client/cmd.go b/cmd/arvados-client/cmd.go
index cb1546211..783fb0ca9 100644
--- a/cmd/arvados-client/cmd.go
+++ b/cmd/arvados-client/cmd.go
@@ -61,6 +61,7 @@ var (
"shell": shellCommand{},
"connect-ssh": connectSSHCommand{},
"diagnostics": diagnostics.Command{},
+ "root": rootCommand{},
})
)
diff --git a/cmd/arvados-client/root.go b/cmd/arvados-client/root.go
new file mode 100644
index 000000000..5d9845ce4
--- /dev/null
+++ b/cmd/arvados-client/root.go
@@ -0,0 +1,48 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package main
+
+import (
+ "flag"
+ "fmt"
+ "io"
+ "os"
+
+ "git.arvados.org/arvados.git/lib/cmd"
+ "git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+)
+
+// rootCommand runs another command using API connection info and
+// SystemRootToken from the system config file instead of the caller's
+// environment vars.
+type rootCommand struct{}
+
+func (rootCommand) RunCommand(prog string, args []string, stdin io.Reader, stdout, stderr io.Writer) int {
+ ldr := config.NewLoader(stdin, ctxlog.New(stderr, "text", "info"))
+ flags := flag.NewFlagSet(prog, flag.ContinueOnError)
+ ldr.SetupFlags(flags)
+ if ok, code := cmd.ParseFlags(flags, prog, args, "subcommand ...", stderr); !ok {
+ return code
+ }
+ cfg, err := ldr.Load()
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ cluster, err := cfg.GetCluster("")
+ if err != nil {
+ fmt.Fprintln(stderr, err)
+ return 1
+ }
+ os.Setenv("ARVADOS_API_HOST", cluster.Services.Controller.ExternalURL.Host)
+ os.Setenv("ARVADOS_API_TOKEN", cluster.SystemRootToken)
+ if cluster.TLS.Insecure {
+ os.Setenv("ARVADOS_API_HOST_INSECURE", "1")
+ } else {
+ os.Unsetenv("ARVADOS_API_HOST_INSECURE")
+ }
+ return handler.RunCommand(prog, flags.Args(), stdin, stdout, stderr)
+}
commit e9a2dc72da07f08c1f003092bdd873cf1fc4e1b6
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 14:00:14 2022 -0400
17344: 'systemctl reload arvados' sends SIGHUP to reload config.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/arvados.service b/lib/install/arvados.service
index cb411c63a..3b68f31e9 100644
--- a/lib/install/arvados.service
+++ b/lib/install/arvados.service
@@ -15,10 +15,11 @@ StartLimitIntervalSec=0
Type=notify
EnvironmentFile=-/etc/arvados/environment
ExecStart=/usr/bin/arvados-server boot
-# Set a reasonable default for the open file limit
-LimitNOFILE=65536
+ExecReload=/usr/bin/arvados-server config-check
+ExecReload=kill -HUP $MAINPID
Restart=always
RestartSec=1
+LimitNOFILE=65536
# systemd<=219 (centos:7, debian:8, ubuntu:trusty) obeys StartLimitInterval in the [Service] section
StartLimitInterval=0
commit 790bf5e75a5cbbd5e7a84092907b51c7be98653c
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 11:28:10 2022 -0400
17344: Ensure arvados-server is in PATH for rake tasks.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 11ce0f188..57bb9af51 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -621,8 +621,8 @@ yarn install
{"chown", "www-data:www-data", ".", "public/assets"},
// {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"},
- {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"},
- {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"},
+ {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "npm:install"},
+ {"sudo", "-u", "www-data", "ARVADOS_CONFIG=none", "RAILS_GROUPS=assets", "RAILS_ENV=production", "PATH=/var/lib/arvados/bin:" + os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "assets:precompile"},
{"chown", "root:root", "."},
{"chown", "-R", "root:root", "public/assets", "vendor"},
commit b4f1381721d332115e7f42241f0c4d3c3b320250
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 09:51:21 2022 -0400
17344: Silence misleading warning during gem install.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index c58450d86..c337d8563 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -34,15 +34,7 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
return fmt.Errorf("arvados-server install failed: exit code %d", exitcode)
}
- cmd := exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- err := cmd.Run()
- if err != nil {
- return fmt.Errorf("gem install fpm: %w", err)
- }
-
- cmd = exec.Command("/var/lib/arvados/bin/gem", "env", "gempath")
+ cmd := exec.Command("/var/lib/arvados/bin/gem", "env", "gempath")
cmd.Stderr = stderr
buf, err := cmd.Output() // /root/.gem/ruby/2.7.0:...
if err != nil || len(buf) == 0 {
@@ -50,6 +42,17 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
}
gempath := string(bytes.TrimRight(bytes.Split(buf, []byte{':'})[0], "\n"))
+ cmd = exec.Command("/var/lib/arvados/bin/gem", "install", "--user", "--no-document", "fpm")
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ // Avoid "WARNING: You don't have [...] in your PATH, gem
+ // executables will not run"
+ cmd.Env = append(os.Environ(), "PATH="+os.Getenv("PATH")+":"+gempath)
+ err = cmd.Run()
+ if err != nil {
+ return fmt.Errorf("gem install fpm: %w", err)
+ }
+
if _, err := os.Stat(gempath + "/gems/fpm-1.11.0/lib/fpm/package/deb.rb"); err == nil {
// Workaround for fpm bug https://github.com/jordansissel/fpm/issues/1739
cmd = exec.Command("sed", "-i", `/require "digest"/a require "zlib"`, gempath+"/gems/fpm-1.11.0/lib/fpm/package/deb.rb")
diff --git a/lib/install/deps.go b/lib/install/deps.go
index a8e95edea..11ce0f188 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -614,9 +614,9 @@ yarn install
{"touch", "log/production.log"},
{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger", "log", "tmp", "vendor", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:" + bundlerversion},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "--set", "--local", "deployment", "true"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "--set", "--local", "path", "/var/www/.gem"},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "--set", "--local", "without", "development test diagnostics performance"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "deployment", "true"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "without", "development test diagnostics performance"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", "8"},
{"chown", "www-data:www-data", ".", "public/assets"},
commit 8c3274edfa29b31f662c3352fa42430ae6a4d412
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 10:14:32 2022 -0400
17344: Remove extra 'go install' cmd.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 6bf390175..a8e95edea 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -563,7 +563,6 @@ yarn install
for _, srcdir := range []string{
"cmd/arvados-client",
"cmd/arvados-server",
- "services/crunch-dispatch-slurm",
} {
fmt.Fprintf(stderr, "building %s...\n", srcdir)
cmd := exec.Command("go", "install", "-ldflags", "-X git.arvados.org/arvados.git/lib/cmd.version="+inst.PackageVersion+" -X main.version="+inst.PackageVersion+" -s -w")
commit 93b596056ed3f6ac7f57db8d2e1bfa61640914f2
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 09:56:27 2022 -0400
17344: Update bundle usage.
[DEPRECATED] The `--deployment` flag is deprecated because it relies
on being remembered across bundler invocations, which bundler will no
longer do in future versions. Instead please use `bundle config set
--local deployment 'true'`, and stop using this flag
[DEPRECATED] The `--path` flag is deprecated because it relies on
being remembered across bundler invocations, which bundler will no
longer do in future versions. Instead please use `bundle config set
--local path '/var/www/.gem'`, and stop using this flag
[DEPRECATED] The `--without` flag is deprecated because it relies on
being remembered across bundler invocations, which bundler will no
longer do in future versions. Instead please use `bundle config set
--local without 'development test diagnostics performance'`, and stop
using this flag
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/deps.go b/lib/install/deps.go
index 017771b63..6bf390175 100644
--- a/lib/install/deps.go
+++ b/lib/install/deps.go
@@ -615,7 +615,10 @@ yarn install
{"touch", "log/production.log"},
{"chown", "-R", "--from=root", "www-data:www-data", "/var/www/.bundle", "/var/www/.gem", "/var/www/.npm", "/var/www/.passenger", "log", "tmp", "vendor", ".bundle", "Gemfile.lock", "config.ru", "config/environment.rb"},
{"sudo", "-u", "www-data", "/var/lib/arvados/bin/gem", "install", "--user", "--conservative", "--no-document", "bundler:" + bundlerversion},
- {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--deployment", "--jobs", "8", "--path", "/var/www/.gem", "--without", "development test diagnostics performance"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "--set", "--local", "deployment", "true"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "--set", "--local", "path", "/var/www/.gem"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "--set", "--local", "without", "development test diagnostics performance"},
+ {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "install", "--jobs", "8"},
{"chown", "www-data:www-data", ".", "public/assets"},
// {"sudo", "-u", "www-data", "/var/lib/arvados/bin/bundle", "config", "set", "--local", "system", "true"},
commit 618fb1a50a910e70d50f097f38afd56ad45c6c17
Author: Tom Clegg <tom at curii.com>
Date: Mon Jul 11 09:41:39 2022 -0400
17344: Check postgresql situation earlier.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index f6691b9b1..11a62f18e 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -111,6 +111,23 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
+ // Do the "create extension" thing early. This way, if there's
+ // no local postgresql server (a likely failure mode), we can
+ // bail out without any side effects, and the user can start
+ // over easily.
+ fmt.Fprintln(stderr, "installing pg_trgm postgresql extension...")
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+ "-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`)
+ cmd.Dir = "/"
+ cmd.Stdout = stdout
+ cmd.Stderr = stderr
+ err = cmd.Run()
+ if err != nil {
+ err = fmt.Errorf("error preparing postgresql server: %w", err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
+
wwwuser, err := user.Lookup("www-data")
if err != nil {
err = fmt.Errorf("user.Lookup(%q): %w", "www-data", err)
@@ -279,6 +296,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
ldr := config.NewLoader(nil, logger)
ldr.SkipLegacy = true
+ ldr.Path = conffile // load the file we just wrote, even if $ARVADOS_CONFIG is set
cfg, err := ldr.Load()
if err != nil {
err = fmt.Errorf("%s: %w", conffile, err)
@@ -297,7 +315,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
fmt.Fprintln(stderr, "...done")
fmt.Fprintln(stderr, "initializing database...")
- cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
+ cmd = exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
cmd.Dir = "/var/lib/arvados/railsapi"
cmd.Stdout = stderr
cmd.Stderr = stderr
@@ -319,6 +337,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
err = fmt.Errorf("%v: %w", cmd.Args, err)
return 1
}
+ fmt.Fprintln(stderr, "...done")
fmt.Fprintln(stderr, "checking controller API endpoint...")
u := url.URL(cluster.Services.Controller.ExternalURL)
@@ -365,7 +384,6 @@ func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.Postgre
cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
"-c", `CREATE USER `+pq.QuoteIdentifier(dbconn["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(dbconn["password"]),
"-c", `CREATE DATABASE `+pq.QuoteIdentifier(dbconn["dbname"])+` WITH TEMPLATE template0 ENCODING 'utf8'`,
- "-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
)
cmd.Dir = "/"
cmd.Stdout = stderr
commit 8bab5f2dcb40fc6b7cb37e73e6878ddc1b9ad3df
Author: Tom Clegg <tom at curii.com>
Date: Sun Jul 10 21:26:14 2022 -0400
17344: Tidy systemd / fpm integration.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/cmd/arvados-package/fpm.go b/cmd/arvados-package/fpm.go
index 9a20ec76f..c58450d86 100644
--- a/cmd/arvados-package/fpm.go
+++ b/cmd/arvados-package/fpm.go
@@ -97,11 +97,10 @@ func fpm(ctx context.Context, opts opts, stdin io.Reader, stdout, stderr io.Writ
"--verbose",
"--deb-use-file-permissions",
"--rpm-use-file-permissions",
- "--deb-systemd", "/etc/systemd/system/multi-user.target.wants/arvados.service",
+ "--deb-systemd", "/lib/systemd/system/arvados.service",
"--deb-systemd-enable",
"--no-deb-systemd-auto-start",
"--no-deb-systemd-restart-after-upgrade",
- "/lib/systemd/system/arvados.service",
"/usr/bin/arvados-client",
"/usr/bin/arvados-server",
"/usr/bin/arv",
commit 4f60d302023918fbf7bf047496a30da9ee3575fb
Author: Tom Clegg <tom at curii.com>
Date: Fri Jul 8 23:02:52 2022 -0400
17344: Improve "init" transcript.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 531c02508..f6691b9b1 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -122,13 +122,15 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
}
initcmd.PostgreSQLPassword = initcmd.RandomHex(32)
+ fmt.Fprintln(stderr, "creating data storage directory /var/lib/arvados/keep ...")
err = os.Mkdir("/var/lib/arvados/keep", 0600)
if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir /var/lib/arvados/keep: %w", err)
return 1
}
- fmt.Fprintln(stderr, "created /var/lib/arvados/keep")
+ fmt.Fprintln(stderr, "...done")
+ fmt.Fprintln(stderr, "creating config file", conffile, "...")
err = os.Mkdir(confdir, 0750)
if err != nil && !os.IsExist(err) {
err = fmt.Errorf("mkdir %s: %w", confdir, err)
@@ -139,9 +141,9 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
err = fmt.Errorf("chown 0:%d %s: %w", wwwgid, confdir, err)
return 1
}
- f, err := os.OpenFile(conffile, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
+ f, err := os.OpenFile(conffile+".tmp", os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644)
if err != nil {
- err = fmt.Errorf("open %s: %w", conffile, err)
+ err = fmt.Errorf("open %s: %w", conffile+".tmp", err)
return 1
}
tmpl, err := template.New("config").Parse(`Clusters:
@@ -260,15 +262,20 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
}
err = tmpl.Execute(f, initcmd)
if err != nil {
- err = fmt.Errorf("%s: tmpl.Execute: %w", conffile, err)
+ err = fmt.Errorf("%s: tmpl.Execute: %w", conffile+".tmp", err)
return 1
}
err = f.Close()
if err != nil {
- err = fmt.Errorf("%s: close: %w", conffile, err)
+ err = fmt.Errorf("%s: close: %w", conffile+".tmp", err)
return 1
}
- fmt.Fprintln(stderr, "created", conffile)
+ err = os.Rename(conffile+".tmp", conffile)
+ if err != nil {
+ err = fmt.Errorf("rename %s -> %s: %w", conffile+".tmp", conffile, err)
+ return 1
+ }
+ fmt.Fprintln(stderr, "...done")
ldr := config.NewLoader(nil, logger)
ldr.SkipLegacy = true
@@ -282,11 +289,14 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
+ fmt.Fprintln(stderr, "creating postresql user and database...")
err = initcmd.createDB(ctx, cluster.PostgreSQL.Connection, stderr)
if err != nil {
return 1
}
+ fmt.Fprintln(stderr, "...done")
+ fmt.Fprintln(stderr, "initializing database...")
cmd := exec.CommandContext(ctx, "sudo", "-u", "www-data", "-E", "HOME=/var/www", "PATH=/var/lib/arvados/bin:"+os.Getenv("PATH"), "/var/lib/arvados/bin/bundle", "exec", "rake", "db:setup")
cmd.Dir = "/var/lib/arvados/railsapi"
cmd.Stdout = stderr
@@ -296,10 +306,10 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
err = fmt.Errorf("rake db:setup failed: %w", err)
return 1
}
- fmt.Fprintln(stderr, "initialized database")
+ fmt.Fprintln(stderr, "...done")
if initcmd.Start {
- fmt.Fprintln(stderr, "starting systemd service")
+ fmt.Fprintln(stderr, "starting systemd service...")
cmd := exec.CommandContext(ctx, "systemctl", "start", "arvados")
cmd.Dir = "/"
cmd.Stdout = stderr
@@ -310,7 +320,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
return 1
}
- fmt.Fprintln(stderr, "checking controller API endpoint")
+ fmt.Fprintln(stderr, "checking controller API endpoint...")
u := url.URL(cluster.Services.Controller.ExternalURL)
conn := rpc.NewConn(cluster.ClusterID, &u, cluster.TLS.Insecure, rpc.PassthroughTokenProvider)
ctx := auth.NewContext(context.Background(), auth.NewCredentials(cluster.SystemRootToken))
@@ -319,9 +329,10 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
err = fmt.Errorf("API request failed: %w", err)
return 1
}
+ fmt.Fprintln(stderr, "...looks good")
}
- fmt.Fprintln(stderr, "Log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String())
+ fmt.Fprintln(stderr, "Setup complete. You should now be able to log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String())
return 0
}
@@ -351,19 +362,17 @@ func (initcmd *initCommand) RandomHex(chars int) string {
}
func (initcmd *initCommand) createDB(ctx context.Context, dbconn arvados.PostgreSQLConnection, stderr io.Writer) error {
- for _, sql := range []string{
- `CREATE USER ` + pq.QuoteIdentifier(dbconn["user"]) + ` WITH SUPERUSER ENCRYPTED PASSWORD ` + pq.QuoteLiteral(dbconn["password"]),
- `CREATE DATABASE ` + pq.QuoteIdentifier(dbconn["dbname"]) + ` WITH TEMPLATE template0 ENCODING 'utf8'`,
- `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
- } {
- cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "-c", sql)
- cmd.Dir = "/"
- cmd.Stdout = stderr
- cmd.Stderr = stderr
- err := cmd.Run()
- if err != nil {
- return fmt.Errorf("error setting up arvados user/database: %w", err)
- }
+ cmd := exec.CommandContext(ctx, "sudo", "-u", "postgres", "psql", "--quiet",
+ "-c", `CREATE USER `+pq.QuoteIdentifier(dbconn["user"])+` WITH SUPERUSER ENCRYPTED PASSWORD `+pq.QuoteLiteral(dbconn["password"]),
+ "-c", `CREATE DATABASE `+pq.QuoteIdentifier(dbconn["dbname"])+` WITH TEMPLATE template0 ENCODING 'utf8'`,
+ "-c", `CREATE EXTENSION IF NOT EXISTS pg_trgm`,
+ )
+ cmd.Dir = "/"
+ cmd.Stdout = stderr
+ cmd.Stderr = stderr
+ err := cmd.Run()
+ if err != nil {
+ return fmt.Errorf("error setting up arvados user/database: %w", err)
}
return nil
}
commit f6d9bb7edd91faf30accb61b03e28a3b6c804708
Author: Tom Clegg <tom at curii.com>
Date: Fri Jul 8 16:44:03 2022 -0400
17344: After init/startup, check API and show wb2 URL.
Arvados-DCO-1.1-Signed-off-by: Tom Clegg <tom at curii.com>
diff --git a/lib/install/init.go b/lib/install/init.go
index 792262b59..531c02508 100644
--- a/lib/install/init.go
+++ b/lib/install/init.go
@@ -13,6 +13,7 @@ import (
"flag"
"fmt"
"io"
+ "net/url"
"os"
"os/exec"
"os/user"
@@ -23,7 +24,9 @@ import (
"git.arvados.org/arvados.git/lib/cmd"
"git.arvados.org/arvados.git/lib/config"
+ "git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/auth"
"git.arvados.org/arvados.git/sdk/go/ctxlog"
"github.com/lib/pq"
)
@@ -297,7 +300,7 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
if initcmd.Start {
fmt.Fprintln(stderr, "starting systemd service")
- cmd := exec.CommandContext(ctx, "systemctl", "start", "--no-block", "arvados")
+ cmd := exec.CommandContext(ctx, "systemctl", "start", "arvados")
cmd.Dir = "/"
cmd.Stdout = stderr
cmd.Stderr = stderr
@@ -306,8 +309,20 @@ func (initcmd *initCommand) RunCommand(prog string, args []string, stdin io.Read
err = fmt.Errorf("%v: %w", cmd.Args, err)
return 1
}
+
+ fmt.Fprintln(stderr, "checking controller API endpoint")
+ u := url.URL(cluster.Services.Controller.ExternalURL)
+ conn := rpc.NewConn(cluster.ClusterID, &u, cluster.TLS.Insecure, rpc.PassthroughTokenProvider)
+ ctx := auth.NewContext(context.Background(), auth.NewCredentials(cluster.SystemRootToken))
+ _, err = conn.UserGetCurrent(ctx, arvados.GetOptions{})
+ if err != nil {
+ err = fmt.Errorf("API request failed: %w", err)
+ return 1
+ }
}
+ fmt.Fprintln(stderr, "Log in to workbench2 at", cluster.Services.Workbench2.ExternalURL.String())
+
return 0
}
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list