[ARVADOS] created: 2.1.0-217-gc0916b956
Git user
git at public.arvados.org
Mon Dec 14 21:43:41 UTC 2020
at c0916b956054b2e16ff61bd33b11bfe07e81787d (commit)
commit c0916b956054b2e16ff61bd33b11bfe07e81787d
Author: Nico Cesar <nico at nicocesar.com>
Date: Mon Dec 14 16:40:26 2020 -0500
Added /a/v1/<cr_uuid>/datapoints endpoint to controller
This builds on top of the work we've done in 17014 this is
the inclusion of the forecast datapoints and the relevant tests
refs #16462
refs #17014
Arvados-DCO-1.1-Signed-off-by: Nico Cesar <nico at curii.com>
diff --git a/.licenseignore b/.licenseignore
index 7ebc82667..881f3b691 100644
--- a/.licenseignore
+++ b/.licenseignore
@@ -86,3 +86,4 @@ sdk/python/tests/fed-migrate/CWLFile
sdk/python/tests/fed-migrate/*.cwl
sdk/python/tests/fed-migrate/*.cwlex
doc/install/*.xlsx
+*.golden
\ No newline at end of file
diff --git a/lib/controller/federation/conn.go b/lib/controller/federation/conn.go
index ffca3b117..0636f3ebc 100644
--- a/lib/controller/federation/conn.go
+++ b/lib/controller/federation/conn.go
@@ -398,6 +398,10 @@ func (conn *Conn) ContainerRequestDelete(ctx context.Context, options arvados.De
return conn.chooseBackend(options.UUID).ContainerRequestDelete(ctx, options)
}
+func (conn *Conn) ForecastDatapoints(ctx context.Context, options arvados.GetOptions) (resp arvados.ForecastDatapointsResponse, err error) {
+ return conn.chooseBackend(options.UUID).ForecastDatapoints(ctx, options)
+}
+
func (conn *Conn) SpecimenList(ctx context.Context, options arvados.ListOptions) (arvados.SpecimenList, error) {
return conn.generated_SpecimenList(ctx, options)
}
diff --git a/lib/controller/forecast/controller.go b/lib/controller/forecast/controller.go
new file mode 100644
index 000000000..bedb2328e
--- /dev/null
+++ b/lib/controller/forecast/controller.go
@@ -0,0 +1,151 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package forecast
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// New returns a new Controller for a cluster and parent API
+func New(cluster *arvados.Cluster, parent arvados.API) *Controller {
+ return &Controller{
+ cluster: cluster,
+ parent: parent,
+ }
+}
+
+// Controller Is the main object to implement the Forecast endpoints
+type Controller struct {
+ cluster *arvados.Cluster
+ parent arvados.API
+}
+
+func getDatapointsFromSlice(crs []arvados.ContainerRequest) Datapoints {
+ var returnMap Datapoints
+ returnMap = make(Datapoints, len(crs))
+
+ for key, cr := range crs {
+
+ if returnMap[cr.Name] == nil {
+ x := Datapoint{
+ ContainerRequest: &crs[key],
+ ContainerRequestUUID: cr.UUID,
+ CheckpointName: cr.Name,
+ ContainerUUID: cr.ContainerUUID,
+ }
+ returnMap[cr.Name] = &x
+ }
+ }
+ return returnMap
+}
+
+// ChildContainerRequests will get all the leaves from the container request uuid
+func (ctrl *Controller) ChildContainerRequests(ctx context.Context, uuid string) ([]arvados.ContainerRequest, error) {
+ var crs []arvados.ContainerRequest
+ var crsl arvados.ContainerRequestList
+ limit := int64(100)
+ offset := int64(0)
+
+ //err := config.Arv.Call("GET", "container_requests", "", uuid, nil, &cr)
+ cr, err := ctrl.parent.ContainerRequestGet(ctx, arvados.GetOptions{UUID: uuid})
+
+ if err != nil {
+ return nil, err
+ }
+
+ var filters []arvados.Filter
+ filters = append(filters, arvados.Filter{Attr: "requesting_container_uuid", Operator: "=", Operand: cr.ContainerUUID})
+
+ for len(crsl.Items) != 0 || offset == 0 {
+ //err = config.Arv.List("container_requests", arvadosclient.Dict{"filters": filters, "limit": limit, "offset": offset}, &crsl)
+ crsl, err = ctrl.parent.ContainerRequestList(ctx, arvados.ListOptions{Filters: filters, Limit: limit, Offset: offset})
+
+ if err != nil {
+ return nil, err
+ }
+
+ crs = append(crs, crsl.Items...)
+ offset += limit
+ }
+ return crs, nil
+
+}
+
+// transform will be used to transform a input type of a forecast.Datapoint into the output type arvados.Datapoint that
+// will be sent to the client later.
+func transform(input *Datapoint) (output arvados.Datapoint) {
+ var extraInfo string
+ if input.Reuse() {
+ extraInfo += `Reused`
+ }
+
+ d, err := input.Duration()
+ if err == nil {
+ extraInfo += fmt.Sprintf("Container duration: %s\n", d)
+ }
+
+ legend := fmt.Sprintf(`<p>%s</p><p>Container Request: <a href="https://workbench.x2bo2.arvadosapi.com/container_requests/%s">%s</a></p><p>%s</p>`,
+ input.CheckpointName,
+ input.ContainerRequest.UUID,
+ input.ContainerRequest.UUID,
+ extraInfo)
+
+ // ContainerRequest doesn't have an "end time", so we behave differently it it was reused and when it was not.
+ var endTimeContainerRequest string
+ if input.Reuse() {
+ endTimeContainerRequest = input.ContainerRequest.CreatedAt.Format("2006-01-02 15:04:05.000 -0700")
+ }
+
+ var endTimeContainer string
+ if input.Container.FinishedAt != nil {
+ endTimeContainerRequest = input.Container.FinishedAt.Format("2006-01-02 15:04:05.000 -0700")
+ endTimeContainer = input.Container.FinishedAt.Format("2006-01-02 15:04:05.000 -0700")
+ } else { // we havent finished
+ endTimeContainerRequest = time.Now().Format("2006-01-02 15:04:05.000 -0700")
+ endTimeContainer = time.Now().Format("2006-01-02 15:04:05.000 -0700")
+ }
+
+ var startTimeContainer string
+ if input.Container.StartedAt != nil {
+ startTimeContainer = input.Container.StartedAt.Format("2006-01-02 15:04:05.000 -0700")
+ } else { //container has not even started.
+ startTimeContainer = time.Now().Format("2006-01-02 15:04:05.000 -0700")
+ }
+
+ output.Checkpoint = input.CheckpointName
+ output.Start1 = input.ContainerRequest.CreatedAt.Format("2006-01-02 15:04:05.000 -0700")
+ output.End1 = endTimeContainerRequest
+ output.Start2 = startTimeContainer
+ output.End2 = endTimeContainer
+ output.Reuse = input.Reuse()
+ output.Legend = legend
+
+ return
+}
+
+// ForecastDatapoints returns the datapoints we have stored in the database
+// for a Container Request UUID. This will follow the specs described in
+// https://dev.arvados.org/projects/arvados/wiki/API_HistoricalForcasting_data_for_CR
+func (ctrl *Controller) ForecastDatapoints(ctx context.Context, opts arvados.GetOptions) (resp arvados.ForecastDatapointsResponse, err error) {
+
+ crs, err := ctrl.ChildContainerRequests(ctx, opts.UUID)
+ if err != nil {
+ return
+ }
+
+ datapoints := getDatapointsFromSlice(crs)
+
+ // FIXME do this with channels in parallel a
+ for _, datapoint := range datapoints {
+ datapoint.Hydrate(ctx, *ctrl)
+ resp.Datapoints = append(resp.Datapoints, transform(datapoint))
+ }
+
+ return
+}
diff --git a/lib/controller/forecast/controller_test.go b/lib/controller/forecast/controller_test.go
new file mode 100644
index 000000000..7b45fe54c
--- /dev/null
+++ b/lib/controller/forecast/controller_test.go
@@ -0,0 +1,128 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+
+package forecast
+
+import (
+ "context"
+ "encoding/json"
+ "flag"
+ "io/ioutil"
+ "net/url"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "git.arvados.org/arvados.git/lib/controller/rpc"
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+ "git.arvados.org/arvados.git/sdk/go/arvadostest"
+ "git.arvados.org/arvados.git/sdk/go/auth"
+ "git.arvados.org/arvados.git/sdk/go/ctxlog"
+
+ check "gopkg.in/check.v1"
+)
+
+// Gocheck boilerplate
+func Test(t *testing.T) {
+ check.TestingT(t)
+}
+
+var _ = check.Suite(&ForecastSuite{})
+
+type ForecastSuite struct {
+ ctrl *Controller
+ ctx context.Context
+ stub *arvadostest.APIStub
+ rollback func()
+}
+
+func integrationTestCluster() *arvados.Cluster {
+ cfg, err := arvados.GetConfig(filepath.Join(os.Getenv("WORKSPACE"), "tmp", "arvados.yml"))
+ if err != nil {
+ panic(err)
+ }
+ cc, err := cfg.GetCluster("zzzzz")
+ if err != nil {
+ panic(err)
+ }
+ return cc
+}
+
+func (s *ForecastSuite) SetUpTest(c *check.C) {
+ s.ctx = context.Background()
+ // default user that has access to the fixtures
+ s.ctx = auth.NewContext(s.ctx, &auth.Credentials{Tokens: []string{arvadostest.ActiveTokenV2}})
+ s.ctx = ctxlog.Context(s.ctx, ctxlog.New(os.Stderr, "json", "debug"))
+ cluster := &arvados.Cluster{
+ ClusterID: "zzzzz",
+ PostgreSQL: integrationTestCluster().PostgreSQL,
+ }
+ cluster.API.RequestTimeout = arvados.Duration(5 * time.Minute)
+ cluster.TLS.Insecure = true
+ arvadostest.SetServiceURL(&cluster.Services.RailsAPI, "https://"+os.Getenv("ARVADOS_TEST_API_HOST"))
+ arvadostest.SetServiceURL(&cluster.Services.Controller, "http://localhost:/")
+
+ s.ctrl = New(cluster, rpc.NewConn(
+ cluster.ClusterID,
+ &url.URL{Scheme: "https", Host: os.Getenv("ARVADOS_TEST_API_HOST")},
+ true, rpc.PassthroughTokenProvider))
+
+}
+
+func (s *ForecastSuite) TearDownTest(c *check.C) {
+ if s.rollback != nil {
+ s.rollback()
+ s.rollback = nil
+ }
+}
+
+func (s *ForecastSuite) TestDatapoints(c *check.C) {
+ // Just as basic test to make sure the datapoints endpoint is there and giving out some data
+ // HasherRootUUID (zzzzz-xvhdp-p1i7h1gy5z1ft4p) is hasher_root from services/api/test/fixtures/container_requests.yml
+ // this container request start all the
+ resp, err := s.ctrl.ForecastDatapoints(s.ctx, arvados.GetOptions{UUID: arvadostest.HasherRootUUID})
+ c.Check(err, check.IsNil)
+ c.Check(len(resp.Datapoints), check.Equals, 3)
+}
+
+// A great way to update golden files if needed: go test -update
+var update = flag.Bool("update", false, "Update golden files")
+
+func (s *ForecastSuite) TestDatapointsValues(c *check.C) {
+ cases := []struct {
+ Name string
+ Checkpoint string
+ }{
+ {"hasher1_data", "hasher1"},
+ {"hasher2_data", "hasher2"},
+ {"hasher3_data", "hasher3"},
+ }
+ resp, err := s.ctrl.ForecastDatapoints(s.ctx, arvados.GetOptions{UUID: arvadostest.HasherRootUUID})
+ c.Check(err, check.IsNil)
+
+ for _, tc := range cases {
+ var actual arvados.Datapoint
+ for _, d := range resp.Datapoints {
+ if d.Checkpoint == tc.Checkpoint {
+ actual = d
+ }
+ }
+ c.Check(actual, check.NotNil)
+
+ actualBytes, err := json.Marshal(actual)
+ c.Check(err, check.IsNil)
+
+ golden := filepath.Join("test-fixtures", tc.Name+".golden")
+
+ if *update {
+ err = ioutil.WriteFile(golden, actualBytes, 0644)
+ c.Check(err, check.IsNil)
+ }
+
+ expected, err := ioutil.ReadFile(golden)
+ c.Check(err, check.IsNil)
+ c.Check(actualBytes, check.DeepEquals, expected)
+ }
+}
diff --git a/lib/controller/forecast/datapoint.go b/lib/controller/forecast/datapoint.go
new file mode 100644
index 000000000..31c989ee8
--- /dev/null
+++ b/lib/controller/forecast/datapoint.go
@@ -0,0 +1,101 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+package forecast
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// Datapoint structure will hold a container and possible a container request
+// to extract all the information needed.
+// By filling ContainerRequestUUID and/or ContainerUUID the function hydrate
+// will fetch the data. (usually good to make it with a gorutine)
+type Datapoint struct {
+ // CheckpointName is used to group the values, this is the id of the step or
+ // the name of the container usually
+ CheckpointName string
+ ContainerRequestUUID string
+ ContainerRequest *arvados.ContainerRequest
+ ContainerUUID string
+ Container arvados.Container
+
+ // Scattered will be true if the name is "..._NNN". representing that is part of a scattered
+ // process.
+
+ // TODO: As the first process will have no "_NNN" as soon as we find one identifier that matches
+ // scattered pattern, we also assign it to the first process. This is in TODO for now becasue maps
+ // are not thread-safe. We need to implemnt locks to access it, or use sync.Map
+ Scattered bool
+}
+
+// ErrInvalidDuration is returned when Duration() or other functions are expecting a to have a container
+// and is able to calculate the duration of if (i.e. a container has not finished yet)
+type ErrInvalidDuration struct {
+ ContainerUUID string
+}
+
+func (e *ErrInvalidDuration) Error() string {
+ return fmt.Sprintf("forecast: Duration for container request '%s' can't be calculated", e.ContainerUUID)
+}
+
+// Duration returns a time Duration with the container start and stop
+func (d *Datapoint) Duration() (time.Duration, error) {
+ if d.Container.FinishedAt == nil {
+ return time.Duration(0), &ErrInvalidDuration{ContainerUUID: d.ContainerUUID}
+ }
+
+ return d.Container.FinishedAt.Sub(*d.Container.StartedAt), nil
+}
+
+// Reuse returns a boolean based
+func (d *Datapoint) Reuse() bool {
+
+ if d.ContainerRequest == nil {
+ return false
+ }
+ if d.Container.StartedAt == nil {
+ return false
+ }
+
+ return d.ContainerRequest.CreatedAt.After(*d.Container.StartedAt)
+
+}
+
+// ErrNoContainer is returned when Hydrate() or other functions are expecting a container
+// in a datapoint that still doesn't have it (i.e. has not been executed yet)
+type ErrNoContainer struct {
+ ContainerUUID string
+}
+
+func (e *ErrNoContainer) Error() string {
+ return fmt.Sprintf("forecast: Container Request '%s' doesn't have a container yet", e.ContainerUUID)
+}
+
+// Hydrate will make d.containerRequest and d.container with the values from
+// the cloud (or cache) based on the con
+// FIXME how error should be handle?
+func (d *Datapoint) Hydrate(ctx context.Context, ctrl Controller) (err error) {
+ if d.ContainerUUID == "" {
+ return &ErrNoContainer{ContainerUUID: d.ContainerUUID}
+ }
+
+ c, err := ctrl.parent.ContainerGet(ctx, arvados.GetOptions{UUID: d.ContainerUUID})
+ if err != nil {
+ return
+ }
+
+ // after the retrival we assign it to the data point.
+ d.Container = c
+ // TODO: implement cache at this dfunction to
+ return
+}
+
+// Datapoints is map will have checkpointName as the key to fill in the data as we collect it.
+// the starting point could be by parsung stderr.txt logfile from crunch-run or a container
+// request and analyzing the information from the database.
+type Datapoints map[string]*Datapoint
diff --git a/lib/controller/forecast/test-fixtures/hasher1_data.golden b/lib/controller/forecast/test-fixtures/hasher1_data.golden
new file mode 100644
index 000000000..8dae64618
--- /dev/null
+++ b/lib/controller/forecast/test-fixtures/hasher1_data.golden
@@ -0,0 +1 @@
+{"checkpoint":"hasher1","start_1":"2020-09-28 15:04:38.079 +0000","end_1":"2020-09-28 15:04:40.708 +0000","start_2":"2020-09-28 15:04:40.364 +0000","end_2":"2020-09-28 15:04:40.708 +0000","reuse":false,"legend":"\u003cp\u003ehasher1\u003c/p\u003e\u003cp\u003eContainer Request: \u003ca href=\"https://workbench.x2bo2.arvadosapi.com/container_requests/zzzzz-xvhdp-et5v1vofm3109fn\"\u003ezzzzz-xvhdp-et5v1vofm3109fn\u003c/a\u003e\u003c/p\u003e\u003cp\u003eContainer duration: 343.564ms\n\u003c/p\u003e"}
\ No newline at end of file
diff --git a/lib/controller/forecast/test-fixtures/hasher2_data.golden b/lib/controller/forecast/test-fixtures/hasher2_data.golden
new file mode 100644
index 000000000..a2b1c62c2
--- /dev/null
+++ b/lib/controller/forecast/test-fixtures/hasher2_data.golden
@@ -0,0 +1 @@
+{"checkpoint":"hasher2","start_1":"2020-09-28 15:04:50.144 +0000","end_1":"2020-09-28 15:04:52.783 +0000","start_2":"2020-09-28 15:04:52.463 +0000","end_2":"2020-09-28 15:04:52.783 +0000","reuse":false,"legend":"\u003cp\u003ehasher2\u003c/p\u003e\u003cp\u003eContainer Request: \u003ca href=\"https://workbench.x2bo2.arvadosapi.com/container_requests/zzzzz-xvhdp-k2i0vu6n1ebsyvo\"\u003ezzzzz-xvhdp-k2i0vu6n1ebsyvo\u003c/a\u003e\u003c/p\u003e\u003cp\u003eContainer duration: 319.663ms\n\u003c/p\u003e"}
\ No newline at end of file
diff --git a/lib/controller/forecast/test-fixtures/hasher3_data.golden b/lib/controller/forecast/test-fixtures/hasher3_data.golden
new file mode 100644
index 000000000..42941eb4f
--- /dev/null
+++ b/lib/controller/forecast/test-fixtures/hasher3_data.golden
@@ -0,0 +1 @@
+{"checkpoint":"hasher3","start_1":"2020-09-28 15:05:02.111 +0000","end_1":"2020-09-28 15:05:04.783 +0000","start_2":"2020-09-28 15:05:04.490 +0000","end_2":"2020-09-28 15:05:04.783 +0000","reuse":false,"legend":"\u003cp\u003ehasher3\u003c/p\u003e\u003cp\u003eContainer Request: \u003ca href=\"https://workbench.x2bo2.arvadosapi.com/container_requests/zzzzz-xvhdp-37vzxz1l2k2ywvd\"\u003ezzzzz-xvhdp-37vzxz1l2k2ywvd\u003c/a\u003e\u003c/p\u003e\u003cp\u003eContainer duration: 293.211ms\n\u003c/p\u003e"}
\ No newline at end of file
diff --git a/lib/controller/localdb/conn.go b/lib/controller/localdb/conn.go
index d197675f8..f8ff90482 100644
--- a/lib/controller/localdb/conn.go
+++ b/lib/controller/localdb/conn.go
@@ -7,6 +7,7 @@ package localdb
import (
"context"
+ "git.arvados.org/arvados.git/lib/controller/forecast"
"git.arvados.org/arvados.git/lib/controller/railsproxy"
"git.arvados.org/arvados.git/lib/controller/rpc"
"git.arvados.org/arvados.git/sdk/go/arvados"
@@ -18,6 +19,7 @@ type Conn struct {
cluster *arvados.Cluster
*railsProxy // handles API methods that aren't defined on Conn itself
loginController
+ forecast *forecast.Controller
}
func NewConn(cluster *arvados.Cluster) *Conn {
@@ -26,6 +28,7 @@ func NewConn(cluster *arvados.Cluster) *Conn {
conn = Conn{
cluster: cluster,
railsProxy: railsProxy,
+ forecast: forecast.New(cluster, &conn),
}
conn.loginController = chooseLoginController(cluster, &conn)
return &conn
@@ -45,3 +48,8 @@ func (conn *Conn) Login(ctx context.Context, opts arvados.LoginOptions) (arvados
func (conn *Conn) UserAuthenticate(ctx context.Context, opts arvados.UserAuthenticateOptions) (arvados.APIClientAuthorization, error) {
return conn.loginController.UserAuthenticate(ctx, opts)
}
+
+// Forecast Datapoints handles the container request's datapoint for forecasting purposes
+func (conn *Conn) ForecastDatapoints(ctx context.Context, opts arvados.GetOptions) (resp arvados.ForecastDatapointsResponse, err error) {
+ return conn.forecast.ForecastDatapoints(ctx, opts)
+}
diff --git a/lib/controller/router/router.go b/lib/controller/router/router.go
index 9fb2a0d32..9776a4f56 100644
--- a/lib/controller/router/router.go
+++ b/lib/controller/router/router.go
@@ -203,6 +203,14 @@ func (rtr *router) addRoutes() {
return rtr.backend.ContainerRequestDelete(ctx, *opts.(*arvados.DeleteOptions))
},
},
+ {
+ arvados.EndpointForecastDatapoint,
+ func() interface{} { return &arvados.GetOptions{} },
+ func(ctx context.Context, opts interface{}) (interface{}, error) {
+ return rtr.backend.ForecastDatapoints(ctx, *opts.(*arvados.GetOptions))
+ },
+ },
+
{
arvados.EndpointContainerLock,
func() interface{} {
diff --git a/lib/controller/rpc/conn.go b/lib/controller/rpc/conn.go
index 5ffa66801..36738f64f 100644
--- a/lib/controller/rpc/conn.go
+++ b/lib/controller/rpc/conn.go
@@ -321,6 +321,13 @@ func (conn *Conn) ContainerRequestDelete(ctx context.Context, options arvados.De
return resp, err
}
+func (conn *Conn) ForecastDatapoints(ctx context.Context, options arvados.GetOptions) (arvados.ForecastDatapointsResponse, error) {
+ ep := arvados.EndpointForecastDatapoint
+ var resp arvados.ForecastDatapointsResponse
+ err := conn.requestAndDecode(ctx, &resp, ep, nil, options)
+ return resp, err
+}
+
func (conn *Conn) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
ep := arvados.EndpointSpecimenCreate
var resp arvados.Specimen
diff --git a/sdk/go/arvados/api.go b/sdk/go/arvados/api.go
index a11872971..8af0c831f 100644
--- a/sdk/go/arvados/api.go
+++ b/sdk/go/arvados/api.go
@@ -46,6 +46,7 @@ var (
EndpointContainerRequestGet = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}", ""}
EndpointContainerRequestList = APIEndpoint{"GET", "arvados/v1/container_requests", ""}
EndpointContainerRequestDelete = APIEndpoint{"DELETE", "arvados/v1/container_requests/{uuid}", ""}
+ EndpointForecastDatapoint = APIEndpoint{"GET", "arvados/v1/container_requests/{uuid}/datapoints", ""}
EndpointUserActivate = APIEndpoint{"POST", "arvados/v1/users/{uuid}/activate", ""}
EndpointUserCreate = APIEndpoint{"POST", "arvados/v1/users", "user"}
EndpointUserCurrent = APIEndpoint{"GET", "arvados/v1/users/current", ""}
@@ -185,6 +186,7 @@ type API interface {
ContainerRequestGet(ctx context.Context, options GetOptions) (ContainerRequest, error)
ContainerRequestList(ctx context.Context, options ListOptions) (ContainerRequestList, error)
ContainerRequestDelete(ctx context.Context, options DeleteOptions) (ContainerRequest, error)
+ ForecastDatapoints(ctx context.Context, options GetOptions) (ForecastDatapointsResponse, error)
SpecimenCreate(ctx context.Context, options CreateOptions) (Specimen, error)
SpecimenUpdate(ctx context.Context, options UpdateOptions) (Specimen, error)
SpecimenGet(ctx context.Context, options GetOptions) (Specimen, error)
diff --git a/sdk/go/arvados/forecast.go b/sdk/go/arvados/forecast.go
new file mode 100644
index 000000000..fab91d4ef
--- /dev/null
+++ b/sdk/go/arvados/forecast.go
@@ -0,0 +1,33 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+// Datapoint is the format that gets outputted to the user
+type Datapoint struct {
+ //UUID string `json:"uuid"` // REVIEW: do we want each individual datapoint with a UUID
+
+ // This is a generic "Checkpoint" name, that is derived from the container name. For Example,
+ // bwamem_2902 will become checkpoint "bwamem", making this easy to agregate values
+
+ Checkpoint string `json:"checkpoint"`
+ Start1 string `json:"start_1"`
+ End1 string `json:"end_1"`
+ Start2 string `json:"start_2"`
+ End2 string `json:"end_2"`
+ Reuse bool `json:"reuse"`
+ Legend string `json:"legend"`
+}
+
+// ForecastDatapointsOptions will have the paramenter to fetch from the local or remote cluster
+// all datapoins
+//type ForecastDatapointsOptions struct {
+// ClusterID string `json:"cluster_id"`
+// ContainerRequestUUID string `json:"container_request_uuid"`
+//}
+
+// ForecastDatapointsResponse is the format that the user gets the data.
+type ForecastDatapointsResponse struct {
+ Datapoints []Datapoint `json:"datapoints"`
+}
diff --git a/sdk/go/arvados/forecaster.go b/sdk/go/arvados/forecaster.go
new file mode 100644
index 000000000..963e70888
--- /dev/null
+++ b/sdk/go/arvados/forecaster.go
@@ -0,0 +1,42 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: Apache-2.0
+
+package arvados
+
+// CheckpointNode is the representation with dependencies for the checkpoints as a graph
+// this is usefull to split the dependency tree code from the specific values.
+type CheckpointNode struct {
+ Name string `json:"name"`
+
+ // Dependencies are what is needed to run
+ Dependencies []*CheckpointNode `json:"dependencies"`
+}
+
+// Checkpoint is an individual ckeckpoint. All times are expressed in seconds,
+// we should review this decision when integrating to arvados if they make sense
+// or not.
+type Checkpoint struct {
+ CheckpointNode
+ // TimeCummulative is the time rounded to the nearest secod to run this checkpoint.
+ // is used in TimeAvg() to return the average time needed for the run
+
+ TimeAvg float64 `json:"time_average,omitempty"`
+ TimeCount int `json:"time_count,omitempty"`
+ // keep track of the max and min time for from the cache.
+ TimeMin float64 `json:"time_min,omitempty"`
+ TimeMax float64 `json:"time_max,omitempty"`
+
+ TimeMinComment string `json:"time_min_comment,omitempty"`
+ TimeMaxComment string `json:"time_max_comment,omitempty"`
+ //In the future ScatterCummulative and Scatter Count will be used in ScatterAvg() to
+ // return is the average number of Scattered, this will help to estimate costs of a run.
+ //ScatterCummulative uint
+ //ScatterCount int
+}
+
+type Checkpoints struct {
+ UUID string `json:"uuid"`
+ Checkpoints []Checkpoint `json:"checkpoints"`
+ //EstimatedToltalTime float64 `json:"estimated_total_time"`
+}
diff --git a/sdk/go/arvadostest/api.go b/sdk/go/arvadostest/api.go
index df3e46feb..a3725ed21 100644
--- a/sdk/go/arvadostest/api.go
+++ b/sdk/go/arvadostest/api.go
@@ -125,6 +125,10 @@ func (as *APIStub) ContainerRequestDelete(ctx context.Context, options arvados.D
as.appendCall(ctx, as.ContainerRequestDelete, options)
return arvados.ContainerRequest{}, as.Error
}
+func (as *APIStub) ForecastDatapoints(ctx context.Context, options arvados.GetOptions) (resp arvados.ForecastDatapointsResponse, err error) {
+ as.appendCall(ctx, as.ForecastDatapoints, options)
+ return arvados.ForecastDatapointsResponse{}, as.Error
+}
func (as *APIStub) SpecimenCreate(ctx context.Context, options arvados.CreateOptions) (arvados.Specimen, error) {
as.appendCall(ctx, as.SpecimenCreate, options)
return arvados.Specimen{}, as.Error
diff --git a/sdk/go/arvadostest/fixtures.go b/sdk/go/arvadostest/fixtures.go
index aeb5a47e6..fd7bef5ca 100644
--- a/sdk/go/arvadostest/fixtures.go
+++ b/sdk/go/arvadostest/fixtures.go
@@ -66,6 +66,15 @@ const (
Hasher2LogCollectionUUID = "zzzzz-4zz18-dlogcollhash002"
Hasher3LogCollectionUUID = "zzzzz-4zz18-dlogcollhash003"
+ HasherRootUUID = "zzzzz-xvhdp-p1i7h1gy5z1ft4p"
+ HasherRootContainerUUID = "zzzzz-dz642-gxfj5yt5x6x6nr2"
+ HasherHasher1UUID = "zzzzz-xvhdp-et5v1vofm3109fn"
+ HasherHasher1ContainerUUID = "zzzzz-dz642-1wfe84lcxvojijy"
+ HasherHasher2UUID = "zzzzz-xvhdp-k2i0vu6n1ebsyvo"
+ HasherHasher2ContainerUUID = "zzzzz-dz642-80yksfp68lxz3gs"
+ HasherHasher3UUID = "zzzzz-xvhdp-37vzxz1l2k2ywvd"
+ HasherHasher3ContainerUUID = "zzzzz-dz642-xzw7j59tkq7u8pe"
+
ArvadosRepoUUID = "zzzzz-s0uqq-arvadosrepo0123"
ArvadosRepoName = "arvados"
FooRepoUUID = "zzzzz-s0uqq-382brsig8rp3666"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list