[ARVADOS] updated: 1.3.0-2907-g0875fa95a
Git user
git at public.arvados.org
Fri Sep 18 16:17:54 UTC 2020
Summary of changes:
lib/controller/forecast/controller.go | 126 +++++++++++++++++++++++++++++-----
lib/controller/forecast/datapoint.go | 101 +++++++++++++++++++++++++++
2 files changed, 208 insertions(+), 19 deletions(-)
create mode 100644 lib/controller/forecast/datapoint.go
via 0875fa95ae42ef981edac1dff80e3d6566332280 (commit)
from 549c154850bcefddce7b144ce8f05fea410d2a7b (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 0875fa95ae42ef981edac1dff80e3d6566332280
Author: Nico Cesar <nico at nicocesar.com>
Date: Fri Sep 18 12:15:20 2020 -0400
First working datapoint endpoints for CR
refs #16462
Arvados-DCO-1.1-Signed-off-by: Nico Cesar <nico at curii.com>
diff --git a/lib/controller/forecast/controller.go b/lib/controller/forecast/controller.go
index f73dfd834..bedb2328e 100644
--- a/lib/controller/forecast/controller.go
+++ b/lib/controller/forecast/controller.go
@@ -7,8 +7,8 @@ package forecast
import (
"context"
"fmt"
+ "time"
- "git.arvados.org/arvados.git/lib/ctrlctx"
"git.arvados.org/arvados.git/sdk/go/arvados"
)
@@ -26,11 +26,107 @@ type Controller struct {
parent arvados.API
}
-var crs []arvados.ContainerRequest
+func getDatapointsFromSlice(crs []arvados.ContainerRequest) Datapoints {
+ var returnMap Datapoints
+ returnMap = make(Datapoints, len(crs))
-type wrapperCR struct {
- arvados.ContainerRequest
- ID int
+ for key, cr := range crs {
+
+ if returnMap[cr.Name] == nil {
+ x := Datapoint{
+ ContainerRequest: &crs[key],
+ ContainerRequestUUID: cr.UUID,
+ CheckpointName: cr.Name,
+ ContainerUUID: cr.ContainerUUID,
+ }
+ returnMap[cr.Name] = &x
+ }
+ }
+ return returnMap
+}
+
+// ChildContainerRequests will get all the leaves from the container request uuid
+func (ctrl *Controller) ChildContainerRequests(ctx context.Context, uuid string) ([]arvados.ContainerRequest, error) {
+ var crs []arvados.ContainerRequest
+ var crsl arvados.ContainerRequestList
+ limit := int64(100)
+ offset := int64(0)
+
+ //err := config.Arv.Call("GET", "container_requests", "", uuid, nil, &cr)
+ cr, err := ctrl.parent.ContainerRequestGet(ctx, arvados.GetOptions{UUID: uuid})
+
+ if err != nil {
+ return nil, err
+ }
+
+ var filters []arvados.Filter
+ filters = append(filters, arvados.Filter{Attr: "requesting_container_uuid", Operator: "=", Operand: cr.ContainerUUID})
+
+ for len(crsl.Items) != 0 || offset == 0 {
+ //err = config.Arv.List("container_requests", arvadosclient.Dict{"filters": filters, "limit": limit, "offset": offset}, &crsl)
+ crsl, err = ctrl.parent.ContainerRequestList(ctx, arvados.ListOptions{Filters: filters, Limit: limit, Offset: offset})
+
+ if err != nil {
+ return nil, err
+ }
+
+ crs = append(crs, crsl.Items...)
+ offset += limit
+ }
+ return crs, nil
+
+}
+
+// transform will be used to transform a input type of a forecast.Datapoint into the output type arvados.Datapoint that
+// will be sent to the client later.
+func transform(input *Datapoint) (output arvados.Datapoint) {
+ var extraInfo string
+ if input.Reuse() {
+ extraInfo += `Reused`
+ }
+
+ d, err := input.Duration()
+ if err == nil {
+ extraInfo += fmt.Sprintf("Container duration: %s\n", d)
+ }
+
+ legend := fmt.Sprintf(`<p>%s</p><p>Container Request: <a href="https://workbench.x2bo2.arvadosapi.com/container_requests/%s">%s</a></p><p>%s</p>`,
+ input.CheckpointName,
+ input.ContainerRequest.UUID,
+ input.ContainerRequest.UUID,
+ extraInfo)
+
+ // ContainerRequest doesn't have an "end time", so we behave differently it it was reused and when it was not.
+ var endTimeContainerRequest string
+ if input.Reuse() {
+ endTimeContainerRequest = input.ContainerRequest.CreatedAt.Format("2006-01-02 15:04:05.000 -0700")
+ }
+
+ var endTimeContainer string
+ if input.Container.FinishedAt != nil {
+ endTimeContainerRequest = input.Container.FinishedAt.Format("2006-01-02 15:04:05.000 -0700")
+ endTimeContainer = input.Container.FinishedAt.Format("2006-01-02 15:04:05.000 -0700")
+ } else { // we havent finished
+ endTimeContainerRequest = time.Now().Format("2006-01-02 15:04:05.000 -0700")
+ endTimeContainer = time.Now().Format("2006-01-02 15:04:05.000 -0700")
+ }
+
+ var startTimeContainer string
+ if input.Container.StartedAt != nil {
+ startTimeContainer = input.Container.StartedAt.Format("2006-01-02 15:04:05.000 -0700")
+ } else { //container has not even started.
+ startTimeContainer = time.Now().Format("2006-01-02 15:04:05.000 -0700")
+ }
+
+ output.Checkpoint = input.CheckpointName
+ output.Start1 = input.ContainerRequest.CreatedAt.Format("2006-01-02 15:04:05.000 -0700")
+ output.End1 = endTimeContainerRequest
+ output.Start2 = startTimeContainer
+ output.End2 = endTimeContainer
+ output.Reuse = input.Reuse()
+ output.Legend = legend
+
+ return
}
// ForecastDatapoints returns the datapoints we have stored in the database
@@ -38,25 +134,17 @@ type wrapperCR struct {
// https://dev.arvados.org/projects/arvados/wiki/API_HistoricalForcasting_data_for_CR
func (ctrl *Controller) ForecastDatapoints(ctx context.Context, opts arvados.GetOptions) (resp arvados.ForecastDatapointsResponse, err error) {
- // Direct database access
- tx, err := ctrlctx.CurrentTx(ctx)
+ crs, err := ctrl.ChildContainerRequests(ctx, opts.UUID)
if err != nil {
return
}
- query := "select * from container_requests where requesting_container_uuid IN (select container_uuid from container_requests where uuid = $1)"
- var wcr wrapperCR
- rows, err := tx.QueryxContext(ctx, query, opts.UUID)
- if err != nil {
- return
- }
+ datapoints := getDatapointsFromSlice(crs)
- for rows.Next() {
- err = rows.StructScan(&wcr)
- if err != nil {
- return
- }
- fmt.Printf("%v", wcr)
+ // FIXME do this with channels in parallel a
+ for _, datapoint := range datapoints {
+ datapoint.Hydrate(ctx, *ctrl)
+ resp.Datapoints = append(resp.Datapoints, transform(datapoint))
}
return
diff --git a/lib/controller/forecast/datapoint.go b/lib/controller/forecast/datapoint.go
new file mode 100644
index 000000000..3e80264c5
--- /dev/null
+++ b/lib/controller/forecast/datapoint.go
@@ -0,0 +1,101 @@
+// Copyright (C) The Arvados Authors. All rights reserved.
+//
+// SPDX-License-Identifier: AGPL-3.0
+package forecast
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "git.arvados.org/arvados.git/sdk/go/arvados"
+)
+
+// Datapoint structure will hold a container and possible a container request
+// to extract all the information needed.
+// By filling ContainerRequestUUID and/or ContainerUUID the function hydrate
+// will fetch the data. (usually good to make it with a gorutine)
+type Datapoint struct {
+ // CheckpointName is used to group the values, this is the id of the step or
+ // the name of the container usually
+ CheckpointName string
+ ContainerRequestUUID string
+ ContainerRequest *arvados.ContainerRequest
+ ContainerUUID string
+ Container arvados.Container
+
+ // Scattered will be true if the name is "..._NNN". representing that is part of a scattered
+ // process.
+
+ // TODO: As the first process will have no "_NNN" as soon as we find one identifier that matches
+ // scattered pattern, we also assign it to the first process. This is in TODO for now becasue maps
+ // are not thread-safe. We need to implemnt locks to access it, or use sync.Map
+ Scattered bool
+}
+
+// ErrInvalidDuration is returned when Duration() or other functions are expecting a to have a container
+// and is able to calculate the duration of if (i.e. a container has not finished yet)
+type ErrInvalidDuration struct {
+ ContainerUUID string
+}
+
+func (e *ErrInvalidDuration) Error() string {
+ return fmt.Sprintf("forecast: Diration for container request '%s' cant be calculated", e.ContainerUUID)
+}
+
+// Duration returns a time Duration with the container start and stop
+func (d *Datapoint) Duration() (time.Duration, error) {
+ if d.Container.FinishedAt == nil {
+ return time.Duration(0), &ErrInvalidDuration{ContainerUUID: d.ContainerUUID}
+ }
+
+ return d.Container.FinishedAt.Sub(*d.Container.StartedAt), nil
+}
+
+// Reuse returns a boolean based
+func (d *Datapoint) Reuse() bool {
+
+ if d.ContainerRequest == nil {
+ return false
+ }
+ if d.Container.StartedAt == nil {
+ return false
+ }
+
+ return d.ContainerRequest.CreatedAt.After(*d.Container.StartedAt)
+
+}
+
+// ErrNoContainer is returned when Hydrate() or other functions are expecting a container
+// in a datapoint that still doesn't have it (i.e. has not been executed yet)
+type ErrNoContainer struct {
+ ContainerUUID string
+}
+
+func (e *ErrNoContainer) Error() string {
+ return fmt.Sprintf("forecast: Container Request '%s' doesn't have a container yet", e.ContainerUUID)
+}
+
+// Hydrate will make d.containerRequest and d.container with the values from
+// the cloud (or cache) based on the con
+// FIXME how error should be handle?
+func (d *Datapoint) Hydrate(ctx context.Context, ctrl Controller) (err error) {
+ if d.ContainerUUID == "" {
+ return &ErrNoContainer{ContainerUUID: d.ContainerUUID}
+ }
+
+ c, err := ctrl.parent.ContainerGet(ctx, arvados.GetOptions{UUID: d.ContainerUUID})
+ if err != nil {
+ return
+ }
+
+ // after the retrival we assign it to the data point.
+ d.Container = c
+ // TODO: implement cache at this dfunction to
+ return
+}
+
+// Datapoints is map will have checkpointName as the key to fill in the data as we collect it.
+// the starting point could be by parsung stderr.txt logfile from crunch-run or a container
+// request and analyzing the information from the database.
+type Datapoints map[string]*Datapoint
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list