[arvados] updated: 2.7.0-6265-gbdd11f9c2f
git repository hosting
git at public.arvados.org
Tue Apr 30 17:40:17 UTC 2024
Summary of changes:
.../arvados_cluster_activity/main.py | 93 ++++++++++++----------
1 file changed, 53 insertions(+), 40 deletions(-)
via bdd11f9c2f0d9dc9005a9f205cbb037fa3d71be4 (commit)
via cadf87cb581aa0b116c972564edbd771478d87dc (commit)
via 6aa663c821d825dbb71771b368d034de49417116 (commit)
via 6dcd01cf0e7a461a040f9c41c7010ef82ed8b97b (commit)
via 67389f499bfc0449ae34d21bf6d74528b672c6fd (commit)
from 6ca9ad01a665754673293d48a16cbd0bbd392eaa (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit bdd11f9c2f0d9dc9005a9f205cbb037fa3d71be4
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Apr 30 13:39:59 2024 -0400
21121: typo fix store_True -> store_true
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 429deaca3a..4a23b06bd7 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -28,7 +28,7 @@ def parse_arguments(arguments):
arg_parser.add_argument('--cost-report-file', type=str, help='Export cost report to specified CSV file')
arg_parser.add_argument('--include-workflow-steps', type=bool,
default=False,
- action="store_True", help='Include individual workflow steps')
+ action="store_true", help='Include individual workflow steps')
if prometheus_support:
arg_parser.add_argument('--cluster', type=str, help='Cluster to query for prometheus stats')
arg_parser.add_argument('--prometheus-auth', type=str, help='Authorization file with prometheus info')
commit cadf87cb581aa0b116c972564edbd771478d87dc
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Apr 30 13:37:38 2024 -0400
21121: Add --include-workflow-steps
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 2cc61df67d..429deaca3a 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -26,6 +26,9 @@ def parse_arguments(arguments):
arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC), default "now"')
arg_parser.add_argument('--days', type=int, help='Number of days before "end" to start the report')
arg_parser.add_argument('--cost-report-file', type=str, help='Export cost report to specified CSV file')
+ arg_parser.add_argument('--include-workflow-steps', type=bool,
+ default=False,
+ action="store_True", help='Include individual workflow steps')
if prometheus_support:
arg_parser.add_argument('--cluster', type=str, help='Cluster to query for prometheus stats')
arg_parser.add_argument('--prometheus-auth', type=str, help='Authorization file with prometheus info')
@@ -197,7 +200,7 @@ def report_from_prometheus(cluster, since, to):
container_usage(prom, since, to, "sum(arvados_dispatchcloud_instances_price{cluster='%s'})" % cluster, '$%.2f spent on compute', lambda x: x/60)
print()
-def flush_containers(arv_client, csvwriter, pending):
+def flush_containers(arv_client, csvwriter, pending, include_steps):
containers = {}
for container in arvados.util.keyset_list_all(
@@ -240,20 +243,21 @@ def flush_containers(arv_client, csvwriter, pending):
select=["uuid", "full_name", "first_name", "last_name"]):
projects[pr["uuid"]] = pr["full_name"]
- name_regex = re.compile(r"(.+)_[0-9]+")
- child_crs = {}
- for cr in arvados.util.keyset_list_all(
- arv_client.container_requests().list,
- filters=[
- ["requesting_container_uuid", "in", list(containers.keys())],
- ],
- select=["uuid", "name", "cumulative_cost", "requesting_container_uuid", "container_uuid"]):
+ if include_steps:
+ name_regex = re.compile(r"(.+)_[0-9]+")
+ child_crs = {}
+ for cr in arvados.util.keyset_list_all(
+ arv_client.container_requests().list,
+ filters=[
+ ["requesting_container_uuid", "in", list(containers.keys())],
+ ],
+ select=["uuid", "name", "cumulative_cost", "requesting_container_uuid", "container_uuid"]):
- g = name_regex.fullmatch(cr["name"])
- if g:
- cr["name"] = g[1]
+ g = name_regex.fullmatch(cr["name"])
+ if g:
+ cr["name"] = g[1]
- child_crs.setdefault(cr["requesting_container_uuid"], []).append(cr)
+ child_crs.setdefault(cr["requesting_container_uuid"], []).append(cr)
for container_request in pending:
if not container_request["container_uuid"] or not containers[container_request["container_uuid"]]["started_at"] or not containers[container_request["container_uuid"]]["finished_at"]:
@@ -273,24 +277,25 @@ def flush_containers(arv_client, csvwriter, pending):
projects.get(container_request["modified_by_user_uuid"], "unknown user"),
container_request["created_at"],
#"%i:%02i:%02i:%02i" % (length.days, hours, minutes, seconds),
- round(containers[container_request["container_uuid"]]["cost"], 3),
+ round(containers[container_request["container_uuid"]]["cost"] if include_steps else container_request["cumulative_cost"], 3),
container_request["uuid"]
))
- for child_cr in child_crs.get(container_request["container_uuid"], []):
- csvwriter.writerow((
- projects.get(container_request["owner_uuid"], "unknown owner"),
- workflows.get(container_request["properties"].get("template_uuid", "none"), "workflow missing"),
- child_cr["name"],
- container_request["name"],
- projects.get(container_request["modified_by_user_uuid"], "unknown user"),
- child_cr["created_at"],
- round(child_cr["cumulative_cost"], 3),
- child_cr["uuid"]
- ))
-
-
-def report_from_api(since, to, out):
+ if include_steps:
+ for child_cr in child_crs.get(container_request["container_uuid"], []):
+ csvwriter.writerow((
+ projects.get(container_request["owner_uuid"], "unknown owner"),
+ workflows.get(container_request["properties"].get("template_uuid", "none"), "workflow missing"),
+ child_cr["name"],
+ container_request["name"],
+ projects.get(container_request["modified_by_user_uuid"], "unknown user"),
+ child_cr["created_at"],
+ round(child_cr["cumulative_cost"], 3),
+ child_cr["uuid"]
+ ))
+
+
+def report_from_api(since, to, out, include_steps):
arv_client = arvados.api()
csvwriter = csv.writer(out)
@@ -312,10 +317,10 @@ def report_from_api(since, to, out):
else:
count += len(pending)
logging.info("Exporting rows, %s", count)
- flush_containers(arv_client, csvwriter, pending)
+ flush_containers(arv_client, csvwriter, pending, include_steps)
pending.clear()
- flush_containers(arv_client, csvwriter, pending)
+ flush_containers(arv_client, csvwriter, pending, include_steps)
def main(arguments=None):
if arguments is None:
@@ -334,7 +339,7 @@ def main(arguments=None):
if args.cost_report_file:
with open(args.cost_report_file, "wt") as f:
- report_from_api(since, to, f)
+ report_from_api(since, to, f, args.include_workflow_steps)
else:
logging.warn("--cost-report-file not provided, not writing cost report")
commit 6aa663c821d825dbb71771b368d034de49417116
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Apr 30 13:32:45 2024 -0400
21121: Add UUID column
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index e01b00230b..2cc61df67d 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -274,6 +274,7 @@ def flush_containers(arv_client, csvwriter, pending):
container_request["created_at"],
#"%i:%02i:%02i:%02i" % (length.days, hours, minutes, seconds),
round(containers[container_request["container_uuid"]]["cost"], 3),
+ container_request["uuid"]
))
for child_cr in child_crs.get(container_request["container_uuid"], []):
@@ -285,6 +286,7 @@ def flush_containers(arv_client, csvwriter, pending):
projects.get(container_request["modified_by_user_uuid"], "unknown user"),
child_cr["created_at"],
round(child_cr["cumulative_cost"], 3),
+ child_cr["uuid"]
))
@@ -292,10 +294,11 @@ def report_from_api(since, to, out):
arv_client = arvados.api()
csvwriter = csv.writer(out)
- csvwriter.writerow(("Project", "Workflow", "Step", "Sample", "User", "Submitted", "Cost"))
+ csvwriter.writerow(("Project", "Workflow", "Step", "Sample", "User", "Submitted", "Cost", "UUID"))
pending = []
+ count = 0
for container_request in arvados.util.keyset_list_all(
arv_client.container_requests().list,
filters=[
@@ -307,6 +310,8 @@ def report_from_api(since, to, out):
if len(pending) < 1000:
pending.append(container_request)
else:
+ count += len(pending)
+ logging.info("Exporting rows, %s", count)
flush_containers(arv_client, csvwriter, pending)
pending.clear()
commit 6dcd01cf0e7a461a040f9c41c7010ef82ed8b97b
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Apr 30 13:23:51 2024 -0400
21121: Optional prometheus support
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index f677d7f3ee..e01b00230b 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -69,7 +69,7 @@ def parse_arguments(arguments):
exit(1)
- if args.prometheus_auth:
+ if prometheus_support and args.prometheus_auth:
with open(args.prometheus_auth, "rt") as f:
for line in f:
sp = line.strip().split("=")
commit 67389f499bfc0449ae34d21bf6d74528b672c6fd
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Apr 30 13:19:28 2024 -0400
21121: Make the prometheus features optional
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 2da4fbcce1..f677d7f3ee 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -18,6 +18,7 @@ import re
from datetime import timedelta, timezone
import base64
+prometheus_support = True
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
@@ -25,8 +26,9 @@ def parse_arguments(arguments):
arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC), default "now"')
arg_parser.add_argument('--days', type=int, help='Number of days before "end" to start the report')
arg_parser.add_argument('--cost-report-file', type=str, help='Export cost report to specified CSV file')
- arg_parser.add_argument('--cluster', type=str, help='Cluster to query for prometheus stats')
- arg_parser.add_argument('--prometheus-auth', type=str, help='Authorization file with prometheus info')
+ if prometheus_support:
+ arg_parser.add_argument('--cluster', type=str, help='Cluster to query for prometheus stats')
+ arg_parser.add_argument('--prometheus-auth', type=str, help='Authorization file with prometheus info')
args = arg_parser.parse_args(arguments)
@@ -316,13 +318,14 @@ def main(arguments=None):
args, since, to = parse_arguments(arguments)
- if "PROMETHEUS_HOST" in os.environ:
- if args.cluster:
- report_from_prometheus(args.cluster, since, to)
+ if prometheus_support:
+ if "PROMETHEUS_HOST" in os.environ:
+ if args.cluster:
+ report_from_prometheus(args.cluster, since, to)
+ else:
+ logging.warn("--cluster not provided, not collecting activity from Prometheus")
else:
- logging.warn("--cluster not provided, not collecting activity from Prometheus")
- else:
- logging.warn("PROMETHEUS_HOST not found, not collecting activity from Prometheus")
+ logging.warn("PROMETHEUS_HOST not found, not collecting activity from Prometheus")
if args.cost_report_file:
with open(args.cost_report_file, "wt") as f:
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list