[arvados] updated: 2.7.0-5034-g4d3a6d30c4
git repository hosting
git at public.arvados.org
Fri Oct 20 20:47:43 UTC 2023
Summary of changes:
.../arvados_cluster_activity/main.py | 61 +++++++++++++++-------
1 file changed, 43 insertions(+), 18 deletions(-)
via 4d3a6d30c44cb4b99ba889b2f0d4200ab0ad24f1 (commit)
via 2d6e6c2aba1297ddc1cb40a31becc9717b0fe1c6 (commit)
via e17c96b5885f853f04564a9a2fbd7d61ddca65df (commit)
from 4219d52e66a7d555ac0bbc20a60f52534b241b67 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 4d3a6d30c44cb4b99ba889b2f0d4200ab0ad24f1
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Oct 20 16:44:13 2023 -0400
21121: Ignore errors
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 447e30bd43..52ba743070 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -170,15 +170,20 @@ def main(arguments=None):
if prom_user:
headers["Authorization"] = "Basic %s" % str(base64.b64encode(bytes("%s:%s" % (prom_user, prom_pw), 'utf-8')), 'utf-8')
- print(headers)
prom = PrometheusConnect(url=prom_host, headers=headers)
cluster = args.cluster
print(cluster, "between", since, "and", to, "timespan", (to-since))
- data_usage(prom, since, cluster, "at start:")
- data_usage(prom, to - timedelta(minutes=240), cluster, "current :")
+ try:
+ data_usage(prom, since, cluster, "at start:")
+ except:
+ pass
+ try:
+ data_usage(prom, to - timedelta(minutes=240), cluster, "current :")
+ except:
+ pass
container_usage(prom, since, to, "arvados_dispatchcloud_containers_running{cluster='%s'}" % cluster, '%.1f container hours', lambda x: x/60)
container_usage(prom, since, to, "sum(arvados_dispatchcloud_instances_price{cluster='%s'})" % cluster, '$%.2f spent on compute', lambda x: x/60)
commit 2d6e6c2aba1297ddc1cb40a31becc9717b0fe1c6
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Oct 20 16:42:28 2023 -0400
21121: Fix Basic auth
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index a9f7683338..447e30bd43 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -165,12 +165,13 @@ def main(arguments=None):
headers = {}
if prom_token:
- headers["Authorization"] = "Bearer "+prom_token
+ headers["Authorization"] = "Bearer %s" % prom_token
if prom_user:
- headers["Authorization"] = "Basic "+(base64.b64encode("%s:%s" % (prom_user, prom_pw)))
+ headers["Authorization"] = "Basic %s" % str(base64.b64encode(bytes("%s:%s" % (prom_user, prom_pw), 'utf-8')), 'utf-8')
- prom = PrometheusConnect(url=prom_host, headers=header)
+ print(headers)
+ prom = PrometheusConnect(url=prom_host, headers=headers)
cluster = args.cluster
commit e17c96b5885f853f04564a9a2fbd7d61ddca65df
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Fri Oct 20 16:33:24 2023 -0400
21121: Support both Bearer and Basic authentication
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 1d01cd75ab..a9f7683338 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -15,34 +15,44 @@ import os
from prometheus_api_client.utils import parse_datetime
from datetime import timedelta
import pandas
+import base64
from prometheus_api_client import PrometheusConnect, MetricsList, Metric
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--start', help='Start date for the report in YYYY-MM-DD format (UTC)')
- arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC)')
- arg_parser.add_argument('--days', type=int, help='Number of days before now() to start the report')
+ arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC), default "now"')
+ arg_parser.add_argument('--days', type=int, help='Number of days before "end" to start the report')
arg_parser.add_argument('--cluster', type=str, help='Cluster to query')
args = arg_parser.parse_args(arguments)
- if args.days and (args.start or args.end):
+ if args.days and args.start:
arg_parser.print_help()
print("Error: either specify --days or both --start and --end")
exit(1)
- if not args.days and (not args.start or not args.end):
+ if not args.days and not args.start:
arg_parser.print_help()
print("\nError: either specify --days or both --start and --end")
exit(1)
- if (args.start and not args.end) or (args.end and not args.start):
+ if (args.start and not args.end):
arg_parser.print_help()
print("\nError: no start or end date found, either specify --days or both --start and --end")
exit(1)
- if args.days:
+ if args.end:
+ try:
+ to = datetime.datetime.strptime(args.end,"%Y-%m-%d")
+ except:
+ arg_parser.print_help()
+ print("\nError: end date must be in YYYY-MM-DD format")
+ exit(1)
+ else:
to = datetime.datetime.utcnow()
+
+ if args.days:
since = to - datetime.timedelta(days=args.days)
if args.start:
@@ -53,13 +63,6 @@ def parse_arguments(arguments):
print("\nError: start date must be in YYYY-MM-DD format")
exit(1)
- if args.end:
- try:
- to = datetime.datetime.strptime(args.end,"%Y-%m-%d")
- except:
- arg_parser.print_help()
- print("\nError: end date must be in YYYY-MM-DD format")
- exit(1)
return args, since, to
@@ -69,6 +72,10 @@ def data_usage(prom, timestamp, cluster, label):
params={"time": timestamp.timestamp()})
metric_object_list = MetricsList(metric_data)
+
+ if len(metric_data) == 0:
+ return
+
my_metric_object = metric_object_list[0] # one of the metrics from the list
value = my_metric_object.metric_values.iloc[0]["y"]
summary_value = value
@@ -77,6 +84,9 @@ def data_usage(prom, timestamp, cluster, label):
label_config={"cluster": cluster},
params={"time": timestamp.timestamp()})
+ if len(metric_data) == 0:
+ return
+
my_metric_object = MetricsList(metric_data)[0]
dedup_ratio = my_metric_object.metric_values.iloc[0]["y"]
@@ -149,9 +159,18 @@ def main(arguments=None):
#arv = arvados.api()
prom_host = os.environ["PROMETHEUS_HOST"]
- prom_token = os.environ["PROMETHEUS_APIKEY"]
+ prom_token = os.environ.get("PROMETHEUS_APIKEY")
+ prom_user = os.environ.get("PROMETHEUS_USER")
+ prom_pw = os.environ.get("PROMETHEUS_PASSWORD")
+
+ headers = {}
+ if prom_token:
+ headers["Authorization"] = "Bearer "+prom_token
+
+ if prom_user:
+ headers["Authorization"] = "Basic "+(base64.b64encode("%s:%s" % (prom_user, prom_pw)))
- prom = PrometheusConnect(url=prom_host, headers={"Authorization": "Bearer "+prom_token})
+ prom = PrometheusConnect(url=prom_host, headers=header)
cluster = args.cluster
@@ -161,7 +180,7 @@ def main(arguments=None):
data_usage(prom, to - timedelta(minutes=240), cluster, "current :")
container_usage(prom, since, to, "arvados_dispatchcloud_containers_running{cluster='%s'}" % cluster, '%.1f container hours', lambda x: x/60)
- container_usage(prom, since, to, "sum(arvados_dispatchcloud_instances_price{cluster='%s'})" % cluster, '$%.2f spent on compute')
+ container_usage(prom, since, to, "sum(arvados_dispatchcloud_instances_price{cluster='%s'})" % cluster, '$%.2f spent on compute', lambda x: x/60)
print()
if __name__ == "__main__":
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list