[arvados] updated: 2.7.0-6257-gd76df952a1
git repository hosting
git at public.arvados.org
Tue Mar 26 20:56:45 UTC 2024
Summary of changes:
.../arvados_cluster_activity/main.py | 31 ++++++++++++++++------
tools/cluster-activity/setup.py | 2 +-
2 files changed, 24 insertions(+), 9 deletions(-)
via d76df952a1c7d9eddbd28a5284948da75b64f229 (commit)
via 4959458b8ecc651f5a77e18afee2c5d9a6f39530 (commit)
via d85447ed805623aff89d186785c6148d1863075d (commit)
via af8705b9663229b50c7fe9a1a8c4478ab43ff091 (commit)
via 57d6eff23a15e8b2740b737e61be4781f7e42960 (commit)
via 084664fa5194047f57e3974635a3bd4dfdf7c0fe (commit)
from 51ca68aacb0756eea9a9a25a4e4b273df679e086 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit d76df952a1c7d9eddbd28a5284948da75b64f229
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 26 16:56:15 2024 -0400
21121: Specify prometheus auth with a file
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 5447f74904..299f4796f5 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -22,8 +22,10 @@ def parse_arguments(arguments):
arg_parser.add_argument('--start', help='Start date for the report in YYYY-MM-DD format (UTC)')
arg_parser.add_argument('--end', help='End date for the report in YYYY-MM-DD format (UTC), default "now"')
arg_parser.add_argument('--days', type=int, help='Number of days before "end" to start the report')
- arg_parser.add_argument('--cluster', type=str, help='Cluster to query')
arg_parser.add_argument('--cost-report-file', type=str, help='Export cost report to specified CSV file')
+ arg_parser.add_argument('--cluster', type=str, help='Cluster to query for prometheus stats')
+ arg_parser.add_argument('--prometheus-auth', type=str, help='Authorization file with prometheus info')
+
args = arg_parser.parse_args(arguments)
if args.days and args.start:
@@ -63,6 +65,13 @@ def parse_arguments(arguments):
exit(1)
+ if args.prometheus_auth:
+ with open(args.prometheus_auth, "rt") as f:
+ for line in f:
+ sp = line.split("=")
+ if sp[0].startswith("PROMETHEUS_"):
+ os.environ[sp[0]] = sp[1]
+
return args, since, to
def data_usage(prom, timestamp, cluster, label):
@@ -275,7 +284,7 @@ def main(arguments=None):
args, since, to = parse_arguments(arguments)
- if "PROMETHEUS_HOST" in os.environ:
+ if "PROMETHEUS_HOST" in os.environ and args.cluster:
report_from_prometheus(args.cluster, since, to)
if args.cost_report_file:
commit 4959458b8ecc651f5a77e18afee2c5d9a6f39530
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 26 16:48:35 2024 -0400
21121: Fix typo
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/setup.py b/tools/cluster-activity/setup.py
index efd5641b5c..f337adf89c 100755
--- a/tools/cluster-activity/setup.py
+++ b/tools/cluster-activity/setup.py
@@ -33,6 +33,6 @@ setup(name='arvados-user-activity',
install_requires=[
'arvados-python-client >= 2.2.0.dev20201118185221',
],
- extras_require={"prometheus": ["prometheus-api-client"]}
+ extras_require={"prometheus": ["prometheus-api-client"]},
zip_safe=True,
)
commit d85447ed805623aff89d186785c6148d1863075d
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 26 16:47:10 2024 -0400
21121: Don't require prometheus to run cost report
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 1e4c94f6b8..5447f74904 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -12,12 +12,10 @@ import datetime
import ciso8601
import csv
import os
-from prometheus_api_client.utils import parse_datetime
+
from datetime import timedelta
-import pandas
import base64
-from prometheus_api_client import PrometheusConnect, MetricsList, Metric
def parse_arguments(arguments):
arg_parser = argparse.ArgumentParser()
@@ -111,6 +109,10 @@ def data_usage(prom, timestamp, cluster, label):
def container_usage(prom, start_time, end_time, metric, label, fn=None):
+ from prometheus_api_client.utils import parse_datetime
+ from prometheus_api_client import PrometheusConnect, MetricsList, Metric
+ import pandas
+
start = start_time
chunk_size = timedelta(days=1)
cumulative = 0
@@ -151,6 +153,8 @@ def container_usage(prom, start_time, end_time, metric, label, fn=None):
print(label % cumulative)
def report_from_prometheus(cluster, since, to):
+ from prometheus_api_client import PrometheusConnect
+
prom_host = os.environ.get("PROMETHEUS_HOST")
prom_token = os.environ.get("PROMETHEUS_APIKEY")
prom_user = os.environ.get("PROMETHEUS_USER")
diff --git a/tools/cluster-activity/setup.py b/tools/cluster-activity/setup.py
index 4f77f4d37b..efd5641b5c 100755
--- a/tools/cluster-activity/setup.py
+++ b/tools/cluster-activity/setup.py
@@ -32,7 +32,7 @@ setup(name='arvados-user-activity',
],
install_requires=[
'arvados-python-client >= 2.2.0.dev20201118185221',
- 'prometheus-api-client',
],
+ extras_require={"prometheus": ["prometheus-api-client"]}
zip_safe=True,
)
commit af8705b9663229b50c7fe9a1a8c4478ab43ff091
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 26 16:43:00 2024 -0400
21121: Round off costs to tenths of a cent
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index a4eea4c584..1e4c94f6b8 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -237,7 +237,7 @@ def flush_containers(arv_client, csvwriter, pending):
container_request["name"],
containers[container_request["container_uuid"]]["started_at"],
"%i:%02i:%02i:%02i" % (length.days, hours, minutes, seconds),
- container_request["cumulative_cost"],
+ round(container_request["cumulative_cost"], 3),
))
commit 57d6eff23a15e8b2740b737e61be4781f7e42960
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 26 16:39:26 2024 -0400
21121: exclude unfinished
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index 2fe008d67d..a4eea4c584 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -222,6 +222,9 @@ def flush_containers(arv_client, csvwriter, pending):
projects[pr["uuid"]] = pr["full_name"]
for container_request in pending:
+ if not containers[container_request["container_uuid"]]["finished_at"]:
+ continue
+
length = ciso8601.parse_datetime(containers[container_request["container_uuid"]]["finished_at"]) - ciso8601.parse_datetime(containers[container_request["container_uuid"]]["started_at"])
hours = length.seconds // 3600
@@ -246,7 +249,6 @@ def report_from_api(since, to, out):
pending = []
- print(since.isoformat())
for container_request in arvados.util.keyset_list_all(
arv_client.container_requests().list,
filters=[
commit 084664fa5194047f57e3974635a3bd4dfdf7c0fe
Author: Peter Amstutz <peter.amstutz at curii.com>
Date: Tue Mar 26 16:34:32 2024 -0400
21121: Don't require env
Arvados-DCO-1.1-Signed-off-by: Peter Amstutz <peter.amstutz at curii.com>
diff --git a/tools/cluster-activity/arvados_cluster_activity/main.py b/tools/cluster-activity/arvados_cluster_activity/main.py
index b825a2cbb2..2fe008d67d 100755
--- a/tools/cluster-activity/arvados_cluster_activity/main.py
+++ b/tools/cluster-activity/arvados_cluster_activity/main.py
@@ -272,7 +272,7 @@ def main(arguments=None):
if "PROMETHEUS_HOST" in os.environ:
report_from_prometheus(args.cluster, since, to)
- if args.cost_report_file and "ARVADOS_API_HOST" in os.environ:
+ if args.cost_report_file:
with open(args.cost_report_file, "wt") as f:
report_from_api(since, to, f)
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list