[arvados] created: 2.6.0-137-ged3063dd8
git repository hosting
git at public.arvados.org
Sat May 6 18:20:04 UTC 2023
at ed3063dd8a2c6e43c235420c75cf58c7a553e74f (commit)
commit ed3063dd8a2c6e43c235420c75cf58c7a553e74f
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Sat May 6 15:18:54 2023 -0300
20482: Allow the site admin to create a non-public Arvados cluster.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/tools/salt-install/terraform/aws/services/locals.tf b/tools/salt-install/terraform/aws/services/locals.tf
index 523954ce3..d515453cb 100644
--- a/tools/salt-install/terraform/aws/services/locals.tf
+++ b/tools/salt-install/terraform/aws/services/locals.tf
@@ -6,11 +6,14 @@ locals {
region_name = data.terraform_remote_state.vpc.outputs.region_name
cluster_name = data.terraform_remote_state.vpc.outputs.cluster_name
use_external_db = data.terraform_remote_state.data-storage.outputs.use_external_db
+ private_only = data.terraform_remote_state.vpc.outputs.private_only
public_ip = data.terraform_remote_state.vpc.outputs.public_ip
private_ip = data.terraform_remote_state.vpc.outputs.private_ip
pubkey_path = pathexpand(var.pubkey_path)
- pubkey_name = "arvados-deployer-key"
+ pubkey_name = "${local.cluster_name}-arvados-deployer-key"
public_hosts = data.terraform_remote_state.vpc.outputs.public_hosts
private_hosts = data.terraform_remote_state.vpc.outputs.private_hosts
+ user_facing_hosts = data.terraform_remote_state.vpc.outputs.user_facing_hosts
+ internal_service_hosts = data.terraform_remote_state.vpc.outputs.internal_service_hosts
ssl_password_secret_name = "${local.cluster_name}-${var.ssl_password_secret_name_suffix}"
}
diff --git a/tools/salt-install/terraform/aws/services/main.tf b/tools/salt-install/terraform/aws/services/main.tf
index 7ec3b954e..4c96523e7 100644
--- a/tools/salt-install/terraform/aws/services/main.tf
+++ b/tools/salt-install/terraform/aws/services/main.tf
@@ -36,6 +36,7 @@ resource "aws_iam_instance_profile" "dispatcher_instance_profile" {
resource "aws_secretsmanager_secret" "ssl_password_secret" {
name = local.ssl_password_secret_name
+ recovery_window_in_days = 0
}
resource "aws_iam_instance_profile" "default_instance_profile" {
@@ -52,7 +53,7 @@ resource "aws_instance" "arvados_service" {
"hostname": each.value
})
private_ip = local.private_ip[each.value]
- subnet_id = contains(local.public_hosts, each.value) ? data.terraform_remote_state.vpc.outputs.public_subnet_id : data.terraform_remote_state.vpc.outputs.private_subnet_id
+ subnet_id = contains(local.user_facing_hosts, each.value) ? data.terraform_remote_state.vpc.outputs.public_subnet_id : data.terraform_remote_state.vpc.outputs.private_subnet_id
vpc_security_group_ids = [ data.terraform_remote_state.vpc.outputs.arvados_sg_id ]
# This should be done in a more readable way
iam_instance_profile = each.value == "controller" ? aws_iam_instance_profile.dispatcher_instance_profile.name : length(regexall("^keep[0-9]+", each.value)) > 0 ? aws_iam_instance_profile.keepstore_instance_profile.name : aws_iam_instance_profile.default_instance_profile.name
@@ -107,7 +108,7 @@ resource "aws_iam_policy_attachment" "cloud_dispatcher_ec2_access_attachment" {
}
resource "aws_eip_association" "eip_assoc" {
- for_each = toset(local.public_hosts)
+ for_each = local.private_only ? [] : toset(local.public_hosts)
instance_id = aws_instance.arvados_service[each.value].id
allocation_id = data.terraform_remote_state.vpc.outputs.eip_id[each.value]
}
diff --git a/tools/salt-install/terraform/aws/vpc/locals.tf b/tools/salt-install/terraform/aws/vpc/locals.tf
index 00e9d9494..a6e56c585 100644
--- a/tools/salt-install/terraform/aws/vpc/locals.tf
+++ b/tools/salt-install/terraform/aws/vpc/locals.tf
@@ -9,10 +9,18 @@ locals {
ssh: "22",
}
availability_zone = data.aws_availability_zones.available.names[0]
- public_hosts = [ "controller", "workbench" ]
- private_hosts = [ "keep0", "shell" ]
+ route53_public_zone = one(aws_route53_zone.public_zone[*])
+ iam_user_letsencrypt = one(aws_iam_user.letsencrypt[*])
+ iam_access_key_letsencrypt = one(aws_iam_access_key.letsencrypt[*])
+ public_hosts = var.private_only ? [] : var.user_facing_hosts
+ private_hosts = concat(
+ var.internal_service_hosts,
+ var.private_only ? var.user_facing_hosts : []
+ )
arvados_dns_zone = "${var.cluster_name}.${var.domain_name}"
- public_ip = { for k, v in aws_eip.arvados_eip: k => v.public_ip }
+ public_ip = {
+ for k, v in aws_eip.arvados_eip: k => v.public_ip
+ }
private_ip = {
"controller": "10.1.1.11",
"workbench": "10.1.1.15",
diff --git a/tools/salt-install/terraform/aws/vpc/main.tf b/tools/salt-install/terraform/aws/vpc/main.tf
index eba48b9f9..6f1fe96ec 100644
--- a/tools/salt-install/terraform/aws/vpc/main.tf
+++ b/tools/salt-install/terraform/aws/vpc/main.tf
@@ -135,10 +135,11 @@ resource "aws_security_group" "arvados_sg" {
# PUBLIC DNS
resource "aws_route53_zone" "public_zone" {
+ count = var.private_only ? 0 : 1
name = local.arvados_dns_zone
}
resource "aws_route53_record" "public_a_record" {
- zone_id = aws_route53_zone.public_zone.id
+ zone_id = try(local.route53_public_zone.id, "")
for_each = local.public_ip
name = each.key
type = "A"
@@ -146,15 +147,20 @@ resource "aws_route53_record" "public_a_record" {
records = [ each.value ]
}
resource "aws_route53_record" "main_a_record" {
- zone_id = aws_route53_zone.public_zone.id
+ count = var.private_only ? 0 : 1
+ zone_id = try(local.route53_public_zone.id, "")
name = ""
type = "A"
ttl = 300
records = [ local.public_ip["controller"] ]
}
resource "aws_route53_record" "public_cname_record" {
- zone_id = aws_route53_zone.public_zone.id
- for_each = {for i in local.cname_by_host: i.record => "${i.cname}.${local.arvados_dns_zone}" }
+ zone_id = try(local.route53_public_zone.id, "")
+ for_each = {
+ for i in local.cname_by_host: i.record =>
+ "${i.cname}.${local.arvados_dns_zone}"
+ if var.private_only == false
+ }
name = each.key
type = "CNAME"
ttl = 300
@@ -196,16 +202,19 @@ resource "aws_route53_record" "private_cname_record" {
# Route53's credentials for Let's Encrypt
#
resource "aws_iam_user" "letsencrypt" {
+ count = var.private_only ? 0 : 1
name = "${var.cluster_name}-letsencrypt"
path = "/"
}
resource "aws_iam_access_key" "letsencrypt" {
- user = aws_iam_user.letsencrypt.name
+ count = var.private_only ? 0 : 1
+ user = local.iam_user_letsencrypt.name
}
resource "aws_iam_user_policy" "letsencrypt_iam_policy" {
+ count = var.private_only ? 0 : 1
name = "${var.cluster_name}-letsencrypt_iam_policy"
- user = aws_iam_user.letsencrypt.name
+ user = local.iam_user_letsencrypt.name
policy = jsonencode({
"Version": "2012-10-17",
"Statement": [{
@@ -223,7 +232,7 @@ resource "aws_iam_user_policy" "letsencrypt_iam_policy" {
"route53:ChangeResourceRecordSets"
],
"Resource" : [
- "arn:aws:route53:::hostedzone/${aws_route53_zone.public_zone.id}"
+ "arn:aws:route53:::hostedzone/${local.route53_public_zone.id}"
]
}]
})
diff --git a/tools/salt-install/terraform/aws/vpc/outputs.tf b/tools/salt-install/terraform/aws/vpc/outputs.tf
index 09faa04a2..e1c0fe171 100644
--- a/tools/salt-install/terraform/aws/vpc/outputs.tf
+++ b/tools/salt-install/terraform/aws/vpc/outputs.tf
@@ -41,16 +41,28 @@ output "private_hosts" {
value = local.private_hosts
}
+output "user_facing_hosts" {
+ value = var.user_facing_hosts
+}
+
+output "internal_service_hosts" {
+ value = var.internal_service_hosts
+}
+
+output "private_only" {
+ value = var.private_only
+}
+
output "route53_dns_ns" {
- value = aws_route53_zone.public_zone.name_servers
+ value = try(local.route53_public_zone.name_servers, [])
}
output "letsencrypt_iam_access_key_id" {
- value = aws_iam_access_key.letsencrypt.id
+ value = try(local.iam_access_key_letsencrypt.id, "")
}
output "letsencrypt_iam_secret_access_key" {
- value = aws_iam_access_key.letsencrypt.secret
+ value = try(local.iam_access_key_letsencrypt.secret, "")
sensitive = true
}
diff --git a/tools/salt-install/terraform/aws/vpc/terraform.tfvars b/tools/salt-install/terraform/aws/vpc/terraform.tfvars
index cac62ed6f..296e3130c 100644
--- a/tools/salt-install/terraform/aws/vpc/terraform.tfvars
+++ b/tools/salt-install/terraform/aws/vpc/terraform.tfvars
@@ -5,3 +5,6 @@
region_name = "us-east-1"
# cluster_name = "xarv1"
# domain_name = "example.com"
+
+# Uncomment this to create an non-publicly accessible Arvados cluster
+# private_only = true
\ No newline at end of file
diff --git a/tools/salt-install/terraform/aws/vpc/variables.tf b/tools/salt-install/terraform/aws/vpc/variables.tf
index 4237c56c8..276f31433 100644
--- a/tools/salt-install/terraform/aws/vpc/variables.tf
+++ b/tools/salt-install/terraform/aws/vpc/variables.tf
@@ -19,4 +19,22 @@ variable "cluster_name" {
variable "domain_name" {
description = "The domain name under which your Arvados cluster will be hosted"
type = string
+}
+
+variable "private_only" {
+ description = "Don't create infrastructure reachable from the public Internet"
+ type = bool
+ default = false
+}
+
+variable "user_facing_hosts" {
+ description = "List of hostnames for nodes that hold user-accesible Arvados services"
+ type = list(string)
+ default = [ "controller", "workbench" ]
+}
+
+variable "internal_service_hosts" {
+ description = "List of hostnames for nodes that hold internal Arvados services"
+ type = list(string)
+ default = [ "keep0", "shell" ]
}
\ No newline at end of file
commit 57c9ad875306c39b1a6a065c2f5eff0f8c228716
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Sat May 6 15:14:43 2023 -0300
20482: Fixes S3 bucket creation for Keep blocks due to changes in AWS defaults.
ACLs are now not accepted on newly created S3 buckets, and by default they're
set as private, so there's no need for us to explicitly asking for that.
See: https://aws.amazon.com/about-aws/whats-new/2022/12/amazon-s3-automatically-enable-block-public-access-disable-access-control-lists-buckets-april-2023/
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/tools/salt-install/terraform/aws/data-storage/main.tf b/tools/salt-install/terraform/aws/data-storage/main.tf
index d4a3a7d21..6f7e233fd 100644
--- a/tools/salt-install/terraform/aws/data-storage/main.tf
+++ b/tools/salt-install/terraform/aws/data-storage/main.tf
@@ -24,20 +24,6 @@ resource "aws_s3_bucket" "keep_volume" {
bucket = "${local.cluster_name}-nyw5e-000000000000000-volume"
}
-resource "aws_s3_bucket_acl" "keep_volume_acl" {
- bucket = aws_s3_bucket.keep_volume.id
- acl = "private"
-}
-
-# Avoid direct public access to Keep blocks
-resource "aws_s3_bucket_public_access_block" "keep_volume_public_access" {
- bucket = aws_s3_bucket.keep_volume.id
-
- block_public_acls = true
- block_public_policy = true
- ignore_public_acls = true
-}
-
resource "aws_iam_role" "keepstore_iam_role" {
name = "${local.cluster_name}-keepstore-00-iam-role"
assume_role_policy = "${file("../assumerolepolicy.json")}"
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list