[arvados] updated: 2.1.0-2958-g664cc427e
git repository hosting
git at public.arvados.org
Fri Nov 25 14:02:23 UTC 2022
Summary of changes:
tools/salt-install/terraform/aws/.gitignore | 3 +
.../terraform/aws/assumerolepolicy.json | 13 ++
.../terraform/aws/data-storage/.terraform.lock.hcl | 42 +++++
.../terraform/aws/data-storage/data.tf | 10 ++
.../terraform/aws/data-storage/locals.tf | 8 +
.../terraform/aws/data-storage/main.tf | 69 +++++++
.../terraform/aws/data-storage/outputs.tf | 11 ++
.../terraform/aws/data-storage/terraform.tfvars | 7 +
.../terraform/aws/data-storage/variables.tf | 15 ++
.../terraform/aws/services/.terraform.lock.hcl | 22 +++
tools/salt-install/terraform/aws/services/data.tf | 31 ++++
.../salt-install/terraform/aws/services/locals.tf | 12 ++
tools/salt-install/terraform/aws/services/main.tf | 109 +++++++++++
.../salt-install/terraform/aws/services/outputs.tf | 48 +++++
.../terraform/aws/services/terraform.tfvars | 9 +
.../terraform/aws/services/user_data.sh | 19 ++
.../terraform/aws/services/variables.tf | 15 ++
.../terraform/aws/vpc/.terraform.lock.hcl | 22 +++
tools/salt-install/terraform/aws/vpc/locals.tf | 36 ++++
tools/salt-install/terraform/aws/vpc/main.tf | 200 +++++++++++++++++++++
tools/salt-install/terraform/aws/vpc/outputs.tf | 55 ++++++
.../terraform/aws/vpc/terraform.tfvars | 9 +-
tools/salt-install/terraform/aws/vpc/variables.tf | 22 +++
23 files changed, 781 insertions(+), 6 deletions(-)
create mode 100644 tools/salt-install/terraform/aws/.gitignore
create mode 100644 tools/salt-install/terraform/aws/assumerolepolicy.json
create mode 100644 tools/salt-install/terraform/aws/data-storage/.terraform.lock.hcl
create mode 100644 tools/salt-install/terraform/aws/data-storage/data.tf
create mode 100644 tools/salt-install/terraform/aws/data-storage/locals.tf
create mode 100644 tools/salt-install/terraform/aws/data-storage/main.tf
create mode 100644 tools/salt-install/terraform/aws/data-storage/outputs.tf
create mode 100644 tools/salt-install/terraform/aws/data-storage/terraform.tfvars
create mode 100644 tools/salt-install/terraform/aws/data-storage/variables.tf
create mode 100644 tools/salt-install/terraform/aws/services/.terraform.lock.hcl
create mode 100644 tools/salt-install/terraform/aws/services/data.tf
create mode 100644 tools/salt-install/terraform/aws/services/locals.tf
create mode 100644 tools/salt-install/terraform/aws/services/main.tf
create mode 100644 tools/salt-install/terraform/aws/services/outputs.tf
create mode 100644 tools/salt-install/terraform/aws/services/terraform.tfvars
create mode 100644 tools/salt-install/terraform/aws/services/user_data.sh
create mode 100644 tools/salt-install/terraform/aws/services/variables.tf
create mode 100644 tools/salt-install/terraform/aws/vpc/.terraform.lock.hcl
create mode 100644 tools/salt-install/terraform/aws/vpc/locals.tf
create mode 100644 tools/salt-install/terraform/aws/vpc/main.tf
create mode 100644 tools/salt-install/terraform/aws/vpc/outputs.tf
copy doc/Gemfile => tools/salt-install/terraform/aws/vpc/terraform.tfvars (50%)
create mode 100644 tools/salt-install/terraform/aws/vpc/variables.tf
via 664cc427ef0b3bdd896240f4e1c80b033b90982c (commit)
from f9321759166612825b540fe1c908e72713caf592 (commit)
Those revisions listed above that are new to this repository have
not appeared on any other notification email; so we list those
revisions in full, below.
commit 664cc427ef0b3bdd896240f4e1c80b033b90982c
Author: Lucas Di Pentima <lucas.dipentima at curii.com>
Date: Tue Nov 22 22:29:27 2022 -0300
19215: Adds initial version of terraform code for multi-host deploy in AWS.
It's separated in 3 sections: vpc, data-storage & services. This is to limit
the 'blast radius' of a potential error when applying changes, as recommended
in many places.
Each state should be applied in the order described above, and their outputs
feed the following states with important data.
The shared 'terraform.tfvars' file allows the operator to customize their
deployment.
Arvados-DCO-1.1-Signed-off-by: Lucas Di Pentima <lucas.dipentima at curii.com>
diff --git a/tools/salt-install/terraform/aws/.gitignore b/tools/salt-install/terraform/aws/.gitignore
new file mode 100644
index 000000000..5454e9c4e
--- /dev/null
+++ b/tools/salt-install/terraform/aws/.gitignore
@@ -0,0 +1,3 @@
+**/.terraform
+**/terraform.tfstate*
+**/.infracost
diff --git a/tools/salt-install/terraform/aws/assumerolepolicy.json b/tools/salt-install/terraform/aws/assumerolepolicy.json
new file mode 100644
index 000000000..8b9ed694f
--- /dev/null
+++ b/tools/salt-install/terraform/aws/assumerolepolicy.json
@@ -0,0 +1,13 @@
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
\ No newline at end of file
diff --git a/tools/salt-install/terraform/aws/data-storage/.terraform.lock.hcl b/tools/salt-install/terraform/aws/data-storage/.terraform.lock.hcl
new file mode 100644
index 000000000..473eebc2b
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/.terraform.lock.hcl
@@ -0,0 +1,42 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "4.38.0"
+ hashes = [
+ "h1:LympybKZJE3L0H12nMmDnFH1iexD9S2GqZbDMo4fuPI=",
+ "h1:bhDPZioOF9Uz9mavezCHfYbD5YJ3fEPsixLpcWgV/kU=",
+ "zh:0ae61458acf7acecf47f7a02e08da1f7adeee9532e053c0d80432f16197e4799",
+ "zh:1ece9bcef41ffc75e0955419d7f8b1708ab7ffe4518bc9a2afe3bc5c79a9e79b",
+ "zh:302065a7c3ae798345b92a465b650b025d9c4e9abc3e78421ecc69a17b8c3d6a",
+ "zh:52d61f6a3ed6726b821a78f1fb78df818cf24a4d2378cc16afded297b37d4b7b",
+ "zh:6c365ed0cae031acdbcca04560997589a94629269cb456d468cbe51a3a020386",
+ "zh:70987a51d782f3458f124efea320157a48453864c420421051c56d41e463a948",
+ "zh:8b5a5f30240c67e596a89ccd76aa81133e6ae253c8a06a932b8901ef2b4a7486",
+ "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+ "zh:d672167515ece7c2db4663faf180dfb6cfc6dbf5e149f868d05c39bb54b9ca03",
+ "zh:df1bc9926674b2e1246c9ebffd8bf8c4e380f50910a7f0b3ded957e8768ae27a",
+ "zh:e304b6e2bd66e7992326aa0446152547eb97e8f77d00bc1a9096022ac37e5d71",
+ "zh:f033690f11446af1383ad74149f429fae19e2784af5e151a22f46965dff21b29",
+ ]
+}
+
+provider "registry.terraform.io/hashicorp/external" {
+ version = "2.2.2"
+ hashes = [
+ "h1:VUkgcWvCliS0HO4kt7oEQhFD2gcx/59XpwMqxfCU1kE=",
+ "h1:e7RpnZ2PbJEEPnfsg7V0FNwbfSk0/Z3FdrLsXINBmDY=",
+ "zh:0b84ab0af2e28606e9c0c1289343949339221c3ab126616b831ddb5aaef5f5ca",
+ "zh:10cf5c9b9524ca2e4302bf02368dc6aac29fb50aeaa6f7758cce9aa36ae87a28",
+ "zh:56a016ee871c8501acb3f2ee3b51592ad7c3871a1757b098838349b17762ba6b",
+ "zh:719d6ef39c50e4cffc67aa67d74d195adaf42afcf62beab132dafdb500347d39",
+ "zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
+ "zh:7fbfc4d37435ac2f717b0316f872f558f608596b389b895fcb549f118462d327",
+ "zh:8ac71408204db606ce63fe8f9aeaf1ddc7751d57d586ec421e62d440c402e955",
+ "zh:a4cacdb06f114454b6ed0033add28006afa3f65a0ea7a43befe45fc82e6809fb",
+ "zh:bb5ce3132b52ae32b6cc005bc9f7627b95259b9ffe556de4dad60d47d47f21f0",
+ "zh:bb60d2976f125ffd232a7ccb4b3f81e7109578b23c9c6179f13a11d125dca82a",
+ "zh:f9540ecd2e056d6e71b9ea5f5a5cf8f63dd5c25394b9db831083a9d4ea99b372",
+ "zh:ffd998b55b8a64d4335a090b6956b4bf8855b290f7554dd38db3302de9c41809",
+ ]
+}
diff --git a/tools/salt-install/terraform/aws/data-storage/data.tf b/tools/salt-install/terraform/aws/data-storage/data.tf
new file mode 100644
index 000000000..17fd9d351
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/data.tf
@@ -0,0 +1,10 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+data "terraform_remote_state" "vpc" {
+ backend = "local"
+ config = {
+ path = "../vpc/terraform.tfstate"
+ }
+}
diff --git a/tools/salt-install/terraform/aws/data-storage/locals.tf b/tools/salt-install/terraform/aws/data-storage/locals.tf
new file mode 100644
index 000000000..5b9f68969
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/locals.tf
@@ -0,0 +1,8 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+locals {
+ region_name = data.terraform_remote_state.vpc.outputs.region_name
+ cluster_name = data.terraform_remote_state.vpc.outputs.cluster_name
+}
diff --git a/tools/salt-install/terraform/aws/data-storage/main.tf b/tools/salt-install/terraform/aws/data-storage/main.tf
new file mode 100644
index 000000000..d4a3a7d21
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/main.tf
@@ -0,0 +1,69 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ }
+ }
+}
+
+provider "aws" {
+ region = local.region_name
+ default_tags {
+ tags = {
+ Arvados = local.cluster_name
+ }
+ }
+}
+
+# S3 bucket and access resources for Keep blocks
+resource "aws_s3_bucket" "keep_volume" {
+ bucket = "${local.cluster_name}-nyw5e-000000000000000-volume"
+}
+
+resource "aws_s3_bucket_acl" "keep_volume_acl" {
+ bucket = aws_s3_bucket.keep_volume.id
+ acl = "private"
+}
+
+# Avoid direct public access to Keep blocks
+resource "aws_s3_bucket_public_access_block" "keep_volume_public_access" {
+ bucket = aws_s3_bucket.keep_volume.id
+
+ block_public_acls = true
+ block_public_policy = true
+ ignore_public_acls = true
+}
+
+resource "aws_iam_role" "keepstore_iam_role" {
+ name = "${local.cluster_name}-keepstore-00-iam-role"
+ assume_role_policy = "${file("../assumerolepolicy.json")}"
+}
+
+resource "aws_iam_policy" "s3_full_access" {
+ name = "${local.cluster_name}_s3_full_access"
+ policy = jsonencode({
+ Version: "2012-10-17",
+ Id: "arvados-keepstore policy",
+ Statement: [{
+ Effect: "Allow",
+ Action: [
+ "s3:*",
+ ],
+ Resource: [
+ "arn:aws:s3:::${local.cluster_name}-nyw5e-000000000000000-volume",
+ "arn:aws:s3:::${local.cluster_name}-nyw5e-000000000000000-volume/*"
+ ]
+ }]
+ })
+}
+
+resource "aws_iam_policy_attachment" "s3_full_access_policy_attachment" {
+ name = "${local.cluster_name}_s3_full_access_attachment"
+ roles = [ aws_iam_role.keepstore_iam_role.name ]
+ policy_arn = aws_iam_policy.s3_full_access.arn
+}
+
diff --git a/tools/salt-install/terraform/aws/data-storage/outputs.tf b/tools/salt-install/terraform/aws/data-storage/outputs.tf
new file mode 100644
index 000000000..6298f926a
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/outputs.tf
@@ -0,0 +1,11 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+output "keepstore_iam_role_name" {
+ value = aws_iam_role.keepstore_iam_role.name
+}
+
+output "use_external_db" {
+ value = var.use_external_db
+}
\ No newline at end of file
diff --git a/tools/salt-install/terraform/aws/data-storage/terraform.tfvars b/tools/salt-install/terraform/aws/data-storage/terraform.tfvars
new file mode 100644
index 000000000..6515339fb
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/terraform.tfvars
@@ -0,0 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+# Set to true if the database server won't be running in any service instance.
+# Default: false
+# use_external_db = true
diff --git a/tools/salt-install/terraform/aws/data-storage/variables.tf b/tools/salt-install/terraform/aws/data-storage/variables.tf
new file mode 100644
index 000000000..45e1531aa
--- /dev/null
+++ b/tools/salt-install/terraform/aws/data-storage/variables.tf
@@ -0,0 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+variable "use_external_db" {
+ description = "Enable this if the database service won't be installed on these instances"
+ type = bool
+ default = false
+}
+
+variable "keep_cluster_data" {
+ description = "Avoids state (database & keep blocks) to be destroyed. Needed for production clusters"
+ type = bool
+ default = false
+}
diff --git a/tools/salt-install/terraform/aws/services/.terraform.lock.hcl b/tools/salt-install/terraform/aws/services/.terraform.lock.hcl
new file mode 100644
index 000000000..63116a5ea
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/.terraform.lock.hcl
@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "4.38.0"
+ hashes = [
+ "h1:LympybKZJE3L0H12nMmDnFH1iexD9S2GqZbDMo4fuPI=",
+ "h1:bhDPZioOF9Uz9mavezCHfYbD5YJ3fEPsixLpcWgV/kU=",
+ "zh:0ae61458acf7acecf47f7a02e08da1f7adeee9532e053c0d80432f16197e4799",
+ "zh:1ece9bcef41ffc75e0955419d7f8b1708ab7ffe4518bc9a2afe3bc5c79a9e79b",
+ "zh:302065a7c3ae798345b92a465b650b025d9c4e9abc3e78421ecc69a17b8c3d6a",
+ "zh:52d61f6a3ed6726b821a78f1fb78df818cf24a4d2378cc16afded297b37d4b7b",
+ "zh:6c365ed0cae031acdbcca04560997589a94629269cb456d468cbe51a3a020386",
+ "zh:70987a51d782f3458f124efea320157a48453864c420421051c56d41e463a948",
+ "zh:8b5a5f30240c67e596a89ccd76aa81133e6ae253c8a06a932b8901ef2b4a7486",
+ "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+ "zh:d672167515ece7c2db4663faf180dfb6cfc6dbf5e149f868d05c39bb54b9ca03",
+ "zh:df1bc9926674b2e1246c9ebffd8bf8c4e380f50910a7f0b3ded957e8768ae27a",
+ "zh:e304b6e2bd66e7992326aa0446152547eb97e8f77d00bc1a9096022ac37e5d71",
+ "zh:f033690f11446af1383ad74149f429fae19e2784af5e151a22f46965dff21b29",
+ ]
+}
diff --git a/tools/salt-install/terraform/aws/services/data.tf b/tools/salt-install/terraform/aws/services/data.tf
new file mode 100644
index 000000000..3587f2526
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/data.tf
@@ -0,0 +1,31 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+data "terraform_remote_state" "vpc" {
+ backend = "local"
+ config = {
+ path = "../vpc/terraform.tfstate"
+ }
+}
+
+data "terraform_remote_state" "data-storage" {
+ backend = "local"
+ config = {
+ path = "../data-storage/terraform.tfstate"
+ }
+}
+
+# https://wiki.debian.org/Cloud/AmazonEC2Image/Bullseye
+data "aws_ami" "debian-11" {
+ most_recent = true
+ owners = ["136693071363"]
+ filter {
+ name = "name"
+ values = ["debian-11-amd64-*"]
+ }
+ filter {
+ name = "virtualization-type"
+ values = ["hvm"]
+ }
+}
\ No newline at end of file
diff --git a/tools/salt-install/terraform/aws/services/locals.tf b/tools/salt-install/terraform/aws/services/locals.tf
new file mode 100644
index 000000000..80dc33784
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/locals.tf
@@ -0,0 +1,12 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+locals {
+ region_name = data.terraform_remote_state.vpc.outputs.region_name
+ cluster_name = data.terraform_remote_state.vpc.outputs.cluster_name
+ use_external_db = data.terraform_remote_state.data-storage.outputs.use_external_db
+ public_ip = data.terraform_remote_state.vpc.outputs.public_ip
+ private_ip = data.terraform_remote_state.vpc.outputs.private_ip
+ hostnames = [ for hostname, eip_id in data.terraform_remote_state.vpc.outputs.eip_id: hostname ]
+}
diff --git a/tools/salt-install/terraform/aws/services/main.tf b/tools/salt-install/terraform/aws/services/main.tf
new file mode 100644
index 000000000..911311f83
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/main.tf
@@ -0,0 +1,109 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ }
+ }
+}
+
+provider "aws" {
+ region = local.region_name
+ default_tags {
+ tags = {
+ Arvados = local.cluster_name
+ }
+ }
+}
+
+locals {
+ pubkey_path = pathexpand(var.pubkey_path)
+ pubkey_name = "arvados-deployer-key"
+}
+resource "aws_key_pair" "deployer" {
+ key_name = local.pubkey_name
+ public_key = file(local.pubkey_path)
+}
+
+resource "aws_iam_instance_profile" "keepstore_instance_profile" {
+ name = "${local.cluster_name}-keepstore-00-iam-role"
+ role = data.terraform_remote_state.data-storage.outputs.keepstore_iam_role_name
+}
+
+resource "aws_iam_instance_profile" "dispatcher_instance_profile" {
+ name = "${local.cluster_name}_dispatcher_instance_profile"
+ role = aws_iam_role.cloud_dispatcher_iam_role.name
+}
+
+resource "aws_instance" "arvados_service" {
+ for_each = toset(local.hostnames)
+ ami = data.aws_ami.debian-11.image_id
+ instance_type = var.default_instance_type
+ key_name = local.pubkey_name
+ user_data = templatefile("user_data.sh", {
+ "hostname": each.value
+ })
+ private_ip = local.private_ip[each.value]
+ subnet_id = data.terraform_remote_state.vpc.outputs.arvados_subnet_id
+ vpc_security_group_ids = [ data.terraform_remote_state.vpc.outputs.arvados_sg_id ]
+ # This should be done in a more readable way
+ iam_instance_profile = each.value == "controller" ? aws_iam_instance_profile.dispatcher_instance_profile.name : length(regexall("^keep[0-9]+", each.value)) > 0 ? aws_iam_instance_profile.keepstore_instance_profile.name : ""
+ tags = {
+ Name = "arvados_service_${each.value}"
+ }
+ root_block_device {
+ volume_type = "gp3"
+ volume_size = (each.value == "controller" && !local.use_external_db) ? 70 : 20
+ }
+
+ lifecycle {
+ ignore_changes = [
+ # Avoids recreating the instance when the latest AMI changes.
+ # Use 'terraform taint' or 'terraform apply -replace' to force
+ # an AMI change.
+ ami,
+ ]
+ }
+}
+
+resource "aws_iam_policy" "cloud_dispatcher_ec2_access" {
+ name = "${local.cluster_name}_cloud_dispatcher_ec2_access"
+ policy = jsonencode({
+ Version: "2012-10-17",
+ Id: "arvados-dispatch-cloud policy",
+ Statement: [{
+ Effect: "Allow",
+ Action: [
+ "iam:PassRole",
+ "ec2:DescribeKeyPairs",
+ "ec2:ImportKeyPair",
+ "ec2:RunInstances",
+ "ec2:DescribeInstances",
+ "ec2:CreateTags",
+ "ec2:TerminateInstances"
+ ],
+ Resource: "*"
+ }]
+ })
+}
+
+resource "aws_iam_role" "cloud_dispatcher_iam_role" {
+ name = "${local.cluster_name}-dispatcher-00-iam-role"
+ assume_role_policy = "${file("../assumerolepolicy.json")}"
+}
+
+resource "aws_iam_policy_attachment" "cloud_dispatcher_ec2_access_attachment" {
+ name = "${local.cluster_name}_cloud_dispatcher_ec2_access_attachment"
+ roles = [ aws_iam_role.cloud_dispatcher_iam_role.name ]
+ policy_arn = aws_iam_policy.cloud_dispatcher_ec2_access.arn
+}
+
+resource "aws_eip_association" "eip_assoc" {
+ for_each = toset(local.hostnames)
+ instance_id = aws_instance.arvados_service[each.value].id
+ allocation_id = data.terraform_remote_state.vpc.outputs.eip_id[each.value]
+ # private_ip_address = local.private_ip[each.value]
+}
diff --git a/tools/salt-install/terraform/aws/services/outputs.tf b/tools/salt-install/terraform/aws/services/outputs.tf
new file mode 100644
index 000000000..96af4054d
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/outputs.tf
@@ -0,0 +1,48 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+output "vpc_id" {
+ value = data.terraform_remote_state.vpc.outputs.arvados_vpc_id
+}
+
+output "vpc_cidr" {
+ value = data.terraform_remote_state.vpc.outputs.arvados_vpc_cidr
+}
+
+output "subnet_id" {
+ value = data.terraform_remote_state.vpc.outputs.arvados_subnet_id
+}
+
+output "arvados_sg_id" {
+ value = data.terraform_remote_state.vpc.outputs.arvados_sg_id
+}
+
+output "public_ip" {
+ value = local.public_ip
+}
+
+output "private_ip" {
+ value = local.private_ip
+}
+
+output "route53_dns_ns" {
+ value = data.terraform_remote_state.vpc.outputs.route53_dns_ns
+}
+
+output "letsencrypt_iam_access_key_id" {
+ value = data.terraform_remote_state.vpc.outputs.letsencrypt_iam_access_key_id
+}
+
+output "letsencrypt_iam_secret_access_key" {
+ value = data.terraform_remote_state.vpc.outputs.letsencrypt_iam_secret_access_key
+ sensitive = true
+}
+
+output "cluster_name" {
+ value = data.terraform_remote_state.vpc.outputs.cluster_name
+}
+
+output "domain_name" {
+ value = data.terraform_remote_state.vpc.outputs.domain_name
+}
diff --git a/tools/salt-install/terraform/aws/services/terraform.tfvars b/tools/salt-install/terraform/aws/services/terraform.tfvars
new file mode 100644
index 000000000..374ecbe08
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/terraform.tfvars
@@ -0,0 +1,9 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+# Set to a specific SSH public key path. Default: ~/.ssh/id_rsa.pub
+# pubkey_path = /path/to/pub.key
+
+# Set the instance type for your hosts. Default: m5a.large
+# default_instance_type = "t2.micro"
diff --git a/tools/salt-install/terraform/aws/services/user_data.sh b/tools/salt-install/terraform/aws/services/user_data.sh
new file mode 100644
index 000000000..6c5b574dd
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/user_data.sh
@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+hostname ${hostname}
+echo ${hostname} > /etc/hostname
+
+# Retry just in case internet access is not yet ready
+while true; do
+ apt-get -o Acquire::ForceIPv4=true update
+ ERR=$?
+ if [ "$${ERR}" = "0" ]; then
+ break
+ fi
+done
+
+apt-get -o Acquire::ForceIPv4=true install -y git curl
diff --git a/tools/salt-install/terraform/aws/services/variables.tf b/tools/salt-install/terraform/aws/services/variables.tf
new file mode 100644
index 000000000..89b1886c1
--- /dev/null
+++ b/tools/salt-install/terraform/aws/services/variables.tf
@@ -0,0 +1,15 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+variable "default_instance_type" {
+ description = "The default EC2 instance type to use on the nodes"
+ type = string
+ default = "m5a.large"
+}
+
+variable "pubkey_path" {
+ description = "Path to the file containing the public SSH key"
+ type = string
+ default = "~/.ssh/id_rsa.pub"
+}
diff --git a/tools/salt-install/terraform/aws/vpc/.terraform.lock.hcl b/tools/salt-install/terraform/aws/vpc/.terraform.lock.hcl
new file mode 100644
index 000000000..63116a5ea
--- /dev/null
+++ b/tools/salt-install/terraform/aws/vpc/.terraform.lock.hcl
@@ -0,0 +1,22 @@
+# This file is maintained automatically by "terraform init".
+# Manual edits may be lost in future updates.
+
+provider "registry.terraform.io/hashicorp/aws" {
+ version = "4.38.0"
+ hashes = [
+ "h1:LympybKZJE3L0H12nMmDnFH1iexD9S2GqZbDMo4fuPI=",
+ "h1:bhDPZioOF9Uz9mavezCHfYbD5YJ3fEPsixLpcWgV/kU=",
+ "zh:0ae61458acf7acecf47f7a02e08da1f7adeee9532e053c0d80432f16197e4799",
+ "zh:1ece9bcef41ffc75e0955419d7f8b1708ab7ffe4518bc9a2afe3bc5c79a9e79b",
+ "zh:302065a7c3ae798345b92a465b650b025d9c4e9abc3e78421ecc69a17b8c3d6a",
+ "zh:52d61f6a3ed6726b821a78f1fb78df818cf24a4d2378cc16afded297b37d4b7b",
+ "zh:6c365ed0cae031acdbcca04560997589a94629269cb456d468cbe51a3a020386",
+ "zh:70987a51d782f3458f124efea320157a48453864c420421051c56d41e463a948",
+ "zh:8b5a5f30240c67e596a89ccd76aa81133e6ae253c8a06a932b8901ef2b4a7486",
+ "zh:9b12af85486a96aedd8d7984b0ff811a4b42e3d88dad1a3fb4c0b580d04fa425",
+ "zh:d672167515ece7c2db4663faf180dfb6cfc6dbf5e149f868d05c39bb54b9ca03",
+ "zh:df1bc9926674b2e1246c9ebffd8bf8c4e380f50910a7f0b3ded957e8768ae27a",
+ "zh:e304b6e2bd66e7992326aa0446152547eb97e8f77d00bc1a9096022ac37e5d71",
+ "zh:f033690f11446af1383ad74149f429fae19e2784af5e151a22f46965dff21b29",
+ ]
+}
diff --git a/tools/salt-install/terraform/aws/vpc/locals.tf b/tools/salt-install/terraform/aws/vpc/locals.tf
new file mode 100644
index 000000000..e8864c522
--- /dev/null
+++ b/tools/salt-install/terraform/aws/vpc/locals.tf
@@ -0,0 +1,36 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+locals {
+ allowed_ports = {
+ http: "80",
+ https: "443",
+ ssh: "22",
+ }
+ hostnames = [ "controller", "workbench", "keep0", "keep1", "keepproxy", "shell" ]
+ arvados_dns_zone = "${var.cluster_name}.${var.domain_name}"
+ public_ip = { for k, v in aws_eip.arvados_eip: k => v.public_ip }
+ private_ip = {
+ "controller": "10.1.1.1",
+ "workbench": "10.1.1.5",
+ "keepproxy": "10.1.1.2",
+ "shell": "10.1.1.7",
+ "keep0": "10.1.1.3",
+ "keep1": "10.1.1.4"
+ }
+ aliases = {
+ controller: ["ws"]
+ workbench: ["workbench2", "webshell"]
+ keepproxy: ["keep", "download", "*.collections"]
+ }
+ cname_by_host = flatten([
+ for host, aliases in local.aliases : [
+ for alias in aliases : {
+ record = alias
+ cname = host
+ }
+ ]
+ ])
+}
+
diff --git a/tools/salt-install/terraform/aws/vpc/main.tf b/tools/salt-install/terraform/aws/vpc/main.tf
new file mode 100644
index 000000000..4581d5b6f
--- /dev/null
+++ b/tools/salt-install/terraform/aws/vpc/main.tf
@@ -0,0 +1,200 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ }
+ }
+}
+
+provider "aws" {
+ region = var.region_name
+ default_tags {
+ tags = {
+ Arvados = var.cluster_name
+ }
+ }
+}
+
+resource "aws_vpc" "arvados_vpc" {
+ cidr_block = "10.1.0.0/16"
+ enable_dns_hostnames = true
+ enable_dns_support = true
+}
+resource "aws_subnet" "arvados_subnet" {
+ vpc_id = aws_vpc.arvados_vpc.id
+ availability_zone = "${var.region_name}a"
+ cidr_block = aws_vpc.arvados_vpc.cidr_block
+}
+
+#
+# VPC S3 access
+#
+resource "aws_vpc_endpoint" "s3" {
+ vpc_id = aws_vpc.arvados_vpc.id
+ service_name = "com.amazonaws.${var.region_name}.s3"
+}
+resource "aws_vpc_endpoint_route_table_association" "s3_route" {
+ vpc_endpoint_id = aws_vpc_endpoint.s3.id
+ route_table_id = aws_route_table.arvados_rt.id
+}
+
+#
+# VPC Internet access
+#
+resource "aws_internet_gateway" "arvados_gw" {
+ vpc_id = aws_vpc.arvados_vpc.id
+}
+resource "aws_eip" "arvados_eip" {
+ for_each = toset(local.hostnames)
+ depends_on = [
+ aws_internet_gateway.arvados_gw
+ ]
+}
+resource "aws_route_table" "arvados_rt" {
+ vpc_id = aws_vpc.arvados_vpc.id
+ route {
+ cidr_block = "0.0.0.0/0"
+ gateway_id = aws_internet_gateway.arvados_gw.id
+ }
+}
+resource "aws_route_table_association" "arvados_subnet_assoc" {
+ subnet_id = aws_subnet.arvados_subnet.id
+ route_table_id = aws_route_table.arvados_rt.id
+}
+resource "aws_security_group" "arvados_sg" {
+ name = "arvados_sg"
+ vpc_id = aws_vpc.arvados_vpc.id
+
+ dynamic "ingress" {
+ for_each = local.allowed_ports
+ content {
+ description = "Ingress rule for ${ingress.key}"
+ from_port = "${ingress.value}"
+ to_port = "${ingress.value}"
+ protocol = "tcp"
+ cidr_blocks = ["0.0.0.0/0"]
+ ipv6_cidr_blocks = ["::/0"]
+ }
+ }
+ # Allows communication between nodes in the VPC
+ ingress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = [ aws_vpc.arvados_vpc.cidr_block ]
+ }
+ # Even though AWS auto-creates an "allow all" egress rule,
+ # Terraform deletes it, so we add it explicitly.
+ egress {
+ from_port = 0
+ to_port = 0
+ protocol = "-1"
+ cidr_blocks = ["0.0.0.0/0"]
+ ipv6_cidr_blocks = ["::/0"]
+ }
+}
+
+#
+# Route53 split-horizon DNS zones
+#
+
+# PUBLIC DNS
+resource "aws_route53_zone" "public_zone" {
+ name = local.arvados_dns_zone
+}
+resource "aws_route53_record" "public_a_record" {
+ zone_id = aws_route53_zone.public_zone.id
+ for_each = local.public_ip
+ name = each.key
+ type = "A"
+ ttl = 300
+ records = [ each.value ]
+}
+resource "aws_route53_record" "main_a_record" {
+ zone_id = aws_route53_zone.public_zone.id
+ name = ""
+ type = "A"
+ ttl = 300
+ records = [ local.public_ip["controller"] ]
+}
+resource "aws_route53_record" "public_cname_record" {
+ zone_id = aws_route53_zone.public_zone.id
+ for_each = {for i in local.cname_by_host: i.record => "${i.cname}.${local.arvados_dns_zone}" }
+ name = each.key
+ type = "CNAME"
+ ttl = 300
+ records = [ each.value ]
+}
+
+# PRIVATE DNS
+resource "aws_route53_zone" "private_zone" {
+ name = local.arvados_dns_zone
+ vpc {
+ vpc_id = aws_vpc.arvados_vpc.id
+ }
+}
+resource "aws_route53_record" "private_a_record" {
+ zone_id = aws_route53_zone.private_zone.id
+ for_each = local.private_ip
+ name = each.key
+ type = "A"
+ ttl = 300
+ records = [ each.value ]
+}
+resource "aws_route53_record" "private_main_a_record" {
+ zone_id = aws_route53_zone.private_zone.id
+ name = ""
+ type = "A"
+ ttl = 300
+ records = [ local.private_ip["controller"] ]
+}
+resource "aws_route53_record" "private_cname_record" {
+ zone_id = aws_route53_zone.private_zone.id
+ for_each = {for i in local.cname_by_host: i.record => "${i.cname}.${local.arvados_dns_zone}" }
+ name = each.key
+ type = "CNAME"
+ ttl = 300
+ records = [ each.value ]
+}
+
+#
+# Route53's credentials for Let's Encrypt
+#
+resource "aws_iam_user" "letsencrypt" {
+ name = "${var.cluster_name}-letsencrypt"
+ path = "/"
+}
+
+resource "aws_iam_access_key" "letsencrypt" {
+ user = aws_iam_user.letsencrypt.name
+}
+resource "aws_iam_user_policy" "letsencrypt_iam_policy" {
+ name = "${var.cluster_name}-letsencrypt_iam_policy"
+ user = aws_iam_user.letsencrypt.name
+ policy = jsonencode({
+ "Version": "2012-10-17",
+ "Statement": [{
+ "Effect": "Allow",
+ "Action": [
+ "route53:ListHostedZones",
+ "route53:GetChange"
+ ],
+ "Resource": [
+ "*"
+ ]
+ },{
+ "Effect" : "Allow",
+ "Action" : [
+ "route53:ChangeResourceRecordSets"
+ ],
+ "Resource" : [
+ "arn:aws:route53:::hostedzone/${aws_route53_zone.public_zone.id}"
+ ]
+ }]
+ })
+}
+
diff --git a/tools/salt-install/terraform/aws/vpc/outputs.tf b/tools/salt-install/terraform/aws/vpc/outputs.tf
new file mode 100644
index 000000000..4ae90a5ef
--- /dev/null
+++ b/tools/salt-install/terraform/aws/vpc/outputs.tf
@@ -0,0 +1,55 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+output "arvados_vpc_id" {
+ value = aws_vpc.arvados_vpc.id
+}
+output "arvados_vpc_cidr" {
+ value = aws_vpc.arvados_vpc.cidr_block
+}
+
+output "arvados_subnet_id" {
+ value = aws_subnet.arvados_subnet.id
+}
+
+output "arvados_sg_id" {
+ value = aws_security_group.arvados_sg.id
+}
+
+output "eip_id" {
+ value = { for k, v in aws_eip.arvados_eip: k => v.id }
+}
+
+output "public_ip" {
+ value = local.public_ip
+}
+
+output "private_ip" {
+ value = local.private_ip
+}
+
+output "route53_dns_ns" {
+ value = aws_route53_zone.public_zone.name_servers
+}
+
+output "letsencrypt_iam_access_key_id" {
+ value = aws_iam_access_key.letsencrypt.id
+}
+
+output "letsencrypt_iam_secret_access_key" {
+ value = aws_iam_access_key.letsencrypt.secret
+ sensitive = true
+}
+
+output "region_name" {
+ value = var.region_name
+}
+
+output "cluster_name" {
+ value = var.cluster_name
+}
+
+output "domain_name" {
+ value = var.domain_name
+}
diff --git a/tools/salt-install/terraform/aws/vpc/terraform.tfvars b/tools/salt-install/terraform/aws/vpc/terraform.tfvars
new file mode 100644
index 000000000..cac62ed6f
--- /dev/null
+++ b/tools/salt-install/terraform/aws/vpc/terraform.tfvars
@@ -0,0 +1,7 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+region_name = "us-east-1"
+# cluster_name = "xarv1"
+# domain_name = "example.com"
diff --git a/tools/salt-install/terraform/aws/vpc/variables.tf b/tools/salt-install/terraform/aws/vpc/variables.tf
new file mode 100644
index 000000000..4237c56c8
--- /dev/null
+++ b/tools/salt-install/terraform/aws/vpc/variables.tf
@@ -0,0 +1,22 @@
+# Copyright (C) The Arvados Authors. All rights reserved.
+#
+# SPDX-License-Identifier: CC-BY-SA-3.0
+
+variable "region_name" {
+ description = "Name of the AWS Region where to install Arvados"
+ type = string
+}
+
+variable "cluster_name" {
+ description = "A 5-char alphanum identifier for your Arvados cluster"
+ type = string
+ validation {
+ condition = length(var.cluster_name) == 5
+ error_message = "cluster_name should be 5 chars long."
+ }
+}
+
+variable "domain_name" {
+ description = "The domain name under which your Arvados cluster will be hosted"
+ type = string
+}
\ No newline at end of file
-----------------------------------------------------------------------
hooks/post-receive
--
More information about the arvados-commits
mailing list