From 1e27974ecc48d8611b87af1b9cd51e29afa8c792 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 17:15:59 +0100 Subject: test(fx): fix prepare_layer - broken --- terraform/lambda.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 67fd6eb..27e6266 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -89,14 +89,13 @@ locals { } resource "null_resource" "prepare_layer" { - triggers = { - requirements_hash = filesha1(local.requirements) - } provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 11:21:58 +0100 Subject: layers block update, function resources to inlcude attributes: layers, correct handler and source_code_hash --- terraform/lambda.tf | 70 +++++++++++++++++++++++++++++------------------------ terraform/s3.tf | 5 ++++ 2 files changed, 44 insertions(+), 31 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 27e6266..e33bc79 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -12,12 +12,14 @@ resource "aws_s3_object" "extract_lambda_code" { } resource "aws_lambda_function" "extract_lambda" { - function_name = var.extract_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.extract_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "extract_lambda.extract" - runtime = "python3.11" + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -40,12 +42,14 @@ resource "aws_s3_object" "transform_lambda_code" { } resource "aws_lambda_function" "transform_lambda" { - function_name = var.transform_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.transform_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "transform_lambda.transform" - runtime = "python3.11" + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -68,12 +72,14 @@ resource "aws_s3_object" "load_lambda_code" { } resource "aws_lambda_function" "load_lambda" { - function_name = var.load_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.load_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "load_lambda.load" - runtime = "python3.11" + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -82,10 +88,12 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } +# Lambda Layer Specification locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "lambda_layer" + requirements = "requirements.txt" + layer_zip = "layer.zip" + layer_name = "lambda_layer_dev" } resource "null_resource" "prepare_layer" { @@ -95,23 +103,23 @@ resource "null_resource" "prepare_layer" { rm -rf python mkdir python pip3 install -r ${local.requirements} -t python/ - zip -r ${local.layer_zip} python/ - EOT - } + zip -r ${local.layer_zip} python + EOT + } #removed / at the end of python in line 99 } -resource "aws_s3_object" "layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "layer.zip" +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "lambda_layer/${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = "lambda_layer" + layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.layer_zip.key + s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true - depends_on = [aws_s3_object.layer_zip] + depends_on = [aws_s3_object.lambda_layer_zip] } diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..b3a863c 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,3 +12,8 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } + +### LAMBDA LAYER BUCKET +resource "aws_s3_bucket" "lambda_layer_bucket" { + bucket_prefix = "lambda-layer-dev-" +} \ No newline at end of file -- cgit v1.2.3 From 57d1e1ee5a13269f1bef6c3b754cb8374a657202 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 14:55:39 +0100 Subject: style: remove redundant comment --- terraform/lambda.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index e33bc79..714ffa5 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -99,13 +99,9 @@ locals { resource "null_resource" "prepare_layer" { provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 15:02:39 +0100 Subject: infra(tf): modify variables & remove past zip creation --- terraform/lambda.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 714ffa5..986170f 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,17 +90,16 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "lambda_layer" + layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" layer_name = "lambda_layer_dev" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { provisioner "local-exec" { - command = < Date: Mon, 19 Aug 2024 15:06:35 +0100 Subject: infra(tf): remove lambda layer dev reference --- terraform/s3.tf | 5 ----- 1 file changed, 5 deletions(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index b3a863c..d5cdee3 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,8 +12,3 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } - -### LAMBDA LAYER BUCKET -resource "aws_s3_bucket" "lambda_layer_bucket" { - bucket_prefix = "lambda-layer-dev-" -} \ No newline at end of file -- cgit v1.2.3 From 284a52df866c34d925b85ccd4f06d6141e67ce70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:12:56 +0100 Subject: fix(tf): correct layer.zip output path --- terraform/lambda.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 986170f..8a4207d 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -93,7 +93,7 @@ locals { layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" - layer_name = "lambda_layer_dev" + layer_name = "lambda_layer" script_dir = "../scripts" } @@ -105,7 +105,7 @@ resource "null_resource" "prepare_layer" { resource "aws_s3_object" "lambda_layer_zip" { bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "lambda_layer/${local.layer_name}/${local.layer_zip}" + key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_bucket = aws_s3_bucket.lambda_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From cbf1d083dc0bf4d78da83cb169da49731f8ace65 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:18:22 +0100 Subject: fix(tf): correct s3_bucket value for lambda_layer --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 8a4207d..bf96747 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From 4b3b80a2f2177456ed6c2857a7ae0987d7304360 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:40:01 +0100 Subject: chore(tf): remove unused requirements variable --- terraform/lambda.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bf96747..72aae04 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,11 +90,10 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "../" - requirements = "requirements.txt" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { -- cgit v1.2.3 From 8b4e78b781617f68554efebcda75d982a382f650 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 16:31:50 +0100 Subject: fix(tf): fix permissions for bucket/object access --- terraform/iam.tf | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 0e5fa6d..7585ff8 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -28,17 +28,19 @@ resource "aws_iam_role" "multi_service_role" { ######################################################################## # S3 SETUP # Description: allows allows retention/tagging/access control settings -# Lambda IAM Policy for S3 Write +# Lambda IAM Policy for S3 ######################################################################## # S3 DEFINE POLICY data "aws_iam_policy_document" "s3_data_policy_doc" { statement { + effect = "Allow" actions = [ "s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging", - "s3:PutObjectAcl" + "s3:PutObjectAcl", + "s3:ListObjects" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -46,6 +48,17 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "${aws_s3_bucket.lambda_code_bucket.arn}/*", ] } + + statement { + effect = "Allow" + actions = [ + "s3:ListBuckets", + "s3:ListAllMyBuckets" + ] + resources = [ + "arn:aws:s3:::*", + ] + } } -- cgit v1.2.3 From e537bdef11d1d518d4df1c057f3624e3fe6da24d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:05:41 +0100 Subject: infra(tf): remove rds.tf --- terraform/rds.tf | 70 -------------------------------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 terraform/rds.tf (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf deleted file mode 100644 index a013fb3..0000000 --- a/terraform/rds.tf +++ /dev/null @@ -1,70 +0,0 @@ -# data "aws_availability_zones" "available" {} - -# module "vpc" { -# source = "terraform-aws-modules/vpc/aws" -# version = "5.12.1" - -# name = var.project_name -# cidr = "10.0.0.0/16" -# azs = data.aws_availability_zones.available.names -# public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] -# enable_dns_hostnames = true -# enable_dns_support = true -# } - -# resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { -# name = "tt-db-subnet" -# subnet_ids = module.vpc.public_subnets - -# tags = { -# Name = "${var.project_name}" -# } -# } - -# resource "aws_security_group" "rds" { -# name = "${var.project_name}-rds" -# vpc_id = module.vpc.vpc_id - -# ingress { -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# } - -# egress { -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# } - -# tags = { -# Name = "${var.project_name}-rds" -# } -# } - -# resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { -# name = "tt-db-param" -# family = "postgres14" - -# parameter { -# name = "log_connections" -# value = "1" -# } -# } - -# resource "aws_db_instance" "terrific-totes-rds" { -# db_name = var.project_name -# instance_class = "db.t3.micro" -# allocated_storage = 5 -# engine = "postgres" -# engine_version = "14.10" -# username = "" -# password = "" -# db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name -# vpc_security_group_ids = [aws_security_group.rds.id] -# parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name -# publicly_accessible = false -# skip_final_snapshot = true -# } -- cgit v1.2.3 From 56b2c376a925132f3bf2c7e6cad4911400955129 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:07:30 +0100 Subject: infra(tf): enforce version constraint on terraform --- terraform/main.tf | 1 + 1 file changed, 1 insertion(+) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 310a251..206fc74 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,4 +1,5 @@ terraform { + required_version = ">= 1.8.0" required_providers { aws = { source = "hashicorp/aws" -- cgit v1.2.3 From 35bf4e8668309cb28175ef0224a6bce453abb47f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:19:44 +0100 Subject: chore(tf): replace static tag values in main.tf with variables --- terraform/main.tf | 8 ++++---- terraform/vars.tf | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 206fc74..5ccbec2 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -25,11 +25,11 @@ provider "aws" { region = "eu-west-2" default_tags { tags = { - ProjectName = "Terrific-Totes" - Team = "Team-Bentley" - Environment = "Dev" - GitHubRepo = "de-project-bentley" + ProjectName = var.project_name + Environment = var.environment ManagedBy = "Terraform" + GitHubRepo = var.github_repo + Team = var.team_name } } } diff --git a/terraform/vars.tf b/terraform/vars.tf index 3c88731..1adbcf7 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -33,6 +33,26 @@ variable "project_name" { default = "tt" } +variable "aws_region" { + type = string + default = "eu-west-2" +} + +variable "environment" { + type = string + default = "dev" +} + +variable "github_repo" { + type = string + default = "de-project-bentley" +} + +variable "team_name" { + type = string + default = "Team-Bentley" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} -- cgit v1.2.3 From 95c4fe80aea75a9a63b1cfd85abadaab6b96b876 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:24:23 +0100 Subject: infra(tf): add state file encryption --- terraform/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 5ccbec2..33c760c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -15,9 +15,10 @@ terraform { } } backend "s3" { - bucket = "bentley-project-secrets" - key = "bentley-project/terraform.tfstate" - region = "eu-west-2" + bucket = "bentley-project-secrets" + key = "bentley-project/terraform.tfstate" + region = "eu-west-2" + encrypt = true } } -- cgit v1.2.3 From 57e855a797f225cd77401e85a671cde95e07ee70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:55:19 +0100 Subject: style(tf): improve legibility of lambda.tf sections --- terraform/lambda.tf | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 72aae04..b7b362b 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,4 +1,7 @@ -# Extract Lambda Function +########################### +# Extract Lambda Function # +########################### + data "archive_file" "extract_lambda_zip" { type = "zip" source_file = "${path.module}/../src/extract_lambda.py" @@ -28,7 +31,10 @@ resource "aws_lambda_function" "extract_lambda" { depends_on = [aws_s3_object.extract_lambda_code] } -# Transform Lambda Function +############################# +# Transform Lambda Function # +############################# + data "archive_file" "transform_lambda_zip" { type = "zip" source_file = "${path.module}/../src/transform_lambda.py" @@ -58,7 +64,10 @@ resource "aws_lambda_function" "transform_lambda" { depends_on = [aws_s3_object.transform_lambda_code] } -# Load Lambda Function +######################## +# Load Lambda Function # +######################## + data "archive_file" "load_lambda_zip" { type = "zip" source_file = "${path.module}/../src/load_lambda.py" @@ -88,7 +97,10 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -# Lambda Layer Specification +###################### +# Lambda Layer Setup # +###################### + locals { layer_dir = "../" layer_zip = "layer.zip" -- cgit v1.2.3 From d0e7b1304efe4ab6de2dc5bef1691b389a5bc449 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:58:46 +0100 Subject: refactor(tf): move sections in lambda.tf for better readability --- terraform/lambda.tf | 69 ++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b7b362b..aa730c1 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,3 +1,40 @@ +#################### +# Common Variables # +#################### + +locals { + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" +} + +###################### +# Lambda Layer Setup # +###################### + +resource "null_resource" "prepare_layer" { + provisioner "local-exec" { + command = "bash ${local.script_dir}/make_layer_zip.sh" + } +} + +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "${local.layer_name}/${local.layer_zip}" + source = "${local.layer_dir}/${local.layer_zip}" + depends_on = [null_resource.prepare_layer] +} + +resource "aws_lambda_layer_version" "lambda_layer" { + layer_name = local.layer_name + compatible_runtimes = ["python3.11"] + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.lambda_layer_zip.key + skip_destroy = true + depends_on = [aws_s3_object.lambda_layer_zip] +} + ########################### # Extract Lambda Function # ########################### @@ -97,35 +134,3 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -###################### -# Lambda Layer Setup # -###################### - -locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" -} - -resource "null_resource" "prepare_layer" { - provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" - } -} - -resource "aws_s3_object" "lambda_layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "${local.layer_name}/${local.layer_zip}" - source = "${local.layer_dir}/${local.layer_zip}" - depends_on = [null_resource.prepare_layer] -} - -resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = local.layer_name - compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.lambda_layer_zip.key - skip_destroy = true - depends_on = [aws_s3_object.lambda_layer_zip] -} -- cgit v1.2.3 From cfd6b462a874da77ada8facb3b2a3c0e85059fa4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:07:10 +0100 Subject: infra(tf): only create layer zip if doesn't exist --- terraform/lambda.tf | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index aa730c1..b1e0d8e 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -3,10 +3,11 @@ #################### locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" + layer_zip_path = "${local.layer_dir}/${local.layer_zip}" } ###################### @@ -14,8 +15,13 @@ locals { ###################### resource "null_resource" "prepare_layer" { + + triggers = { + layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" + } + provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" + command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi" } } -- cgit v1.2.3 From f035f60c7ece05b70275760238c5513b8f113310 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:09:26 +0100 Subject: docs(tf): add information about layer zip creation --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b1e0d8e..fc10431 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -16,6 +16,8 @@ locals { resource "null_resource" "prepare_layer" { + # New change: only run the script if the layer zip does not exist + triggers = { layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" } -- cgit v1.2.3 From 40c2952e628a92e63b3468be4d49f44a234cacce Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:13:03 +0100 Subject: infra(tf): add md5/source_code_hash checks for lambda layer --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index fc10431..f8e7515 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -32,6 +32,7 @@ resource "aws_s3_object" "lambda_layer_zip" { key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] + etag = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null } resource "aws_lambda_layer_version" "lambda_layer" { @@ -39,6 +40,7 @@ resource "aws_lambda_layer_version" "lambda_layer" { compatible_runtimes = ["python3.11"] s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key + source_code_hash = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] } -- cgit v1.2.3 From e5715bc33d4470ceccb17c6853c3e52d4b1035d3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:20:09 +0100 Subject: chore(tf): add tags to s3 buckets --- terraform/s3.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..97910c8 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,14 +1,24 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" + + tags = { + Name = "Ingestion Bucket" + } } ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" + tags = { + Name = "Transform Bucket" + } } ### LAMBDA BUCKET resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" + tags = { + Name = "Load Bucket" + } } -- cgit v1.2.3 From 507b3071633fccc9aa1411880dd984ca346a141b Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:22:29 +0100 Subject: docs(tf): improve legibility of s3.tf sections --- terraform/s3.tf | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index 97910c8..6ff58fd 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,4 +1,7 @@ -### EXTRACT BUCKET SET-UP +######################## +# EXTRACT BUCKET SETUP # +######################## + resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" @@ -7,7 +10,10 @@ resource "aws_s3_bucket" "extract_bucket" { } } -### TRANSFORM BUCKET SET-UP +########################## +# TRANSFORM BUCKET SETUP # +########################## + resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" tags = { @@ -15,7 +21,10 @@ resource "aws_s3_bucket" "transform_bucket" { } } -### LAMBDA BUCKET +####################### +# LAMBDA BUCKET SETUP # +####################### + resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" tags = { -- cgit v1.2.3 From 1cb84bd663261c416a516b0dc59dbf8d62c4c1a7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:23:07 +0100 Subject: docs(tf): correct lambda bucket name tag --- terraform/s3.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index 6ff58fd..3e0f5b7 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -28,6 +28,6 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" tags = { - Name = "Load Bucket" + Name = "Lambda Bucket" } } -- cgit v1.2.3 From 795c7c2917c2780e8ffdf0716cbedf3426dcbd5e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:25:21 +0100 Subject: infra(tf): experimental - add versioning to protect against accidental deletes/overwrites" --- terraform/s3.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index 3e0f5b7..edfe0a0 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -10,6 +10,13 @@ resource "aws_s3_bucket" "extract_bucket" { } } +resource "aws_s3_bucket_versioning" "extract_bucket_versioning" { + bucket = aws_s3_bucket.extract_bucket.id + versioning_configuration { + status = "Enabled" + } +} + ########################## # TRANSFORM BUCKET SETUP # ########################## @@ -21,6 +28,14 @@ resource "aws_s3_bucket" "transform_bucket" { } } + +resource "aws_s3_bucket_versioning" "transform_bucket_versioning" { + bucket = aws_s3_bucket.transform_bucket.id + versioning_configuration { + status = "Enabled" + } +} + ####################### # LAMBDA BUCKET SETUP # ####################### -- cgit v1.2.3 From 1bbc12702a8fa6d5139440c9d04e5bfabd96581d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:32:32 +0100 Subject: infra(tf): add versioning to lambda_code_bucket --- terraform/s3.tf | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index edfe0a0..d17a4fe 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -46,3 +46,10 @@ resource "aws_s3_bucket" "lambda_code_bucket" { Name = "Lambda Bucket" } } + +resource "aws_s3_bucket_versioning" "lambda_bucket_versioning" { + bucket = aws_s3_bucket.lambda_code_bucket.id + versioning_configuration { + status = "Enabled" + } +} -- cgit v1.2.3 From b9a3d9dbaa1eedc25d5f8d12bd2be1a8a3841b42 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:39:05 +0100 Subject: docs(tf): improve legibility of events.tf sections --- terraform/events.tf | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 263141f..c2efbbc 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,3 +1,7 @@ +################# +# Random String # +################# + resource "random_string" "eventbridge_suffix" { length = 8 special = false @@ -16,6 +20,10 @@ resource "random_string" "s3_transform_suffix" { upper = false } +############################# +# EventBridge Configuration # +############################# + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" @@ -41,7 +49,10 @@ resource "aws_lambda_permission" "allow_eventbridge" { } } -# below is step function 1 +######################################## +# S3 Extract Bucket Notification Setup # +######################################## + resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" @@ -66,6 +77,10 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { depends_on = [aws_lambda_permission.allow_s3_ingestion] } +########################################## +# S3 Transform Bucket Notification Setup # +########################################## + resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" -- cgit v1.2.3 From 09b8010a453c99164540981060177fdd2280df7e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:41:31 +0100 Subject: infra(tf): remove repetitive suffix resources in events.tf --- terraform/events.tf | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index c2efbbc..832a965 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -2,19 +2,7 @@ # Random String # ################# -resource "random_string" "eventbridge_suffix" { - length = 8 - special = false - upper = false -} - -resource "random_string" "s3_ingestion_suffix" { - length = 8 - special = false - upper = false -} - -resource "random_string" "s3_transform_suffix" { +resource "random_string" "suffix" { length = 8 special = false upper = false @@ -38,14 +26,14 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" + statement_id = "AllowExecutionFromEventBridge${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn lifecycle { - replace_triggered_by = [random_string.eventbridge_suffix] + replace_triggered_by = [random_string.suffix] } } @@ -54,14 +42,14 @@ resource "aws_lambda_permission" "allow_eventbridge" { ######################################## resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder lifecycle { - replace_triggered_by = [random_string.s3_ingestion_suffix] + replace_triggered_by = [random_string.suffix] } } @@ -82,14 +70,14 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ########################################## resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder lifecycle { - replace_triggered_by = [random_string.s3_transform_suffix] + replace_triggered_by = [random_string.suffix] } } -- cgit v1.2.3 From 367100c2d118847a775f4eba87a8c9033c872cb9 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:44:13 +0100 Subject: docs(tf): remove redundant comments --- terraform/events.tf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 832a965..0113f5f 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + arn = aws_lambda_function.extract_lambda.arn depends_on = [aws_lambda_permission.allow_eventbridge] } @@ -44,9 +44,9 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder + source_arn = aws_s3_bucket.extract_bucket.arn lifecycle { replace_triggered_by = [random_string.suffix] @@ -55,11 +55,11 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder + bucket = aws_s3_bucket.extract_bucket.id lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + lambda_function_arn = aws_lambda_function.transform_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -72,9 +72,9 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder + source_arn = aws_s3_bucket.transform_bucket.arn lifecycle { replace_triggered_by = [random_string.suffix] @@ -83,11 +83,11 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder + bucket = aws_s3_bucket.transform_bucket.id lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + lambda_function_arn = aws_lambda_function.transform_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_transform_bucket] -- cgit v1.2.3 From a9fb82f5c96e0ba98d6d3453ce900f2ca22157ec Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:48:00 +0100 Subject: infra(tf): remove unused declaration in vars.tf --- terraform/vars.tf | 5 ----- 1 file changed, 5 deletions(-) (limited to 'terraform') diff --git a/terraform/vars.tf b/terraform/vars.tf index 1adbcf7..b3e3e47 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -33,11 +33,6 @@ variable "project_name" { default = "tt" } -variable "aws_region" { - type = string - default = "eu-west-2" -} - variable "environment" { type = string default = "dev" -- cgit v1.2.3 From c091506dc8e01741f54f9a8d289515c8d5ffbecf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:49:00 +0100 Subject: infra(tf): add version constraint for random in main.tf --- terraform/main.tf | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 33c760c..ad7b335 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,6 +13,10 @@ terraform { source = "hashicorp/archive" version = "~>2.5.0" } + random_string = { + source = "hashicorp/random" + version = "~>3.6.2" + } } backend "s3" { bucket = "bentley-project-secrets" -- cgit v1.2.3 From ce2761b311523a118cdead885ba7fcf1f7a4cd68 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:52:15 +0100 Subject: fix(tf): correct random_string to random in main.tf --- terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index ad7b335..6577b70 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,7 +13,7 @@ terraform { source = "hashicorp/archive" version = "~>2.5.0" } - random_string = { + random = { source = "hashicorp/random" version = "~>3.6.2" } -- cgit v1.2.3 From a8ce060732ed3064696f2d6c5459ffa176fd02f7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:02:56 +0100 Subject: fix(tf): lambda permissions should be created before destroyed --- terraform/events.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 0113f5f..9885a86 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -33,7 +33,8 @@ resource "aws_lambda_permission" "allow_eventbridge" { source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } @@ -49,7 +50,8 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { source_arn = aws_s3_bucket.extract_bucket.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } @@ -77,7 +79,8 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { source_arn = aws_s3_bucket.transform_bucket.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } -- cgit v1.2.3 From b8574d4c4bf262a8034d21b770fd4287022c2648 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:07:28 +0100 Subject: fix(tf): re-add separate random_string suffixes in events.tf --- terraform/events.tf | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 9885a86..53ae10a 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -2,7 +2,19 @@ # Random String # ################# -resource "random_string" "suffix" { +resource "random_string" "eventbridge_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_ingestion_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_transform_suffix" { length = 8 special = false upper = false @@ -26,7 +38,7 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge${random_string.suffix.result}" + statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" @@ -34,7 +46,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.eventbridge_suffix] } } @@ -43,7 +55,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { ######################################## resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" @@ -51,7 +63,7 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.s3_ingestion_suffix] } } @@ -72,7 +84,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ########################################## resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" @@ -80,7 +92,7 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.s3_transform_suffix] } } -- cgit v1.2.3 From 746f4206b2f30126c3c09ac11a2d49be3259fe6f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:42:54 +0100 Subject: infra(tf): add secrets manager permissions I feel like what I've done is bad but we'll find out soon. --- terraform/iam.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 7585ff8..a36cfdf 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -169,3 +169,30 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.cloudwatch_events_policy.arn } + +######################### +# SECRETS MANAGER SETUP # +######################### + +# Policy Doc +data "aws_iam_policy_document" "secrets_manager_policy_doc" { + statement { + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue" + ] + resources = [] + } +} + +# SM Policy Resource +resource "aws_iam_policy" "secrets_manager_policy" { + name = "secrets_manager_policy" + policy = data.aws_iam_policy_document.secrets_manager_policy_doc.json +} + +# Attach SM Policy to Role +resource "aws_iam_role_policy_attachment" "secrets_manager_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.secrets_manager_policy.arn +} -- cgit v1.2.3 From 2045888e1ae497444c58347096547f0475bba7a1 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:51:11 +0100 Subject: infra(tf): add resource access for secrets-manager policy doc --- terraform/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index a36cfdf..a8054ca 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -181,7 +181,7 @@ data "aws_iam_policy_document" "secrets_manager_policy_doc" { actions = [ "secretsmanager:GetSecretValue" ] - resources = [] + resources = ["arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:bentley-secrets-Na0yc8"] } } -- cgit v1.2.3 From ae57535d9f201d6fd749d4286551884d3c86fd60 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 10:26:48 +0100 Subject: infra(tf): add missing ListObjectsV2 permission --- terraform/iam.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index a8054ca..3ac8c45 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -40,7 +40,8 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectRetention", "s3:PutObjectTagging", "s3:PutObjectAcl", - "s3:ListObjects" + "s3:ListObjects", + "s3:ListObjectsV2" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", -- cgit v1.2.3 From 346aadfbf2208a0660ffc09959a91fc2f7b48c79 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 12:07:17 +0100 Subject: infra(tf): force-destroy buckets --- terraform/s3.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index d17a4fe..14e8835 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -4,7 +4,7 @@ resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" - + force_destroy = true tags = { Name = "Ingestion Bucket" } @@ -23,6 +23,7 @@ resource "aws_s3_bucket_versioning" "extract_bucket_versioning" { resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" + force_destroy = true tags = { Name = "Transform Bucket" } @@ -42,6 +43,7 @@ resource "aws_s3_bucket_versioning" "transform_bucket_versioning" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" + force_destroy = true tags = { Name = "Lambda Bucket" } -- cgit v1.2.3 From 5493cdc71da4730c4e388d9718f278bc2f14badf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:15:28 +0100 Subject: infra(tf): add ListBucket and GetObject permissions --- terraform/iam.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 3ac8c45..3d62b69 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -41,7 +41,8 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectTagging", "s3:PutObjectAcl", "s3:ListObjects", - "s3:ListObjectsV2" + "s3:ListObjectsV2", + "s3:GetObject" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -53,8 +54,10 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { statement { effect = "Allow" actions = [ - "s3:ListBuckets", - "s3:ListAllMyBuckets" + "s3:ListBucket", + "s3:ListAllMyBuckets", + "s3:ListObjectsV2", + "s3:ListObjects" ] resources = [ "arn:aws:s3:::*", -- cgit v1.2.3 From d1f3fd505c4deea8cc811a851cda15d00419ade3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:50:23 +0100 Subject: hotfix: increase lambda timeout --- terraform/lambda.tf | 3 +++ 1 file changed, 3 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index f8e7515..d33a6c9 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -70,6 +70,7 @@ resource "aws_lambda_function" "extract_lambda" { handler = "extract_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true @@ -103,6 +104,7 @@ resource "aws_lambda_function" "transform_lambda" { handler = "transform_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true @@ -136,6 +138,7 @@ resource "aws_lambda_function" "load_lambda" { handler = "load_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true -- cgit v1.2.3