From fe5c2ef279b3190b242df08de2b680b4de5cac4c Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Mon, 12 Aug 2024 16:51:56 +0100 Subject: Completed the main.tf and vars.tf file. Currently: figuring out the configurations for AWS IAM USERS --- terraform/main.tf | 26 ++++++++++++++++++++++++++ terraform/vars.tf | 23 +++++++++++++++++++++++ 2 files changed, 49 insertions(+) create mode 100644 terraform/main.tf create mode 100644 terraform/vars.tf (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..3ca9a3d --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,26 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~>5.0" + } + } + backend "s3" { + bucket = "bentley-secrets" + key = "bentley-project/terraform.tfstate" + region = "eu-west-2" + } +} + +provider "aws" { + region = "eu-west-2" + default_tags { + tags = { + ProjectName = "Terrific-Totes" + Team = "Team-Bentley" + Environment = "Dev" + GitHubRepo = "de-project-bentley" + ManagedBy = "Terraform" + } + } +} \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf new file mode 100644 index 0000000..166f2c5 --- /dev/null +++ b/terraform/vars.tf @@ -0,0 +1,23 @@ +variable "s3_extract_bucket_name" { + type = string + default = "extract-bucket" +} + +variable "s3_transform_bucket_name" { + type = string + default = "transform-bucket" +} + +variable "extract_lambda_name" { + type = string + default = "extract-lambda" +} + +variable "transform_lambda_name" { + type = string + default = "transform-lambda" +} + +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} \ No newline at end of file -- cgit v1.2.3 From ef770c1ea4ee633489323a8ab321b1214b51a770 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 12 Aug 2024 16:57:11 +0100 Subject: chore: add aws_iam_role --- terraform/iam.tf | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 terraform/iam.tf (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..7501373 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,29 @@ +# define + +resource "aws_iam_role" "bentley_service_role" { + assume_role_policy = < Date: Tue, 13 Aug 2024 09:47:21 +0100 Subject: [feat]/add eventbridge schedule and step function trigger 1 of 2 --- terraform/events.tf | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 terraform/events.tf (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf new file mode 100644 index 0000000..25fb35b --- /dev/null +++ b/terraform/events.tf @@ -0,0 +1,52 @@ +resource "aws_cloudwatch_event_rule" "lambda_trigger" { + name = "lambda-scheduled-trigger" + description = "Schedule to trigger the Lambda function" + schedule_expression = "rate(30 minutes)" + +# event_pattern = jsonencode({ +# detail-type = [ +# "AWS Console Sign In via CloudTrail" +# ] +# }) +} + + +resource "aws_cloudwatch_event_target" "lambda" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.my_lambda_function.arn +} + + + +resource "aws_lambda_permission" "allow_eventbridge" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.my_lambda_function.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn +} + + +# below is step function 1 +resource "aws_lambda_permission" "allow_s3_ingestion" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_transform.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract.arn +} + + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract.id + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.lambda_transform.arn + } + + depends_on = [aws_lambda_permission.allow_s3_ingestion] +} + +# need to duplicate and replace "2" with "3" \ No newline at end of file -- cgit v1.2.3 From e3e9817f4e88afc8eb89e0b18a7fe8b1f381e0d4 Mon Sep 17 00:00:00 2001 From: T-Aji Date: Tue, 13 Aug 2024 09:57:14 +0100 Subject: [feat]/add step function trigger 2 of 2 --- terraform/events.tf | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 25fb35b..4d68a23 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -49,4 +49,24 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { depends_on = [aws_lambda_permission.allow_s3_ingestion] } -# need to duplicate and replace "2" with "3" \ No newline at end of file +###### + +resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_transform.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform.arn +} + + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform.id + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.lambda_transform.arn + } + + depends_on = [aws_lambda_permission.allow_s3_transform] +} \ No newline at end of file -- cgit v1.2.3 From c8d8d2cee4262782890ea68cda8fc86f61098b09 Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 13 Aug 2024 10:14:01 +0100 Subject: s3 buckets in tf and initial blank lambda py files set-up in src folder for the next task of lambda tf set up --- src/extract_lambda.py | 0 src/load_lambda.py | 0 src/transform_lambda.py | 0 terraform/s3.tf | 17 +++++++++++++++++ terraform/vars.tf | 5 +++++ 5 files changed, 22 insertions(+) create mode 100644 src/extract_lambda.py create mode 100644 src/load_lambda.py create mode 100644 src/transform_lambda.py create mode 100644 terraform/s3.tf (limited to 'terraform') diff --git a/src/extract_lambda.py b/src/extract_lambda.py new file mode 100644 index 0000000..e69de29 diff --git a/src/load_lambda.py b/src/load_lambda.py new file mode 100644 index 0000000..e69de29 diff --git a/src/transform_lambda.py b/src/transform_lambda.py new file mode 100644 index 0000000..e69de29 diff --git a/terraform/s3.tf b/terraform/s3.tf new file mode 100644 index 0000000..bfe891e --- /dev/null +++ b/terraform/s3.tf @@ -0,0 +1,17 @@ +resource "aws_s3_bucket" "extract_bucket" { + bucket = "${var.s3_extract_bucket_name}" +} + +resource "aws_s3_bucket" "transform_bucket" { + bucket = "${var.s3_transform_bucket_name}" +} + +resource "aws_s3_bucket" "lambda_bucket" { + bucket = "${var.s3_code_bucket_name}" +} + +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.extract_lambda_name}/function_e.zip" + source = "${path.module}/../function_e.zip" +} \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf index 166f2c5..fa84222 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -8,6 +8,11 @@ variable "s3_transform_bucket_name" { default = "transform-bucket" } +variable "s3_code_bucket_name" { + type = string + default = "lambda-bucket" +} + variable "extract_lambda_name" { type = string default = "extract-lambda" -- cgit v1.2.3 From c75e650dbeb1390336d15487a2c87c53337cd8dc Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 11:25:33 +0100 Subject: infra(tf): add s3 policy for list & write --- terraform/iam.tf | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 7501373..b9919a5 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -1,5 +1,3 @@ -# define - resource "aws_iam_role" "bentley_service_role" { assume_role_policy = < Date: Tue, 13 Aug 2024 11:55:00 +0100 Subject: infra(tf): add wip write policy and attach policy --- terraform/iam.tf | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index b9919a5..dda4d74 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -22,17 +22,12 @@ resource "aws_iam_role" "bentley_service_role" { EOF } +# lambda setup + + # s3 setup -# allows to list and retrieve s3 buckets, and allows retention/tagging/access control settings +# allows allows retention/tagging/access control settings data "aws_iam_policy_document" "s3_data_policy_doc" { - statement { - actions = [ - "s3:ListAllMyBuckets", - "s3:GetBucketLocation" - ] - resources = ["arn:aws:s3:::*"] - } - statement { actions = [ "s3:PutObject", @@ -41,8 +36,22 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectAcl" ] resources = [ - "${aws_s3_bucket.data_bucket.arn}/*", - "${aws_s3_bucket.code_bucket.arn}/*" + "${aws_s3_bucket.extract_bucket.arn}/*", + "${aws_s3_bucket.transform_bucket.arn}/*", + "${aws_s3_bucket.lambda_bucket.arn}/*", ] } -} \ No newline at end of file +} + +# write policy +resource "aws_iam_policy" "s3_policy" { + policy = data.aws_iam_policy_document.s3_data_policy_doc.json +} + +# attach policy to role +resource "aws_iam_role_policy_attachment" "s3_policy_attachment" { + role = aws_iam_role.bentley_service_role.name + policy_arn = aws_iam_policy.s3_policy.arn +} + +# lambda setup -- cgit v1.2.3 From 936eee1eb44d8bfdbd148d22b749966e9606fb46 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 11:58:02 +0100 Subject: infra(tf): add wip lambda role --- terraform/iam.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index dda4d74..10b8749 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -23,6 +23,9 @@ resource "aws_iam_role" "bentley_service_role" { } # lambda setup +resource "aws_iam_role" "lambda_role" { + assume_role_policy = data.aws_iam_policy_document.bentley_service_role.json +} # s3 setup @@ -44,14 +47,14 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { } # write policy -resource "aws_iam_policy" "s3_policy" { +resource "aws_iam_policy" "s3_write_policy" { policy = data.aws_iam_policy_document.s3_data_policy_doc.json } # attach policy to role resource "aws_iam_role_policy_attachment" "s3_policy_attachment" { - role = aws_iam_role.bentley_service_role.name - policy_arn = aws_iam_policy.s3_policy.arn + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.s3_write_policy.arn } # lambda setup -- cgit v1.2.3 From eb09f0f6a42e2a2ce9529492a47a34f782ffad53 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 12:17:52 +0100 Subject: infra(tf): clean-up code & init lambda iam setup --- terraform/iam.tf | 123 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 74 insertions(+), 49 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 10b8749..ecc63b1 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -1,60 +1,85 @@ -resource "aws_iam_role" "bentley_service_role" { - assume_role_policy = < Date: Tue, 13 Aug 2024 12:30:20 +0100 Subject: replacement for events.tf variable placeholders, s3 bucket notifications, lambda.tf set up (function, zip, permissions), code bucket to store zipped lambdas --- events - TBD?.tf | 52 +++++++++++++++++++++++++++++++++++++ events.tf | 52 ------------------------------------- terraform/events.tf | 23 ++++++++--------- terraform/lambda.tf | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++ terraform/s3.tf | 39 ++++++++++++++++++++++------ terraform/vars.tf | 5 ++++ 6 files changed, 173 insertions(+), 72 deletions(-) create mode 100644 events - TBD?.tf delete mode 100644 events.tf create mode 100644 terraform/lambda.tf (limited to 'terraform') diff --git a/events - TBD?.tf b/events - TBD?.tf new file mode 100644 index 0000000..25fb35b --- /dev/null +++ b/events - TBD?.tf @@ -0,0 +1,52 @@ +resource "aws_cloudwatch_event_rule" "lambda_trigger" { + name = "lambda-scheduled-trigger" + description = "Schedule to trigger the Lambda function" + schedule_expression = "rate(30 minutes)" + +# event_pattern = jsonencode({ +# detail-type = [ +# "AWS Console Sign In via CloudTrail" +# ] +# }) +} + + +resource "aws_cloudwatch_event_target" "lambda" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.my_lambda_function.arn +} + + + +resource "aws_lambda_permission" "allow_eventbridge" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.my_lambda_function.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn +} + + +# below is step function 1 +resource "aws_lambda_permission" "allow_s3_ingestion" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_transform.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract.arn +} + + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract.id + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.lambda_transform.arn + } + + depends_on = [aws_lambda_permission.allow_s3_ingestion] +} + +# need to duplicate and replace "2" with "3" \ No newline at end of file diff --git a/events.tf b/events.tf deleted file mode 100644 index 25fb35b..0000000 --- a/events.tf +++ /dev/null @@ -1,52 +0,0 @@ -resource "aws_cloudwatch_event_rule" "lambda_trigger" { - name = "lambda-scheduled-trigger" - description = "Schedule to trigger the Lambda function" - schedule_expression = "rate(30 minutes)" - -# event_pattern = jsonencode({ -# detail-type = [ -# "AWS Console Sign In via CloudTrail" -# ] -# }) -} - - -resource "aws_cloudwatch_event_target" "lambda" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn -} - - - -resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn -} - - -# below is step function 1 -resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn -} - - -resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id - - lambda_function { - events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn - } - - depends_on = [aws_lambda_permission.allow_s3_ingestion] -} - -# need to duplicate and replace "2" with "3" \ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 4d68a23..0196dc3 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -11,18 +11,17 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { } -resource "aws_cloudwatch_event_target" "lambda" { +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder } - resource "aws_lambda_permission" "allow_eventbridge" { statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name + function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn } @@ -32,18 +31,18 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn + source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id + bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -54,18 +53,18 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform.arn + source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform.id + bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_transform] diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..09d6697 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,74 @@ +### EXTRACT LAMBDA SET UP +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} + +resource "aws_lambda_function" "extract_lambda" { + function_name = "${var.extract_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "extract_lambda/extract_function.zip" + role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.extract_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn +} + + +### TRANSFORM LAMBDA SET UP +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} + +resource "aws_lambda_function" "transform_lambda" { + function_name = "${var.transform_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "transform_lambda/transform_function.zip" + role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.transform_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn +} + + +### LOAD LAMBDA SET UP +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} + +resource "aws_lambda_function" "load_lambda" { + function_name = "${var.load_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "load_lambda/load_function.zip" + role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" +} + diff --git a/terraform/s3.tf b/terraform/s3.tf index bfe891e..8cb65ef 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,17 +1,40 @@ +### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket = "${var.s3_extract_bucket_name}" } +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.extract_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +} # << is this the correct permission dependency? + + +### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket = "${var.s3_transform_bucket_name}" } -resource "aws_s3_bucket" "lambda_bucket" { - bucket = "${var.s3_code_bucket_name}" -} - -resource "aws_s3_object" "extract_lambda_code" { +resource "aws_s3_object" "transform_lambda_code" { bucket = aws_s3_bucket.s3_code_bucket_name.bucket - key = "${var.extract_lambda_name}/function_e.zip" - source = "${path.module}/../function_e.zip" -} \ No newline at end of file + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.transform_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +} # << is this the correct permission dependency? diff --git a/terraform/vars.tf b/terraform/vars.tf index fa84222..cc9348a 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -23,6 +23,11 @@ variable "transform_lambda_name" { default = "transform-lambda" } +variable "load_lambda_name" { + type = string + default = "load-lambda" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} \ No newline at end of file -- cgit v1.2.3 From 3c824df60374380d044cb9181672fa76b610d84f Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 12:30:53 +0100 Subject: infra(tf): clean-up code --- terraform/iam.tf | 69 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 13 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index ecc63b1..bb8d932 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -4,7 +4,7 @@ ######################################################################## # DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events) -resource "aws_iam_role" "multi_service_role" { +resource "aws_iam_role" "bentley_multi_service_role" { name = "multi_service_role" assume_role_policy = jsonencode({ @@ -61,6 +61,61 @@ resource "aws_iam_policy" "s3_access_policy" { ) } +######################################################################## +# LAMBDA SETUP +# Description: Allows Lambda permission to write to Cloudwatch logs +######################################################################## + +resource "aws_iam_policy" "lambda_execution_policy" { + name = "lambda_execution_policy" + path = "/" + description = "IAM policy for Lambda execution" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "lambda:InvokeFunction", + "lambda:GetFunction" + ] + Resource = "*" + } + ] + } + ) +} + +######################################################################## +# CLOUDWATCH SETUP +# Description: Give permission for Lambda to write to CloudWatch logs +######################################################################## + +data "aws_iam_policy_document" "cw_document" { + statement { + actions = ["logs:CreateLogGroup"] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" + ] + } + + statement { + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] + } +} + +######################################################################## +# POLICY WRITE & ATTACH +######################################################################## + # S3 WRITE POLICY resource "aws_iam_policy" "s3_write_policy" { policy = data.aws_iam_policy_document.s3_data_policy_doc.json @@ -70,16 +125,4 @@ resource "aws_iam_policy" "s3_write_policy" { resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { role = aws_iam_role.lambda_role.name policy_arn = aws_iam_policy.s3_write_policy.arn -} - -######################################################################## -# LAMBDA SETUP -# Description: Allows Lambda permission to write to Cloudwatch logs -######################################################################## - - - -# Uses Iam policy document to assume role for lambda functions -resource "aws_iam_role" "lambda_role" { - assume_role_policy = data.aws_iam_policy_document.bentley_service_role.json } \ No newline at end of file -- cgit v1.2.3 From bb1665fd08d8abf10930875272bdc2b7f8a4c681 Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 13 Aug 2024 16:25:06 +0100 Subject: rds.tf file with some placehodlers. Additional set up is required --- terraform/lambda.tf | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ terraform/rds.tf | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ terraform/vars.tf | 10 +++++++ 3 files changed, 162 insertions(+) create mode 100644 terraform/lambda.tf create mode 100644 terraform/rds.tf (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..09d6697 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,74 @@ +### EXTRACT LAMBDA SET UP +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} + +resource "aws_lambda_function" "extract_lambda" { + function_name = "${var.extract_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "extract_lambda/extract_function.zip" + role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.extract_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn +} + + +### TRANSFORM LAMBDA SET UP +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} + +resource "aws_lambda_function" "transform_lambda" { + function_name = "${var.transform_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "transform_lambda/transform_function.zip" + role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.transform_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn +} + + +### LOAD LAMBDA SET UP +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} + +resource "aws_lambda_function" "load_lambda" { + function_name = "${var.load_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "load_lambda/load_function.zip" + role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" +} + diff --git a/terraform/rds.tf b/terraform/rds.tf new file mode 100644 index 0000000..4b25c5f --- /dev/null +++ b/terraform/rds.tf @@ -0,0 +1,78 @@ +data "aws_availability_zones" "available" {} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.77.0" + + name = "${var.project_name}" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_dns_hostnames = true + enable_dns_support = true +} + +resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { + name = "TT-db-subnet" + subnet_ids = module.vpc.public_subnets + + tags = { + Name = "${var.project_name}" + } +} + +resource "aws_security_group" "rds" { + name = "${var.project_name}-rds" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-rds" + } +} + +resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { + name = "TT-db-param" + family = "postgres14" + + parameter { + name = "log_connections" + value = "1" + } +} + +resource "aws_db_instance" "Terrific-Totes-rds" { + db_name = "${var.project_name}" + instance_class = "db.t3.micro" + allocated_storage = 5 + engine = "postgres" + engine_version = "14.1" + username = "user credentials for the root user" # we could use .env here + password = "user password for the root user" # we could use .env here + ### alternatively to providing username nad password we can specify: +# resource "aws_kms_key" "example_key" { +# description = "Example KMS Key" +# } +# within the resource: +# manage_master_user_password = true +# master_user_secret_kms_key_id = aws_kms_key.example.key_id +# } + db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name + vpc_security_group_ids = [aws_security_group.rds.id] + parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name + publicly_accessible = false + skip_final_snapshot = true +} \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf index fa84222..350c2c6 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -23,6 +23,16 @@ variable "transform_lambda_name" { default = "transform-lambda" } +variable "load_lambda_name" { + type = string + default = "load-lambda" +} + +variable "project_name" { + type = string + default = "Terrific-Totes" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} \ No newline at end of file -- cgit v1.2.3 From 68a0b4740e1aab2c507547ab985c7c1dc436d9c9 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Tue, 13 Aug 2024 17:16:12 +0100 Subject: wip: running terraform apply to continue fixing terraform infrastructure --- .gitignore | 5 +++++ terraform/events.tf | 4 ++-- terraform/iam.tf | 64 ++++++++++++++++++++++++++++------------------------- terraform/lambda.tf | 13 +++++------ terraform/s3.tf | 63 ++++++++++++++++++++++++++++++---------------------- 5 files changed, 84 insertions(+), 65 deletions(-) create mode 100644 .gitignore (limited to 'terraform') diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5861f48 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.tfstate +*.tfstate.* +*.tfvars +*.tfvars.json +.terraform.tfstate.lock.info \ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 0196dc3..7a6b0ad 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -50,7 +50,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ###### -resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { +resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder @@ -67,5 +67,5 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" { lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } - depends_on = [aws_lambda_permission.allow_s3_transform] + depends_on = [aws_lambda_permission.allow_s3_transform_bucket] } \ No newline at end of file diff --git a/terraform/iam.tf b/terraform/iam.tf index bb8d932..f34d58a 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -4,7 +4,7 @@ ######################################################################## # DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events) -resource "aws_iam_role" "bentley_multi_service_role" { +resource "aws_iam_role" "multi_service_role" { name = "multi_service_role" assume_role_policy = jsonencode({ @@ -16,7 +16,7 @@ resource "aws_iam_role" "bentley_multi_service_role" { Principal = { Service = [ "lambda.amazonaws.com", - "states.amazonaws.com", + "cloudwatch.amazonaws.com", "events.amazonaws.com", "s3.amazonaws.com" ] @@ -27,7 +27,6 @@ resource "aws_iam_role" "bentley_multi_service_role" { } - ######################################################################## # S3 SETUP # Description: allows allows retention/tagging/access control settings @@ -35,32 +34,23 @@ resource "aws_iam_role" "bentley_multi_service_role" { ######################################################################## # S3 DEFINE POLICY -resource "aws_iam_policy" "s3_access_policy" { - name = "s3_access_policy" - path = "/" - description = "IAM policy for S3 access" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucket" - ] - resources = [ - "${aws_s3_bucket.extract_bucket.arn}/*", - "${aws_s3_bucket.transform_bucket.arn}/*", - "${aws_s3_bucket.lambda_bucket.arn}/*" - ] - } - ] - } - ) +data "aws_iam_policy_document" "s3_data_policy_doc" { + statement { + actions = [ + "s3:PutObject", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectAcl" + ] + resources = [ + "${aws_s3_bucket.extract_bucket.arn}/*", + "${aws_s3_bucket.transform_bucket.arn}/*", + "${aws_s3_bucket.lambda_code_bucket.arn}/*", + ] + } } + ######################################################################## # LAMBDA SETUP # Description: Allows Lambda permission to write to Cloudwatch logs @@ -112,6 +102,11 @@ data "aws_iam_policy_document" "cw_document" { } } +resource "aws_iam_policy" "cw_policy" { + name = "cw_policy" + policy = data.aws_iam_policy_document.cw_document.json +} + ######################################################################## # POLICY WRITE & ATTACH ######################################################################## @@ -123,6 +118,15 @@ resource "aws_iam_policy" "s3_write_policy" { # S3 ATTACH POLICY resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { - role = aws_iam_role.lambda_role.name - policy_arn = aws_iam_policy.s3_write_policy.arn -} \ No newline at end of file + for_each = toset([ + aws_iam_policy.s3_write_policy.arn, + aws_iam_policy.lambda_execution_policy.arn, + aws_iam_policy.cw_policy.arn + ]) + role = aws_iam_role.multi_service_role.name + policy_arn = each.value +} + +################ +# RDS POLICIES # +################ diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 09d6697..bcbf394 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -7,9 +7,9 @@ data "archive_file" "extract_lambda_zip" { resource "aws_lambda_function" "extract_lambda" { function_name = "${var.extract_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "extract_lambda/extract_function.zip" - role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" environment { @@ -36,9 +36,9 @@ data "archive_file" "transform_lambda_zip" { resource "aws_lambda_function" "transform_lambda" { function_name = "${var.transform_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "transform_lambda/transform_function.zip" - role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" environment { @@ -55,7 +55,6 @@ resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { source_arn = aws_s3_bucket.transform_bucket.arn } - ### LOAD LAMBDA SET UP data "archive_file" "load_lambda_zip" { type = "zip" @@ -65,9 +64,9 @@ data "archive_file" "load_lambda_zip" { resource "aws_lambda_function" "load_lambda" { function_name = "${var.load_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "load_lambda/load_function.zip" - role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" } diff --git a/terraform/s3.tf b/terraform/s3.tf index 8cb65ef..8ab5622 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,40 +1,51 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { - bucket = "${var.s3_extract_bucket_name}" + bucket_prefix = "${var.s3_extract_bucket_name}-" } -resource "aws_s3_object" "extract_lambda_code" { - bucket = aws_s3_bucket.s3_code_bucket_name.bucket - key = "${var.extract_lambda_name}/extract_function.zip" - source = "${path.module}/../extract_function.zip" -} # << can't figure out how this is being used but we seem to need it - -resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract_bucket.id - lambda_function { - lambda_function_arn = aws_lambda_function.extract_lambda.arn - events = ["s3:ObjectCreated:*"] - } - depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] -} # << is this the correct permission dependency? - +# resource "aws_s3_bucket_notification" "extract_bucket_notification" { +# bucket = aws_s3_bucket.extract_bucket.id +# lambda_function { +# lambda_function_arn = aws_lambda_function.extract_lambda.arn +# events = ["s3:ObjectCreated:*"] +# } +# depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +# } # << is this the correct permission dependency? ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { - bucket = "${var.s3_transform_bucket_name}" + bucket_prefix = "${var.s3_transform_bucket_name}-" } +# resource "aws_s3_bucket_notification" "transform_bucket_notification" { +# bucket = aws_s3_bucket.transform_bucket.id +# lambda_function { +# lambda_function_arn = aws_lambda_function.transform_lambda.arn +# events = ["s3:ObjectCreated:*"] +# } +# depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +# } # << is this the correct permission dependency? + + +### LAMBDA BUCKET +resource "aws_s3_bucket" "lambda_code_bucket" { + bucket_prefix = "${var.s3_code_bucket_name}-" +} + +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + resource "aws_s3_object" "transform_lambda_code" { - bucket = aws_s3_bucket.s3_code_bucket_name.bucket + bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.transform_lambda_name}/transform_function.zip" source = "${path.module}/../transform_function.zip" } # << can't figure out how this is being used but we seem to need it -resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform_bucket.id - lambda_function { - lambda_function_arn = aws_lambda_function.transform_lambda.arn - events = ["s3:ObjectCreated:*"] - } - depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] -} # << is this the correct permission dependency? +resource "aws_s3_object" "load_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.load_lambda_name}/load_function.zip" + source = "${path.module}/../load_function.zip" +} \ No newline at end of file -- cgit v1.2.3 From 9966fcff43cc5e748bc3eb406c270d63fa51ca61 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:35:25 +0100 Subject: change the s3-key --- terraform/lambda.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bcbf394..a5da972 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -8,7 +8,7 @@ data "archive_file" "extract_lambda_zip" { resource "aws_lambda_function" "extract_lambda" { function_name = "${var.extract_lambda_name}" s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "extract_lambda/extract_function.zip" + s3_key = "extract-lambda/extract_function.zip" role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" @@ -37,7 +37,7 @@ data "archive_file" "transform_lambda_zip" { resource "aws_lambda_function" "transform_lambda" { function_name = "${var.transform_lambda_name}" s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "transform_lambda/transform_function.zip" + s3_key = "transform-lambda/transform_function.zip" role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" @@ -65,7 +65,7 @@ data "archive_file" "load_lambda_zip" { resource "aws_lambda_function" "load_lambda" { function_name = "${var.load_lambda_name}" s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "load_lambda/load_function.zip" + s3_key = "load-lambda/load_function.zip" role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" -- cgit v1.2.3 From 6caff95389a6054f8fe1d1c9c23db9a251b6b355 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:35:59 +0100 Subject: change the name of bucket in backend bucket --- terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 3ca9a3d..2a048b9 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -6,7 +6,7 @@ terraform { } } backend "s3" { - bucket = "bentley-secrets" + bucket = "bentley-project-secrets" key = "bentley-project/terraform.tfstate" region = "eu-west-2" } -- cgit v1.2.3 From 4d52771ceee6841febc6179e0b9608fb35a792d9 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:36:22 +0100 Subject: change the attachment/policies, individual --- terraform/iam.tf | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index f34d58a..cf4902a 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -117,14 +117,29 @@ resource "aws_iam_policy" "s3_write_policy" { } # S3 ATTACH POLICY -resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { - for_each = toset([ - aws_iam_policy.s3_write_policy.arn, - aws_iam_policy.lambda_execution_policy.arn, - aws_iam_policy.cw_policy.arn - ]) - role = aws_iam_role.multi_service_role.name - policy_arn = each.value +# resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { +# for_each = toset([ +# aws_iam_policy.s3_write_policy.arn, +# aws_iam_policy.lambda_execution_policy.arn, +# aws_iam_policy.cw_policy.arn +# ]) +# role = aws_iam_role.multi_service_role.name +# policy_arn = each.value +# } + +resource "aws_iam_role_policy_attachment" "s3_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.s3_write_policy.arn +} + +resource "aws_iam_role_policy_attachment" "lambda_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.lambda_execution_policy.arn +} + +resource "aws_iam_role_policy_attachment" "cw_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.cw_policy.arn } ################ -- cgit v1.2.3 From f2c0429698edafccb0846a58e4020bf419e7e824 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:37:31 +0100 Subject: style change --- terraform/events.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 7a6b0ad..6744085 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -4,7 +4,7 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { schedule_expression = "rate(30 minutes)" # event_pattern = jsonencode({ -# detail-type = [ +# detail-type = # "AWS Console Sign In via CloudTrail" # ] # }) -- cgit v1.2.3 From 101e1e24cb38b6a45661b723881e2b2d6dd2fb07 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 14:35:05 +0100 Subject: wip: terraform debugging --- .gitignore | 5 ++++- src/load_lambda.py | 2 ++ src/transform_lambda.py | 2 ++ terraform/events.tf | 18 ++++++++++-------- terraform/s3.tf | 34 +++++++++++++++++----------------- 5 files changed, 35 insertions(+), 26 deletions(-) (limited to 'terraform') diff --git a/.gitignore b/.gitignore index 239c7e0..d759665 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ *.tfvars *.tfvars.json .terraform.tfstate.lock.info -*.zip \ No newline at end of file +*.zip +.terraform/ +.terraform* +log* \ No newline at end of file diff --git a/src/load_lambda.py b/src/load_lambda.py index e69de29..6ee681f 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -0,0 +1,2 @@ +def lambda_handler(): + pass \ No newline at end of file diff --git a/src/transform_lambda.py b/src/transform_lambda.py index e69de29..6ee681f 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -0,0 +1,2 @@ +def lambda_handler(): + pass \ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 6744085..9fd89e4 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,7 +1,17 @@ +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + force_destroy = true +} + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" schedule_expression = "rate(30 minutes)" + force_destroy = true + # depends_on = [ + # aws_cloudwatch_event_target.extract_lambda_cw_event] # event_pattern = jsonencode({ # detail-type = @@ -10,14 +20,6 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { # }) } - -resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder -} - - resource "aws_lambda_permission" "allow_eventbridge" { statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" diff --git a/terraform/s3.tf b/terraform/s3.tf index 8ab5622..4c06b8e 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -32,20 +32,20 @@ resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } -resource "aws_s3_object" "extract_lambda_code" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "${var.extract_lambda_name}/extract_function.zip" - source = "${path.module}/../extract_function.zip" -} # << can't figure out how this is being used but we seem to need it - -resource "aws_s3_object" "transform_lambda_code" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "${var.transform_lambda_name}/transform_function.zip" - source = "${path.module}/../transform_function.zip" -} # << can't figure out how this is being used but we seem to need it - -resource "aws_s3_object" "load_lambda_code" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "${var.load_lambda_name}/load_function.zip" - source = "${path.module}/../load_function.zip" -} \ No newline at end of file +# resource "aws_s3_object" "extract_lambda_code" { +# bucket = aws_s3_bucket.lambda_code_bucket.bucket +# key = "${var.extract_lambda_name}/extract_function.zip" +# source = "${path.module}/../extract_function.zip" +# } # << can't figure out how this is being used but we seem to need it + +# resource "aws_s3_object" "transform_lambda_code" { +# bucket = aws_s3_bucket.lambda_code_bucket.bucket +# key = "${var.transform_lambda_name}/transform_function.zip" +# source = "${path.module}/../transform_function.zip" +# } # << can't figure out how this is being used but we seem to need it + +# resource "aws_s3_object" "load_lambda_code" { +# bucket = aws_s3_bucket.lambda_code_bucket.bucket +# key = "${var.load_lambda_name}/load_function.zip" +# source = "${path.module}/../load_function.zip" +# } \ No newline at end of file -- cgit v1.2.3 From 911b2a4ba08e59f46a53b5252a044a5853796f78 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 16:38:36 +0100 Subject: test(tf): attempt to solve ResourceConflictException --- terraform/events.tf | 35 ++++++++--------- terraform/iam.tf | 60 +++++++++++++++++++++-------- terraform/lambda.tf | 109 +++++++++++++++++++++++++--------------------------- terraform/main.tf | 26 ++++++------- terraform/s3.tf | 43 ++------------------- terraform/vars.tf | 26 ++++++------- 6 files changed, 139 insertions(+), 160 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 9fd89e4..d2e2eb5 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,23 +1,18 @@ -resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder - force_destroy = true -} - resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" schedule_expression = "rate(30 minutes)" - force_destroy = true - # depends_on = [ - # aws_cloudwatch_event_target.extract_lambda_cw_event] - -# event_pattern = jsonencode({ -# detail-type = -# "AWS Console Sign In via CloudTrail" -# ] -# }) + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + depends_on = [aws_lambda_permission.allow_eventbridge] } resource "aws_lambda_permission" "allow_eventbridge" { @@ -25,7 +20,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn } @@ -43,7 +38,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder lambda_function { - events = ["s3:ObjectCreated:*"] + events = ["s3:ObjectCreated:*"] lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } @@ -65,9 +60,9 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" { bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder lambda_function { - events = ["s3:ObjectCreated:*"] + events = ["s3:ObjectCreated:*"] lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_transform_bucket] -} \ No newline at end of file +} diff --git a/terraform/iam.tf b/terraform/iam.tf index cf4902a..6c6b4fc 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -16,9 +16,7 @@ resource "aws_iam_role" "multi_service_role" { Principal = { Service = [ "lambda.amazonaws.com", - "cloudwatch.amazonaws.com", - "events.amazonaws.com", - "s3.amazonaws.com" + "scheduler.amazonaws.com" ] } } @@ -57,22 +55,22 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { ######################################################################## resource "aws_iam_policy" "lambda_execution_policy" { - name = "lambda_execution_policy" - path = "/" + name = "lambda_execution_policy" + path = "/" description = "IAM policy for Lambda execution" policy = jsonencode({ Version = "2012-10-17" Statement = [ - { + { Effect = "Allow" Action = [ "lambda:InvokeFunction", "lambda:GetFunction" ] Resource = "*" - } - ] + } + ] } ) } @@ -87,7 +85,7 @@ data "aws_iam_policy_document" "cw_document" { actions = ["logs:CreateLogGroup"] resources = [ "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" - ] + ] } statement { @@ -95,15 +93,15 @@ data "aws_iam_policy_document" "cw_document" { "logs:CreateLogStream", "logs:CreateLogGroup", "logs:PutLogEvents" - ] - resources = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" - ] + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] } } resource "aws_iam_policy" "cw_policy" { - name = "cw_policy" + name = "cw_policy" policy = data.aws_iam_policy_document.cw_document.json } @@ -128,20 +126,48 @@ resource "aws_iam_policy" "s3_write_policy" { # } resource "aws_iam_role_policy_attachment" "s3_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.s3_write_policy.arn } resource "aws_iam_role_policy_attachment" "lambda_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.lambda_execution_policy.arn } resource "aws_iam_role_policy_attachment" "cw_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.cw_policy.arn } ################ # RDS POLICIES # ################ + +################### +# EVENTS POLICIES # +################### + +data "aws_iam_policy_document" "cloudwatch_events_policy" { + statement { + actions = [ + "events:PutRule", + "events:PutTargets", + "events:RemoveTargets", + "events:DeleteRule", + "events:PutEvents" + ] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "cloudwatch_events_policy" { + name = "cloudwatch_events_policy" + policy = data.aws_iam_policy_document.cloudwatch_events_policy.json +} + +resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.cloudwatch_events_policy.arn +} diff --git a/terraform/lambda.tf b/terraform/lambda.tf index a5da972..fb0a666 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,73 +1,68 @@ -### EXTRACT LAMBDA SET UP -data "archive_file" "extract_lambda_zip" { - type = "zip" - source_file = "${path.module}/../src/extract_lambda.py" - output_path = "${path.module}/../extract_function.zip" +# Extract Lambda Function +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" + etag = filemd5("${path.module}/../extract_function.zip") } resource "aws_lambda_function" "extract_lambda" { - function_name = "${var.extract_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "extract-lambda/extract_function.zip" - role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler - handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler - runtime = "python3.11" - environment { - variables = { - output = aws_s3_bucket.extract_bucket.bucket - } - } -} + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.extract" + runtime = "python3.11" -resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.extract_lambda.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract_bucket.arn -} + lifecycle { + create_before_destroy = true + } + depends_on = [aws_s3_object.extract_lambda_code] +} -### TRANSFORM LAMBDA SET UP -data "archive_file" "transform_lambda_zip" { - type = "zip" - source_file = "${path.module}/../src/transform_lambda.py" - output_path = "${path.module}/../transform_function.zip" +# Transform Lambda Function +resource "aws_s3_object" "transform_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" + etag = filemd5("${path.module}/../transform_function.zip") } resource "aws_lambda_function" "transform_lambda" { - function_name = "${var.transform_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "transform-lambda/transform_function.zip" - role = aws_iam_role.multi_service_role.arn # << lambda role placehodler - handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler - runtime = "python3.11" - environment { - variables = { - output = aws_s3_bucket.transform_bucket.bucket - } - } -} + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.transform" + runtime = "python3.11" -resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform_bucket.arn + lifecycle { + create_before_destroy = true + } + + depends_on = [aws_s3_object.transform_lambda_code] } -### LOAD LAMBDA SET UP -data "archive_file" "load_lambda_zip" { - type = "zip" - source_file = "${path.module}/../src/load_lambda.py" - output_path = "${path.module}/../load_function.zip" +# Load Lambda Function +resource "aws_s3_object" "load_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.load_lambda_name}/load_function.zip" + source = "${path.module}/../load_function.zip" + etag = filemd5("${path.module}/../load_function.zip") } resource "aws_lambda_function" "load_lambda" { - function_name = "${var.load_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "load-lambda/load_function.zip" - role = aws_iam_role.multi_service_role.arn # << lambda role placehodler - handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler - runtime = "python3.11" -} + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.load" + runtime = "python3.11" + lifecycle { + create_before_destroy = true + } + + depends_on = [aws_s3_object.load_lambda_code] +} diff --git a/terraform/main.tf b/terraform/main.tf index 2a048b9..3b06701 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,26 +1,26 @@ terraform { required_providers { aws = { - source = "hashicorp/aws" - version = "~>5.0" + source = "hashicorp/aws" + version = "~>5.0" } } backend "s3" { bucket = "bentley-project-secrets" - key = "bentley-project/terraform.tfstate" + key = "bentley-project/terraform.tfstate" region = "eu-west-2" } } provider "aws" { - region = "eu-west-2" - default_tags { - tags = { - ProjectName = "Terrific-Totes" - Team = "Team-Bentley" - Environment = "Dev" - GitHubRepo = "de-project-bentley" - ManagedBy = "Terraform" - } + region = "eu-west-2" + default_tags { + tags = { + ProjectName = "Terrific-Totes" + Team = "Team-Bentley" + Environment = "Dev" + GitHubRepo = "de-project-bentley" + ManagedBy = "Terraform" } -} \ No newline at end of file + } +} diff --git a/terraform/s3.tf b/terraform/s3.tf index 4c06b8e..d5cdee3 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,51 +1,14 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { - bucket_prefix = "${var.s3_extract_bucket_name}-" + bucket_prefix = "${var.s3_extract_bucket_name}-" } -# resource "aws_s3_bucket_notification" "extract_bucket_notification" { -# bucket = aws_s3_bucket.extract_bucket.id -# lambda_function { -# lambda_function_arn = aws_lambda_function.extract_lambda.arn -# events = ["s3:ObjectCreated:*"] -# } -# depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] -# } # << is this the correct permission dependency? - ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { - bucket_prefix = "${var.s3_transform_bucket_name}-" + bucket_prefix = "${var.s3_transform_bucket_name}-" } -# resource "aws_s3_bucket_notification" "transform_bucket_notification" { -# bucket = aws_s3_bucket.transform_bucket.id -# lambda_function { -# lambda_function_arn = aws_lambda_function.transform_lambda.arn -# events = ["s3:ObjectCreated:*"] -# } -# depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] -# } # << is this the correct permission dependency? - - ### LAMBDA BUCKET resource "aws_s3_bucket" "lambda_code_bucket" { - bucket_prefix = "${var.s3_code_bucket_name}-" + bucket_prefix = "${var.s3_code_bucket_name}-" } - -# resource "aws_s3_object" "extract_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.extract_lambda_name}/extract_function.zip" -# source = "${path.module}/../extract_function.zip" -# } # << can't figure out how this is being used but we seem to need it - -# resource "aws_s3_object" "transform_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.transform_lambda_name}/transform_function.zip" -# source = "${path.module}/../transform_function.zip" -# } # << can't figure out how this is being used but we seem to need it - -# resource "aws_s3_object" "load_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.load_lambda_name}/load_function.zip" -# source = "${path.module}/../load_function.zip" -# } \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf index cc9348a..84824ce 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -1,33 +1,33 @@ variable "s3_extract_bucket_name" { - type = string - default = "extract-bucket" + type = string + default = "extract-bucket" } variable "s3_transform_bucket_name" { - type = string - default = "transform-bucket" + type = string + default = "transform-bucket" } variable "s3_code_bucket_name" { - type = string - default = "lambda-bucket" + type = string + default = "lambda-bucket" } variable "extract_lambda_name" { - type = string - default = "extract-lambda" + type = string + default = "extract-lambda" } variable "transform_lambda_name" { - type = string - default = "transform-lambda" + type = string + default = "transform-lambda" } variable "load_lambda_name" { - type = string - default = "load-lambda" + type = string + default = "load-lambda" } data "aws_caller_identity" "current" {} -data "aws_region" "current" {} \ No newline at end of file +data "aws_region" "current" {} -- cgit v1.2.3 From 3f89444cb09f0372b6a7621913944e372acd826c Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 22:58:50 +0100 Subject: infra(tf): temporarily remove events policies --- terraform/iam.tf | 28 ---------------------------- 1 file changed, 28 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 6c6b4fc..20aeab3 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -143,31 +143,3 @@ resource "aws_iam_role_policy_attachment" "cw_attachment" { ################ # RDS POLICIES # ################ - -################### -# EVENTS POLICIES # -################### - -data "aws_iam_policy_document" "cloudwatch_events_policy" { - statement { - actions = [ - "events:PutRule", - "events:PutTargets", - "events:RemoveTargets", - "events:DeleteRule", - "events:PutEvents" - ] - resources = ["*"] - effect = "Allow" - } -} - -resource "aws_iam_policy" "cloudwatch_events_policy" { - name = "cloudwatch_events_policy" - policy = data.aws_iam_policy_document.cloudwatch_events_policy.json -} - -resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { - role = aws_iam_role.multi_service_role.name - policy_arn = aws_iam_policy.cloudwatch_events_policy.arn -} -- cgit v1.2.3 From 8b975ba92d10c8034bdb35fee19c42b6187ea24f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 23:03:14 +0100 Subject: chore(cleanup): remove commented out code --- terraform/iam.tf | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 20aeab3..195b07c 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -114,17 +114,6 @@ resource "aws_iam_policy" "s3_write_policy" { policy = data.aws_iam_policy_document.s3_data_policy_doc.json } -# S3 ATTACH POLICY -# resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { -# for_each = toset([ -# aws_iam_policy.s3_write_policy.arn, -# aws_iam_policy.lambda_execution_policy.arn, -# aws_iam_policy.cw_policy.arn -# ]) -# role = aws_iam_role.multi_service_role.name -# policy_arn = each.value -# } - resource "aws_iam_role_policy_attachment" "s3_attachment" { role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.s3_write_policy.arn -- cgit v1.2.3 From 8ce75bcc1a89000fc0a9fea9b78ad31cd316ca06 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 23:04:32 +0100 Subject: chore(cleanup): remove unused rds section --- terraform/iam.tf | 3 --- 1 file changed, 3 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 195b07c..acb98f4 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -129,6 +129,3 @@ resource "aws_iam_role_policy_attachment" "cw_attachment" { policy_arn = aws_iam_policy.cw_policy.arn } -################ -# RDS POLICIES # -################ -- cgit v1.2.3 From c5f840e96b8c1696a6ed506e9260c4f1c26db10d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 23:05:26 +0100 Subject: feat(tf): add cloudwatch events iam policies --- terraform/iam.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index acb98f4..0e5fa6d 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -129,3 +129,30 @@ resource "aws_iam_role_policy_attachment" "cw_attachment" { policy_arn = aws_iam_policy.cw_policy.arn } +################### +# EVENTS POLICIES # +################### + +data "aws_iam_policy_document" "cloudwatch_events_policy" { + statement { + actions = [ + "events:PutRule", + "events:PutTargets", + "events:RemoveTargets", + "events:DeleteRule", + "events:PutEvents" + ] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "cloudwatch_events_policy" { + name = "cloudwatch_events_policy" + policy = data.aws_iam_policy_document.cloudwatch_events_policy.json +} + +resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.cloudwatch_events_policy.arn +} -- cgit v1.2.3 From d78db8a0f99c778b1e4b401c7cb4a094a5f2b103 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 15 Aug 2024 00:24:41 +0100 Subject: fix(infra/tf): add unique id to permissions This was done to resolve issues with `ResourceConflictException` - add random_string resources for every lambda permission to add as suffixes - also add lifecycle options to force recreation whenever the random strings change --- terraform/events.tf | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index d2e2eb5..263141f 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,11 +1,25 @@ +resource "random_string" "eventbridge_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_ingestion_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_transform_suffix" { + length = 8 + special = false + upper = false +} + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" schedule_expression = "rate(30 minutes)" - - lifecycle { - create_before_destroy = true - } } resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { @@ -16,21 +30,28 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge" + statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn -} + lifecycle { + replace_triggered_by = [random_string.eventbridge_suffix] + } +} # below is step function 1 resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder + + lifecycle { + replace_triggered_by = [random_string.s3_ingestion_suffix] + } } @@ -45,14 +66,16 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { depends_on = [aws_lambda_permission.allow_s3_ingestion] } -###### - resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder + + lifecycle { + replace_triggered_by = [random_string.s3_transform_suffix] + } } -- cgit v1.2.3 From 79d230f3c76609d32a3d8c553d64f37ce9fe6d09 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 15 Aug 2024 10:15:31 +0100 Subject: fix(tf): create lambda zips before referencing --- terraform/lambda.tf | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index fb0a666..72d1306 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,9 +1,14 @@ # Extract Lambda Function +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} resource "aws_s3_object" "extract_lambda_code" { bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.extract_lambda_name}/extract_function.zip" - source = "${path.module}/../extract_function.zip" - etag = filemd5("${path.module}/../extract_function.zip") + source = data.archive_file.extract_lambda_zip.output_path + etag = filemd5(data.archive_file.extract_lambda_zip.output_path) } resource "aws_lambda_function" "extract_lambda" { @@ -22,11 +27,16 @@ resource "aws_lambda_function" "extract_lambda" { } # Transform Lambda Function +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} resource "aws_s3_object" "transform_lambda_code" { bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.transform_lambda_name}/transform_function.zip" - source = "${path.module}/../transform_function.zip" - etag = filemd5("${path.module}/../transform_function.zip") + source = data.archive_file.transform_lambda_zip.output_path + etag = filemd5(data.archive_file.transform_lambda_zip.output_path) } resource "aws_lambda_function" "transform_lambda" { @@ -45,11 +55,16 @@ resource "aws_lambda_function" "transform_lambda" { } # Load Lambda Function +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} resource "aws_s3_object" "load_lambda_code" { bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.load_lambda_name}/load_function.zip" - source = "${path.module}/../load_function.zip" - etag = filemd5("${path.module}/../load_function.zip") + source = data.archive_file.load_lambda_zip.output_path + etag = filemd5(data.archive_file.load_lambda_zip.output_path) } resource "aws_lambda_function" "load_lambda" { -- cgit v1.2.3 From 486fb62af5568a70e22ded622072883758e9ffdf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 15 Aug 2024 17:25:03 +0100 Subject: fix(tf): resolve naming issues with resources --- terraform/rds.tf | 42 ++++++++++++++++++++++-------------------- terraform/vars.tf | 4 ++-- 2 files changed, 24 insertions(+), 22 deletions(-) (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf index 4b25c5f..88783b7 100644 --- a/terraform/rds.tf +++ b/terraform/rds.tf @@ -2,9 +2,9 @@ data "aws_availability_zones" "available" {} module "vpc" { source = "terraform-aws-modules/vpc/aws" - version = "2.77.0" + version = "5.12.1" - name = "${var.project_name}" + name = var.project_name cidr = "10.0.0.0/16" azs = data.aws_availability_zones.available.names public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] @@ -13,7 +13,7 @@ module "vpc" { } resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { - name = "TT-db-subnet" + name = "tt-db-subnet" subnet_ids = module.vpc.public_subnets tags = { @@ -45,7 +45,7 @@ resource "aws_security_group" "rds" { } resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { - name = "TT-db-param" + name = "tt-db-param" family = "postgres14" parameter { @@ -54,25 +54,27 @@ resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { } } -resource "aws_db_instance" "Terrific-Totes-rds" { - db_name = "${var.project_name}" - instance_class = "db.t3.micro" - allocated_storage = 5 - engine = "postgres" - engine_version = "14.1" - username = "user credentials for the root user" # we could use .env here - password = "user password for the root user" # we could use .env here +resource "aws_db_instance" "terrific-totes-rds" { + db_name = var.project_name + instance_class = "db.t3.micro" + allocated_storage = 5 + engine = "postgres" + engine_version = "14.10" + username = "totes" + password = "totes123" + # username = "user credentials for the root user" # we could use .env here + # password = "user password for the root user" # we could use .env here ### alternatively to providing username nad password we can specify: -# resource "aws_kms_key" "example_key" { -# description = "Example KMS Key" -# } -# within the resource: -# manage_master_user_password = true -# master_user_secret_kms_key_id = aws_kms_key.example.key_id -# } + # resource "aws_kms_key" "example_key" { + # description = "Example KMS Key" + # } + # within the resource: + # manage_master_user_password = true + # master_user_secret_kms_key_id = aws_kms_key.example.key_id + # } db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name vpc_security_group_ids = [aws_security_group.rds.id] parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name publicly_accessible = false skip_final_snapshot = true -} \ No newline at end of file +} diff --git a/terraform/vars.tf b/terraform/vars.tf index d5cdafb..3c88731 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -29,8 +29,8 @@ variable "load_lambda_name" { } variable "project_name" { - type = string - default = "Terrific-Totes" + type = string + default = "tt" } data "aws_caller_identity" "current" {} -- cgit v1.2.3 From 0c42e8f165e0f98a6c16252e841432922467ef94 Mon Sep 17 00:00:00 2001 From: Ellie Date: Fri, 16 Aug 2024 10:51:16 +0100 Subject: add lambda layer --- terraform/lambda.tf | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 72d1306..658b8c8 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -81,3 +81,38 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } + +locals { + layer_dir = "${path.module}/../python" + requirements = "${path.module}/../requirements.txt" + layer_zip = "${path.module}/../layer.zip" +} + +resource "null_resource" "prepare_layer" { + triggers = { + requirements_hash = filesha1(local.requirements) + } + provisioner "local-exec" { + command = < Date: Fri, 16 Aug 2024 12:19:54 +0100 Subject: comment out rds.tf to increases tf speed --- terraform/rds.tf | 138 +++++++++++++++++++++++++++---------------------------- 1 file changed, 69 insertions(+), 69 deletions(-) (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf index 88783b7..d1b4959 100644 --- a/terraform/rds.tf +++ b/terraform/rds.tf @@ -1,80 +1,80 @@ -data "aws_availability_zones" "available" {} +# data "aws_availability_zones" "available" {} -module "vpc" { - source = "terraform-aws-modules/vpc/aws" - version = "5.12.1" +# module "vpc" { +# source = "terraform-aws-modules/vpc/aws" +# version = "5.12.1" - name = var.project_name - cidr = "10.0.0.0/16" - azs = data.aws_availability_zones.available.names - public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] - enable_dns_hostnames = true - enable_dns_support = true -} +# name = var.project_name +# cidr = "10.0.0.0/16" +# azs = data.aws_availability_zones.available.names +# public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] +# enable_dns_hostnames = true +# enable_dns_support = true +# } -resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { - name = "tt-db-subnet" - subnet_ids = module.vpc.public_subnets +# resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { +# name = "tt-db-subnet" +# subnet_ids = module.vpc.public_subnets - tags = { - Name = "${var.project_name}" - } -} +# tags = { +# Name = "${var.project_name}" +# } +# } -resource "aws_security_group" "rds" { - name = "${var.project_name}-rds" - vpc_id = module.vpc.vpc_id +# resource "aws_security_group" "rds" { +# name = "${var.project_name}-rds" +# vpc_id = module.vpc.vpc_id - ingress { - from_port = 5432 - to_port = 5432 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } +# ingress { +# from_port = 5432 +# to_port = 5432 +# protocol = "tcp" +# cidr_blocks = ["0.0.0.0/0"] +# } - egress { - from_port = 5432 - to_port = 5432 - protocol = "tcp" - cidr_blocks = ["0.0.0.0/0"] - } +# egress { +# from_port = 5432 +# to_port = 5432 +# protocol = "tcp" +# cidr_blocks = ["0.0.0.0/0"] +# } - tags = { - Name = "${var.project_name}-rds" - } -} +# tags = { +# Name = "${var.project_name}-rds" +# } +# } -resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { - name = "tt-db-param" - family = "postgres14" +# resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { +# name = "tt-db-param" +# family = "postgres14" - parameter { - name = "log_connections" - value = "1" - } -} +# parameter { +# name = "log_connections" +# value = "1" +# } +# } -resource "aws_db_instance" "terrific-totes-rds" { - db_name = var.project_name - instance_class = "db.t3.micro" - allocated_storage = 5 - engine = "postgres" - engine_version = "14.10" - username = "totes" - password = "totes123" - # username = "user credentials for the root user" # we could use .env here - # password = "user password for the root user" # we could use .env here - ### alternatively to providing username nad password we can specify: - # resource "aws_kms_key" "example_key" { - # description = "Example KMS Key" - # } - # within the resource: - # manage_master_user_password = true - # master_user_secret_kms_key_id = aws_kms_key.example.key_id - # } - db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name - vpc_security_group_ids = [aws_security_group.rds.id] - parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name - publicly_accessible = false - skip_final_snapshot = true -} +# resource "aws_db_instance" "terrific-totes-rds" { +# db_name = var.project_name +# instance_class = "db.t3.micro" +# allocated_storage = 5 +# engine = "postgres" +# engine_version = "14.10" +# username = "totes" +# password = "totes123" +# # username = "user credentials for the root user" # we could use .env here +# # password = "user password for the root user" # we could use .env here +# ### alternatively to providing username nad password we can specify: +# # resource "aws_kms_key" "example_key" { +# # description = "Example KMS Key" +# # } +# # within the resource: +# # manage_master_user_password = true +# # master_user_secret_kms_key_id = aws_kms_key.example.key_id +# # } +# db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name +# vpc_security_group_ids = [aws_security_group.rds.id] +# parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name +# publicly_accessible = false +# skip_final_snapshot = true +# } -- cgit v1.2.3 From 39a33cecb5e19f15bed4a099b02bdba56c80c073 Mon Sep 17 00:00:00 2001 From: HastarTara Date: Fri, 16 Aug 2024 16:05:03 +0100 Subject: infra[tf] update lambda extract doesnt work yet --- terraform/lambda.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 658b8c8..71ddd11 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -83,7 +83,7 @@ resource "aws_lambda_function" "load_lambda" { } locals { - layer_dir = "${path.module}/../python" + layer_dir = "${path.module}/.." requirements = "${path.module}/../requirements.txt" layer_zip = "${path.module}/../layer.zip" } @@ -94,9 +94,9 @@ resource "null_resource" "prepare_layer" { } provisioner "local-exec" { command = < Date: Fri, 16 Aug 2024 16:23:56 +0100 Subject: infra(tf): add version constraints for null and archive --- terraform/main.tf | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 3b06701..310a251 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -4,6 +4,14 @@ terraform { source = "hashicorp/aws" version = "~>5.0" } + null = { + source = "hashicorp/null" + version = "~>3.2.2" + } + archive = { + source = "hashicorp/archive" + version = "~>2.5.0" + } } backend "s3" { bucket = "bentley-project-secrets" -- cgit v1.2.3 From 303725f83cf5551b3d165aa02ce81562de488a01 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 16:24:44 +0100 Subject: infra(tf): re-add code that creates layer zip --- terraform/lambda.tf | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 71ddd11..67fd6eb 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -83,9 +83,9 @@ resource "aws_lambda_function" "load_lambda" { } locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "${path.module}/.." + requirements = "${path.module}/../requirements.txt" + layer_zip = "${path.module}/../layer.zip" } resource "null_resource" "prepare_layer" { @@ -96,23 +96,23 @@ resource "null_resource" "prepare_layer" { command = < Date: Fri, 16 Aug 2024 16:28:04 +0100 Subject: chore(tf): remove dummy username/password sorry hackers! --- terraform/rds.tf | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf index d1b4959..a013fb3 100644 --- a/terraform/rds.tf +++ b/terraform/rds.tf @@ -60,18 +60,8 @@ # allocated_storage = 5 # engine = "postgres" # engine_version = "14.10" -# username = "totes" -# password = "totes123" -# # username = "user credentials for the root user" # we could use .env here -# # password = "user password for the root user" # we could use .env here -# ### alternatively to providing username nad password we can specify: -# # resource "aws_kms_key" "example_key" { -# # description = "Example KMS Key" -# # } -# # within the resource: -# # manage_master_user_password = true -# # master_user_secret_kms_key_id = aws_kms_key.example.key_id -# # } +# username = "" +# password = "" # db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name # vpc_security_group_ids = [aws_security_group.rds.id] # parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name -- cgit v1.2.3 From 1e27974ecc48d8611b87af1b9cd51e29afa8c792 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 17:15:59 +0100 Subject: test(fx): fix prepare_layer - broken --- terraform/lambda.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 67fd6eb..27e6266 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -89,14 +89,13 @@ locals { } resource "null_resource" "prepare_layer" { - triggers = { - requirements_hash = filesha1(local.requirements) - } provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 11:21:58 +0100 Subject: layers block update, function resources to inlcude attributes: layers, correct handler and source_code_hash --- terraform/lambda.tf | 70 +++++++++++++++++++++++++++++------------------------ terraform/s3.tf | 5 ++++ 2 files changed, 44 insertions(+), 31 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 27e6266..e33bc79 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -12,12 +12,14 @@ resource "aws_s3_object" "extract_lambda_code" { } resource "aws_lambda_function" "extract_lambda" { - function_name = var.extract_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.extract_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "extract_lambda.extract" - runtime = "python3.11" + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -40,12 +42,14 @@ resource "aws_s3_object" "transform_lambda_code" { } resource "aws_lambda_function" "transform_lambda" { - function_name = var.transform_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.transform_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "transform_lambda.transform" - runtime = "python3.11" + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -68,12 +72,14 @@ resource "aws_s3_object" "load_lambda_code" { } resource "aws_lambda_function" "load_lambda" { - function_name = var.load_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.load_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "load_lambda.load" - runtime = "python3.11" + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -82,10 +88,12 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } +# Lambda Layer Specification locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "lambda_layer" + requirements = "requirements.txt" + layer_zip = "layer.zip" + layer_name = "lambda_layer_dev" } resource "null_resource" "prepare_layer" { @@ -95,23 +103,23 @@ resource "null_resource" "prepare_layer" { rm -rf python mkdir python pip3 install -r ${local.requirements} -t python/ - zip -r ${local.layer_zip} python/ - EOT - } + zip -r ${local.layer_zip} python + EOT + } #removed / at the end of python in line 99 } -resource "aws_s3_object" "layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "layer.zip" +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "lambda_layer/${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = "lambda_layer" + layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.layer_zip.key + s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true - depends_on = [aws_s3_object.layer_zip] + depends_on = [aws_s3_object.lambda_layer_zip] } diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..b3a863c 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,3 +12,8 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } + +### LAMBDA LAYER BUCKET +resource "aws_s3_bucket" "lambda_layer_bucket" { + bucket_prefix = "lambda-layer-dev-" +} \ No newline at end of file -- cgit v1.2.3 From 57d1e1ee5a13269f1bef6c3b754cb8374a657202 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 14:55:39 +0100 Subject: style: remove redundant comment --- terraform/lambda.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index e33bc79..714ffa5 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -99,13 +99,9 @@ locals { resource "null_resource" "prepare_layer" { provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 15:02:39 +0100 Subject: infra(tf): modify variables & remove past zip creation --- terraform/lambda.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 714ffa5..986170f 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,17 +90,16 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "lambda_layer" + layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" layer_name = "lambda_layer_dev" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { provisioner "local-exec" { - command = < Date: Mon, 19 Aug 2024 15:06:35 +0100 Subject: infra(tf): remove lambda layer dev reference --- terraform/s3.tf | 5 ----- 1 file changed, 5 deletions(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index b3a863c..d5cdee3 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,8 +12,3 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } - -### LAMBDA LAYER BUCKET -resource "aws_s3_bucket" "lambda_layer_bucket" { - bucket_prefix = "lambda-layer-dev-" -} \ No newline at end of file -- cgit v1.2.3 From 284a52df866c34d925b85ccd4f06d6141e67ce70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:12:56 +0100 Subject: fix(tf): correct layer.zip output path --- terraform/lambda.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 986170f..8a4207d 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -93,7 +93,7 @@ locals { layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" - layer_name = "lambda_layer_dev" + layer_name = "lambda_layer" script_dir = "../scripts" } @@ -105,7 +105,7 @@ resource "null_resource" "prepare_layer" { resource "aws_s3_object" "lambda_layer_zip" { bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "lambda_layer/${local.layer_name}/${local.layer_zip}" + key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_bucket = aws_s3_bucket.lambda_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From cbf1d083dc0bf4d78da83cb169da49731f8ace65 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:18:22 +0100 Subject: fix(tf): correct s3_bucket value for lambda_layer --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 8a4207d..bf96747 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From 4b3b80a2f2177456ed6c2857a7ae0987d7304360 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:40:01 +0100 Subject: chore(tf): remove unused requirements variable --- terraform/lambda.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bf96747..72aae04 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,11 +90,10 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "../" - requirements = "requirements.txt" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { -- cgit v1.2.3 From 8b4e78b781617f68554efebcda75d982a382f650 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 16:31:50 +0100 Subject: fix(tf): fix permissions for bucket/object access --- terraform/iam.tf | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 0e5fa6d..7585ff8 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -28,17 +28,19 @@ resource "aws_iam_role" "multi_service_role" { ######################################################################## # S3 SETUP # Description: allows allows retention/tagging/access control settings -# Lambda IAM Policy for S3 Write +# Lambda IAM Policy for S3 ######################################################################## # S3 DEFINE POLICY data "aws_iam_policy_document" "s3_data_policy_doc" { statement { + effect = "Allow" actions = [ "s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging", - "s3:PutObjectAcl" + "s3:PutObjectAcl", + "s3:ListObjects" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -46,6 +48,17 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "${aws_s3_bucket.lambda_code_bucket.arn}/*", ] } + + statement { + effect = "Allow" + actions = [ + "s3:ListBuckets", + "s3:ListAllMyBuckets" + ] + resources = [ + "arn:aws:s3:::*", + ] + } } -- cgit v1.2.3 From e537bdef11d1d518d4df1c057f3624e3fe6da24d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:05:41 +0100 Subject: infra(tf): remove rds.tf --- terraform/rds.tf | 70 -------------------------------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 terraform/rds.tf (limited to 'terraform') diff --git a/terraform/rds.tf b/terraform/rds.tf deleted file mode 100644 index a013fb3..0000000 --- a/terraform/rds.tf +++ /dev/null @@ -1,70 +0,0 @@ -# data "aws_availability_zones" "available" {} - -# module "vpc" { -# source = "terraform-aws-modules/vpc/aws" -# version = "5.12.1" - -# name = var.project_name -# cidr = "10.0.0.0/16" -# azs = data.aws_availability_zones.available.names -# public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] -# enable_dns_hostnames = true -# enable_dns_support = true -# } - -# resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { -# name = "tt-db-subnet" -# subnet_ids = module.vpc.public_subnets - -# tags = { -# Name = "${var.project_name}" -# } -# } - -# resource "aws_security_group" "rds" { -# name = "${var.project_name}-rds" -# vpc_id = module.vpc.vpc_id - -# ingress { -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# } - -# egress { -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# } - -# tags = { -# Name = "${var.project_name}-rds" -# } -# } - -# resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { -# name = "tt-db-param" -# family = "postgres14" - -# parameter { -# name = "log_connections" -# value = "1" -# } -# } - -# resource "aws_db_instance" "terrific-totes-rds" { -# db_name = var.project_name -# instance_class = "db.t3.micro" -# allocated_storage = 5 -# engine = "postgres" -# engine_version = "14.10" -# username = "" -# password = "" -# db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name -# vpc_security_group_ids = [aws_security_group.rds.id] -# parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name -# publicly_accessible = false -# skip_final_snapshot = true -# } -- cgit v1.2.3 From 56b2c376a925132f3bf2c7e6cad4911400955129 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:07:30 +0100 Subject: infra(tf): enforce version constraint on terraform --- terraform/main.tf | 1 + 1 file changed, 1 insertion(+) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 310a251..206fc74 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,4 +1,5 @@ terraform { + required_version = ">= 1.8.0" required_providers { aws = { source = "hashicorp/aws" -- cgit v1.2.3 From 35bf4e8668309cb28175ef0224a6bce453abb47f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:19:44 +0100 Subject: chore(tf): replace static tag values in main.tf with variables --- terraform/main.tf | 8 ++++---- terraform/vars.tf | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 206fc74..5ccbec2 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -25,11 +25,11 @@ provider "aws" { region = "eu-west-2" default_tags { tags = { - ProjectName = "Terrific-Totes" - Team = "Team-Bentley" - Environment = "Dev" - GitHubRepo = "de-project-bentley" + ProjectName = var.project_name + Environment = var.environment ManagedBy = "Terraform" + GitHubRepo = var.github_repo + Team = var.team_name } } } diff --git a/terraform/vars.tf b/terraform/vars.tf index 3c88731..1adbcf7 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -33,6 +33,26 @@ variable "project_name" { default = "tt" } +variable "aws_region" { + type = string + default = "eu-west-2" +} + +variable "environment" { + type = string + default = "dev" +} + +variable "github_repo" { + type = string + default = "de-project-bentley" +} + +variable "team_name" { + type = string + default = "Team-Bentley" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} -- cgit v1.2.3 From 95c4fe80aea75a9a63b1cfd85abadaab6b96b876 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:24:23 +0100 Subject: infra(tf): add state file encryption --- terraform/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 5ccbec2..33c760c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -15,9 +15,10 @@ terraform { } } backend "s3" { - bucket = "bentley-project-secrets" - key = "bentley-project/terraform.tfstate" - region = "eu-west-2" + bucket = "bentley-project-secrets" + key = "bentley-project/terraform.tfstate" + region = "eu-west-2" + encrypt = true } } -- cgit v1.2.3 From 57e855a797f225cd77401e85a671cde95e07ee70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:55:19 +0100 Subject: style(tf): improve legibility of lambda.tf sections --- terraform/lambda.tf | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 72aae04..b7b362b 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,4 +1,7 @@ -# Extract Lambda Function +########################### +# Extract Lambda Function # +########################### + data "archive_file" "extract_lambda_zip" { type = "zip" source_file = "${path.module}/../src/extract_lambda.py" @@ -28,7 +31,10 @@ resource "aws_lambda_function" "extract_lambda" { depends_on = [aws_s3_object.extract_lambda_code] } -# Transform Lambda Function +############################# +# Transform Lambda Function # +############################# + data "archive_file" "transform_lambda_zip" { type = "zip" source_file = "${path.module}/../src/transform_lambda.py" @@ -58,7 +64,10 @@ resource "aws_lambda_function" "transform_lambda" { depends_on = [aws_s3_object.transform_lambda_code] } -# Load Lambda Function +######################## +# Load Lambda Function # +######################## + data "archive_file" "load_lambda_zip" { type = "zip" source_file = "${path.module}/../src/load_lambda.py" @@ -88,7 +97,10 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -# Lambda Layer Specification +###################### +# Lambda Layer Setup # +###################### + locals { layer_dir = "../" layer_zip = "layer.zip" -- cgit v1.2.3 From d0e7b1304efe4ab6de2dc5bef1691b389a5bc449 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:58:46 +0100 Subject: refactor(tf): move sections in lambda.tf for better readability --- terraform/lambda.tf | 69 ++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b7b362b..aa730c1 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,3 +1,40 @@ +#################### +# Common Variables # +#################### + +locals { + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" +} + +###################### +# Lambda Layer Setup # +###################### + +resource "null_resource" "prepare_layer" { + provisioner "local-exec" { + command = "bash ${local.script_dir}/make_layer_zip.sh" + } +} + +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "${local.layer_name}/${local.layer_zip}" + source = "${local.layer_dir}/${local.layer_zip}" + depends_on = [null_resource.prepare_layer] +} + +resource "aws_lambda_layer_version" "lambda_layer" { + layer_name = local.layer_name + compatible_runtimes = ["python3.11"] + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.lambda_layer_zip.key + skip_destroy = true + depends_on = [aws_s3_object.lambda_layer_zip] +} + ########################### # Extract Lambda Function # ########################### @@ -97,35 +134,3 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -###################### -# Lambda Layer Setup # -###################### - -locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" -} - -resource "null_resource" "prepare_layer" { - provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" - } -} - -resource "aws_s3_object" "lambda_layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "${local.layer_name}/${local.layer_zip}" - source = "${local.layer_dir}/${local.layer_zip}" - depends_on = [null_resource.prepare_layer] -} - -resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = local.layer_name - compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.lambda_layer_zip.key - skip_destroy = true - depends_on = [aws_s3_object.lambda_layer_zip] -} -- cgit v1.2.3 From cfd6b462a874da77ada8facb3b2a3c0e85059fa4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:07:10 +0100 Subject: infra(tf): only create layer zip if doesn't exist --- terraform/lambda.tf | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index aa730c1..b1e0d8e 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -3,10 +3,11 @@ #################### locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" + layer_zip_path = "${local.layer_dir}/${local.layer_zip}" } ###################### @@ -14,8 +15,13 @@ locals { ###################### resource "null_resource" "prepare_layer" { + + triggers = { + layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" + } + provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" + command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi" } } -- cgit v1.2.3 From f035f60c7ece05b70275760238c5513b8f113310 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:09:26 +0100 Subject: docs(tf): add information about layer zip creation --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b1e0d8e..fc10431 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -16,6 +16,8 @@ locals { resource "null_resource" "prepare_layer" { + # New change: only run the script if the layer zip does not exist + triggers = { layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" } -- cgit v1.2.3 From 40c2952e628a92e63b3468be4d49f44a234cacce Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:13:03 +0100 Subject: infra(tf): add md5/source_code_hash checks for lambda layer --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) (limited to 'terraform') diff --git a/terraform/lambda.tf b/terraform/lambda.tf index fc10431..f8e7515 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -32,6 +32,7 @@ resource "aws_s3_object" "lambda_layer_zip" { key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] + etag = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null } resource "aws_lambda_layer_version" "lambda_layer" { @@ -39,6 +40,7 @@ resource "aws_lambda_layer_version" "lambda_layer" { compatible_runtimes = ["python3.11"] s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key + source_code_hash = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] } -- cgit v1.2.3 From e5715bc33d4470ceccb17c6853c3e52d4b1035d3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:20:09 +0100 Subject: chore(tf): add tags to s3 buckets --- terraform/s3.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..97910c8 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,14 +1,24 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" + + tags = { + Name = "Ingestion Bucket" + } } ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" + tags = { + Name = "Transform Bucket" + } } ### LAMBDA BUCKET resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" + tags = { + Name = "Load Bucket" + } } -- cgit v1.2.3 From 507b3071633fccc9aa1411880dd984ca346a141b Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:22:29 +0100 Subject: docs(tf): improve legibility of s3.tf sections --- terraform/s3.tf | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index 97910c8..6ff58fd 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,4 +1,7 @@ -### EXTRACT BUCKET SET-UP +######################## +# EXTRACT BUCKET SETUP # +######################## + resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" @@ -7,7 +10,10 @@ resource "aws_s3_bucket" "extract_bucket" { } } -### TRANSFORM BUCKET SET-UP +########################## +# TRANSFORM BUCKET SETUP # +########################## + resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" tags = { @@ -15,7 +21,10 @@ resource "aws_s3_bucket" "transform_bucket" { } } -### LAMBDA BUCKET +####################### +# LAMBDA BUCKET SETUP # +####################### + resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" tags = { -- cgit v1.2.3 From 1cb84bd663261c416a516b0dc59dbf8d62c4c1a7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:23:07 +0100 Subject: docs(tf): correct lambda bucket name tag --- terraform/s3.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index 6ff58fd..3e0f5b7 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -28,6 +28,6 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" tags = { - Name = "Load Bucket" + Name = "Lambda Bucket" } } -- cgit v1.2.3 From 795c7c2917c2780e8ffdf0716cbedf3426dcbd5e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:25:21 +0100 Subject: infra(tf): experimental - add versioning to protect against accidental deletes/overwrites" --- terraform/s3.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index 3e0f5b7..edfe0a0 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -10,6 +10,13 @@ resource "aws_s3_bucket" "extract_bucket" { } } +resource "aws_s3_bucket_versioning" "extract_bucket_versioning" { + bucket = aws_s3_bucket.extract_bucket.id + versioning_configuration { + status = "Enabled" + } +} + ########################## # TRANSFORM BUCKET SETUP # ########################## @@ -21,6 +28,14 @@ resource "aws_s3_bucket" "transform_bucket" { } } + +resource "aws_s3_bucket_versioning" "transform_bucket_versioning" { + bucket = aws_s3_bucket.transform_bucket.id + versioning_configuration { + status = "Enabled" + } +} + ####################### # LAMBDA BUCKET SETUP # ####################### -- cgit v1.2.3 From 1bbc12702a8fa6d5139440c9d04e5bfabd96581d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:32:32 +0100 Subject: infra(tf): add versioning to lambda_code_bucket --- terraform/s3.tf | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index edfe0a0..d17a4fe 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -46,3 +46,10 @@ resource "aws_s3_bucket" "lambda_code_bucket" { Name = "Lambda Bucket" } } + +resource "aws_s3_bucket_versioning" "lambda_bucket_versioning" { + bucket = aws_s3_bucket.lambda_code_bucket.id + versioning_configuration { + status = "Enabled" + } +} -- cgit v1.2.3 From b9a3d9dbaa1eedc25d5f8d12bd2be1a8a3841b42 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:39:05 +0100 Subject: docs(tf): improve legibility of events.tf sections --- terraform/events.tf | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 263141f..c2efbbc 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,3 +1,7 @@ +################# +# Random String # +################# + resource "random_string" "eventbridge_suffix" { length = 8 special = false @@ -16,6 +20,10 @@ resource "random_string" "s3_transform_suffix" { upper = false } +############################# +# EventBridge Configuration # +############################# + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" @@ -41,7 +49,10 @@ resource "aws_lambda_permission" "allow_eventbridge" { } } -# below is step function 1 +######################################## +# S3 Extract Bucket Notification Setup # +######################################## + resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" @@ -66,6 +77,10 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { depends_on = [aws_lambda_permission.allow_s3_ingestion] } +########################################## +# S3 Transform Bucket Notification Setup # +########################################## + resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" -- cgit v1.2.3 From 09b8010a453c99164540981060177fdd2280df7e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:41:31 +0100 Subject: infra(tf): remove repetitive suffix resources in events.tf --- terraform/events.tf | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index c2efbbc..832a965 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -2,19 +2,7 @@ # Random String # ################# -resource "random_string" "eventbridge_suffix" { - length = 8 - special = false - upper = false -} - -resource "random_string" "s3_ingestion_suffix" { - length = 8 - special = false - upper = false -} - -resource "random_string" "s3_transform_suffix" { +resource "random_string" "suffix" { length = 8 special = false upper = false @@ -38,14 +26,14 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" + statement_id = "AllowExecutionFromEventBridge${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn lifecycle { - replace_triggered_by = [random_string.eventbridge_suffix] + replace_triggered_by = [random_string.suffix] } } @@ -54,14 +42,14 @@ resource "aws_lambda_permission" "allow_eventbridge" { ######################################## resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder lifecycle { - replace_triggered_by = [random_string.s3_ingestion_suffix] + replace_triggered_by = [random_string.suffix] } } @@ -82,14 +70,14 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ########################################## resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder lifecycle { - replace_triggered_by = [random_string.s3_transform_suffix] + replace_triggered_by = [random_string.suffix] } } -- cgit v1.2.3 From 367100c2d118847a775f4eba87a8c9033c872cb9 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:44:13 +0100 Subject: docs(tf): remove redundant comments --- terraform/events.tf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 832a965..0113f5f 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + arn = aws_lambda_function.extract_lambda.arn depends_on = [aws_lambda_permission.allow_eventbridge] } @@ -44,9 +44,9 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder + source_arn = aws_s3_bucket.extract_bucket.arn lifecycle { replace_triggered_by = [random_string.suffix] @@ -55,11 +55,11 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder + bucket = aws_s3_bucket.extract_bucket.id lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + lambda_function_arn = aws_lambda_function.transform_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -72,9 +72,9 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder + source_arn = aws_s3_bucket.transform_bucket.arn lifecycle { replace_triggered_by = [random_string.suffix] @@ -83,11 +83,11 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder + bucket = aws_s3_bucket.transform_bucket.id lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + lambda_function_arn = aws_lambda_function.transform_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_transform_bucket] -- cgit v1.2.3 From a9fb82f5c96e0ba98d6d3453ce900f2ca22157ec Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:48:00 +0100 Subject: infra(tf): remove unused declaration in vars.tf --- terraform/vars.tf | 5 ----- 1 file changed, 5 deletions(-) (limited to 'terraform') diff --git a/terraform/vars.tf b/terraform/vars.tf index 1adbcf7..b3e3e47 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -33,11 +33,6 @@ variable "project_name" { default = "tt" } -variable "aws_region" { - type = string - default = "eu-west-2" -} - variable "environment" { type = string default = "dev" -- cgit v1.2.3 From c091506dc8e01741f54f9a8d289515c8d5ffbecf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:49:00 +0100 Subject: infra(tf): add version constraint for random in main.tf --- terraform/main.tf | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index 33c760c..ad7b335 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,6 +13,10 @@ terraform { source = "hashicorp/archive" version = "~>2.5.0" } + random_string = { + source = "hashicorp/random" + version = "~>3.6.2" + } } backend "s3" { bucket = "bentley-project-secrets" -- cgit v1.2.3 From ce2761b311523a118cdead885ba7fcf1f7a4cd68 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:52:15 +0100 Subject: fix(tf): correct random_string to random in main.tf --- terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/main.tf b/terraform/main.tf index ad7b335..6577b70 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,7 +13,7 @@ terraform { source = "hashicorp/archive" version = "~>2.5.0" } - random_string = { + random = { source = "hashicorp/random" version = "~>3.6.2" } -- cgit v1.2.3 From a8ce060732ed3064696f2d6c5459ffa176fd02f7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:02:56 +0100 Subject: fix(tf): lambda permissions should be created before destroyed --- terraform/events.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 0113f5f..9885a86 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -33,7 +33,8 @@ resource "aws_lambda_permission" "allow_eventbridge" { source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } @@ -49,7 +50,8 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { source_arn = aws_s3_bucket.extract_bucket.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } @@ -77,7 +79,8 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { source_arn = aws_s3_bucket.transform_bucket.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } -- cgit v1.2.3 From b8574d4c4bf262a8034d21b770fd4287022c2648 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:07:28 +0100 Subject: fix(tf): re-add separate random_string suffixes in events.tf --- terraform/events.tf | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) (limited to 'terraform') diff --git a/terraform/events.tf b/terraform/events.tf index 9885a86..53ae10a 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -2,7 +2,19 @@ # Random String # ################# -resource "random_string" "suffix" { +resource "random_string" "eventbridge_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_ingestion_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_transform_suffix" { length = 8 special = false upper = false @@ -26,7 +38,7 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge${random_string.suffix.result}" + statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" @@ -34,7 +46,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.eventbridge_suffix] } } @@ -43,7 +55,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { ######################################## resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" @@ -51,7 +63,7 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.s3_ingestion_suffix] } } @@ -72,7 +84,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ########################################## resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" @@ -80,7 +92,7 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.s3_transform_suffix] } } -- cgit v1.2.3 From 746f4206b2f30126c3c09ac11a2d49be3259fe6f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:42:54 +0100 Subject: infra(tf): add secrets manager permissions I feel like what I've done is bad but we'll find out soon. --- terraform/iam.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 7585ff8..a36cfdf 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -169,3 +169,30 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.cloudwatch_events_policy.arn } + +######################### +# SECRETS MANAGER SETUP # +######################### + +# Policy Doc +data "aws_iam_policy_document" "secrets_manager_policy_doc" { + statement { + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue" + ] + resources = [] + } +} + +# SM Policy Resource +resource "aws_iam_policy" "secrets_manager_policy" { + name = "secrets_manager_policy" + policy = data.aws_iam_policy_document.secrets_manager_policy_doc.json +} + +# Attach SM Policy to Role +resource "aws_iam_role_policy_attachment" "secrets_manager_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.secrets_manager_policy.arn +} -- cgit v1.2.3 From 2045888e1ae497444c58347096547f0475bba7a1 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:51:11 +0100 Subject: infra(tf): add resource access for secrets-manager policy doc --- terraform/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index a36cfdf..a8054ca 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -181,7 +181,7 @@ data "aws_iam_policy_document" "secrets_manager_policy_doc" { actions = [ "secretsmanager:GetSecretValue" ] - resources = [] + resources = ["arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:bentley-secrets-Na0yc8"] } } -- cgit v1.2.3 From ae57535d9f201d6fd749d4286551884d3c86fd60 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 10:26:48 +0100 Subject: infra(tf): add missing ListObjectsV2 permission --- terraform/iam.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index a8054ca..3ac8c45 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -40,7 +40,8 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectRetention", "s3:PutObjectTagging", "s3:PutObjectAcl", - "s3:ListObjects" + "s3:ListObjects", + "s3:ListObjectsV2" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", -- cgit v1.2.3 From 346aadfbf2208a0660ffc09959a91fc2f7b48c79 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 12:07:17 +0100 Subject: infra(tf): force-destroy buckets --- terraform/s3.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'terraform') diff --git a/terraform/s3.tf b/terraform/s3.tf index d17a4fe..14e8835 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -4,7 +4,7 @@ resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" - + force_destroy = true tags = { Name = "Ingestion Bucket" } @@ -23,6 +23,7 @@ resource "aws_s3_bucket_versioning" "extract_bucket_versioning" { resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" + force_destroy = true tags = { Name = "Transform Bucket" } @@ -42,6 +43,7 @@ resource "aws_s3_bucket_versioning" "transform_bucket_versioning" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" + force_destroy = true tags = { Name = "Lambda Bucket" } -- cgit v1.2.3 From 5493cdc71da4730c4e388d9718f278bc2f14badf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:15:28 +0100 Subject: infra(tf): add ListBucket and GetObject permissions --- terraform/iam.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'terraform') diff --git a/terraform/iam.tf b/terraform/iam.tf index 3ac8c45..3d62b69 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -41,7 +41,8 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectTagging", "s3:PutObjectAcl", "s3:ListObjects", - "s3:ListObjectsV2" + "s3:ListObjectsV2", + "s3:GetObject" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -53,8 +54,10 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { statement { effect = "Allow" actions = [ - "s3:ListBuckets", - "s3:ListAllMyBuckets" + "s3:ListBucket", + "s3:ListAllMyBuckets", + "s3:ListObjectsV2", + "s3:ListObjects" ] resources = [ "arn:aws:s3:::*", -- cgit v1.2.3