From ef770c1ea4ee633489323a8ab321b1214b51a770 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 12 Aug 2024 16:57:11 +0100 Subject: chore: add aws_iam_role --- terraform/iam.tf | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 terraform/iam.tf diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..7501373 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,29 @@ +# define + +resource "aws_iam_role" "bentley_service_role" { + assume_role_policy = < Date: Tue, 13 Aug 2024 11:25:33 +0100 Subject: infra(tf): add s3 policy for list & write --- terraform/iam.tf | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 7501373..b9919a5 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -1,5 +1,3 @@ -# define - resource "aws_iam_role" "bentley_service_role" { assume_role_policy = < Date: Tue, 13 Aug 2024 11:55:00 +0100 Subject: infra(tf): add wip write policy and attach policy --- terraform/iam.tf | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index b9919a5..dda4d74 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -22,17 +22,12 @@ resource "aws_iam_role" "bentley_service_role" { EOF } +# lambda setup + + # s3 setup -# allows to list and retrieve s3 buckets, and allows retention/tagging/access control settings +# allows allows retention/tagging/access control settings data "aws_iam_policy_document" "s3_data_policy_doc" { - statement { - actions = [ - "s3:ListAllMyBuckets", - "s3:GetBucketLocation" - ] - resources = ["arn:aws:s3:::*"] - } - statement { actions = [ "s3:PutObject", @@ -41,8 +36,22 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectAcl" ] resources = [ - "${aws_s3_bucket.data_bucket.arn}/*", - "${aws_s3_bucket.code_bucket.arn}/*" + "${aws_s3_bucket.extract_bucket.arn}/*", + "${aws_s3_bucket.transform_bucket.arn}/*", + "${aws_s3_bucket.lambda_bucket.arn}/*", ] } -} \ No newline at end of file +} + +# write policy +resource "aws_iam_policy" "s3_policy" { + policy = data.aws_iam_policy_document.s3_data_policy_doc.json +} + +# attach policy to role +resource "aws_iam_role_policy_attachment" "s3_policy_attachment" { + role = aws_iam_role.bentley_service_role.name + policy_arn = aws_iam_policy.s3_policy.arn +} + +# lambda setup -- cgit v1.2.3 From 936eee1eb44d8bfdbd148d22b749966e9606fb46 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 11:58:02 +0100 Subject: infra(tf): add wip lambda role --- terraform/iam.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index dda4d74..10b8749 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -23,6 +23,9 @@ resource "aws_iam_role" "bentley_service_role" { } # lambda setup +resource "aws_iam_role" "lambda_role" { + assume_role_policy = data.aws_iam_policy_document.bentley_service_role.json +} # s3 setup @@ -44,14 +47,14 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { } # write policy -resource "aws_iam_policy" "s3_policy" { +resource "aws_iam_policy" "s3_write_policy" { policy = data.aws_iam_policy_document.s3_data_policy_doc.json } # attach policy to role resource "aws_iam_role_policy_attachment" "s3_policy_attachment" { - role = aws_iam_role.bentley_service_role.name - policy_arn = aws_iam_policy.s3_policy.arn + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.s3_write_policy.arn } # lambda setup -- cgit v1.2.3 From eb09f0f6a42e2a2ce9529492a47a34f782ffad53 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 12:17:52 +0100 Subject: infra(tf): clean-up code & init lambda iam setup --- terraform/iam.tf | 123 +++++++++++++++++++++++++++++++++---------------------- 1 file changed, 74 insertions(+), 49 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 10b8749..ecc63b1 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -1,60 +1,85 @@ -resource "aws_iam_role" "bentley_service_role" { - assume_role_policy = < Date: Tue, 13 Aug 2024 12:30:20 +0100 Subject: replacement for events.tf variable placeholders, s3 bucket notifications, lambda.tf set up (function, zip, permissions), code bucket to store zipped lambdas --- events - TBD?.tf | 52 +++++++++++++++++++++++++++++++++++++ events.tf | 52 ------------------------------------- terraform/events.tf | 23 ++++++++--------- terraform/lambda.tf | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++ terraform/s3.tf | 39 ++++++++++++++++++++++------ terraform/vars.tf | 5 ++++ 6 files changed, 173 insertions(+), 72 deletions(-) create mode 100644 events - TBD?.tf delete mode 100644 events.tf create mode 100644 terraform/lambda.tf diff --git a/events - TBD?.tf b/events - TBD?.tf new file mode 100644 index 0000000..25fb35b --- /dev/null +++ b/events - TBD?.tf @@ -0,0 +1,52 @@ +resource "aws_cloudwatch_event_rule" "lambda_trigger" { + name = "lambda-scheduled-trigger" + description = "Schedule to trigger the Lambda function" + schedule_expression = "rate(30 minutes)" + +# event_pattern = jsonencode({ +# detail-type = [ +# "AWS Console Sign In via CloudTrail" +# ] +# }) +} + + +resource "aws_cloudwatch_event_target" "lambda" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.my_lambda_function.arn +} + + + +resource "aws_lambda_permission" "allow_eventbridge" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.my_lambda_function.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn +} + + +# below is step function 1 +resource "aws_lambda_permission" "allow_s3_ingestion" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_transform.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract.arn +} + + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract.id + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.lambda_transform.arn + } + + depends_on = [aws_lambda_permission.allow_s3_ingestion] +} + +# need to duplicate and replace "2" with "3" \ No newline at end of file diff --git a/events.tf b/events.tf deleted file mode 100644 index 25fb35b..0000000 --- a/events.tf +++ /dev/null @@ -1,52 +0,0 @@ -resource "aws_cloudwatch_event_rule" "lambda_trigger" { - name = "lambda-scheduled-trigger" - description = "Schedule to trigger the Lambda function" - schedule_expression = "rate(30 minutes)" - -# event_pattern = jsonencode({ -# detail-type = [ -# "AWS Console Sign In via CloudTrail" -# ] -# }) -} - - -resource "aws_cloudwatch_event_target" "lambda" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn -} - - - -resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn -} - - -# below is step function 1 -resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn -} - - -resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id - - lambda_function { - events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn - } - - depends_on = [aws_lambda_permission.allow_s3_ingestion] -} - -# need to duplicate and replace "2" with "3" \ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 4d68a23..0196dc3 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -11,18 +11,17 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { } -resource "aws_cloudwatch_event_target" "lambda" { +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder } - resource "aws_lambda_permission" "allow_eventbridge" { statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name + function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn } @@ -32,18 +31,18 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn + source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id + bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -54,18 +53,18 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform.arn + source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform.id + bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_transform] diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..09d6697 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,74 @@ +### EXTRACT LAMBDA SET UP +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} + +resource "aws_lambda_function" "extract_lambda" { + function_name = "${var.extract_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "extract_lambda/extract_function.zip" + role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.extract_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn +} + + +### TRANSFORM LAMBDA SET UP +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} + +resource "aws_lambda_function" "transform_lambda" { + function_name = "${var.transform_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "transform_lambda/transform_function.zip" + role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.transform_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn +} + + +### LOAD LAMBDA SET UP +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} + +resource "aws_lambda_function" "load_lambda" { + function_name = "${var.load_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "load_lambda/load_function.zip" + role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" +} + diff --git a/terraform/s3.tf b/terraform/s3.tf index bfe891e..8cb65ef 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,17 +1,40 @@ +### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket = "${var.s3_extract_bucket_name}" } +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.extract_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +} # << is this the correct permission dependency? + + +### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket = "${var.s3_transform_bucket_name}" } -resource "aws_s3_bucket" "lambda_bucket" { - bucket = "${var.s3_code_bucket_name}" -} - -resource "aws_s3_object" "extract_lambda_code" { +resource "aws_s3_object" "transform_lambda_code" { bucket = aws_s3_bucket.s3_code_bucket_name.bucket - key = "${var.extract_lambda_name}/function_e.zip" - source = "${path.module}/../function_e.zip" -} \ No newline at end of file + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.transform_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +} # << is this the correct permission dependency? diff --git a/terraform/vars.tf b/terraform/vars.tf index fa84222..cc9348a 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -23,6 +23,11 @@ variable "transform_lambda_name" { default = "transform-lambda" } +variable "load_lambda_name" { + type = string + default = "load-lambda" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} \ No newline at end of file -- cgit v1.2.3 From 3c824df60374380d044cb9181672fa76b610d84f Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 13 Aug 2024 12:30:53 +0100 Subject: infra(tf): clean-up code --- terraform/iam.tf | 69 +++++++++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 56 insertions(+), 13 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index ecc63b1..bb8d932 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -4,7 +4,7 @@ ######################################################################## # DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events) -resource "aws_iam_role" "multi_service_role" { +resource "aws_iam_role" "bentley_multi_service_role" { name = "multi_service_role" assume_role_policy = jsonencode({ @@ -61,6 +61,61 @@ resource "aws_iam_policy" "s3_access_policy" { ) } +######################################################################## +# LAMBDA SETUP +# Description: Allows Lambda permission to write to Cloudwatch logs +######################################################################## + +resource "aws_iam_policy" "lambda_execution_policy" { + name = "lambda_execution_policy" + path = "/" + description = "IAM policy for Lambda execution" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "lambda:InvokeFunction", + "lambda:GetFunction" + ] + Resource = "*" + } + ] + } + ) +} + +######################################################################## +# CLOUDWATCH SETUP +# Description: Give permission for Lambda to write to CloudWatch logs +######################################################################## + +data "aws_iam_policy_document" "cw_document" { + statement { + actions = ["logs:CreateLogGroup"] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" + ] + } + + statement { + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] + } +} + +######################################################################## +# POLICY WRITE & ATTACH +######################################################################## + # S3 WRITE POLICY resource "aws_iam_policy" "s3_write_policy" { policy = data.aws_iam_policy_document.s3_data_policy_doc.json @@ -70,16 +125,4 @@ resource "aws_iam_policy" "s3_write_policy" { resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { role = aws_iam_role.lambda_role.name policy_arn = aws_iam_policy.s3_write_policy.arn -} - -######################################################################## -# LAMBDA SETUP -# Description: Allows Lambda permission to write to Cloudwatch logs -######################################################################## - - - -# Uses Iam policy document to assume role for lambda functions -resource "aws_iam_role" "lambda_role" { - assume_role_policy = data.aws_iam_policy_document.bentley_service_role.json } \ No newline at end of file -- cgit v1.2.3 From 3e5a5db92f8c41c7363e1bbf1bbe7593c6fb9aa4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 15:01:37 +0100 Subject: ci: initialise quality-checks job --- .github/workflows/deploy.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..b214678 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,33 @@ +name: de-project-bentley + +on: + push: + branches: [development, staging] + pull_request: + branches: [main, development, staging] + +jobs: + quality-checks: + runs-on: ubuntu-latest + steps: + - uses : actions/checkout@v2 + - name : Setup + uses : actions/setup-python@v2 + with: + python-version: 3.11 + - name : Dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pylint black bandit safety + - name : Linting + run: | + flake8 . + pylint **/*.py + - name : Formatting + run: | + black --check . + - name: Security + run: | + bandit -r . + safety check + \ No newline at end of file -- cgit v1.2.3 From 8bbf8803473badc357a9747a5ccff32c8aea4dc8 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 15:33:58 +0100 Subject: fix(ci): replace glob pattern for pylint --- .github/workflows/deploy.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index b214678..71b2871 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -22,7 +22,7 @@ jobs: - name : Linting run: | flake8 . - pylint **/*.py + find . -name "*.py" | xargs pylint - name : Formatting run: | black --check . -- cgit v1.2.3 From 43cc61cd535609b59948ba75d8d9ea859bf8d990 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 15:59:08 +0100 Subject: ci: add terraform quality checks --- .github/workflows/terraform.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/terraform.yml diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml new file mode 100644 index 0000000..d28a357 --- /dev/null +++ b/.github/workflows/terraform.yml @@ -0,0 +1,34 @@ +name: terraform-quality-checks + +on: + push: + branches: [development] + paths: + - 'terraform/**.tf' + - 'terraform/**.tfvars' + pull_request: + branches: [development, staging] + paths: + - 'terraform/**.tf' + - 'terraform/**.tfvars' +jobs: + terraform-validation: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Setup Terraform + uses: hashicorp/setup-terraform@v1 + with: + terraform_version: latest # Using the latest version, but not sure if it's the best practice + - name: Format + run: terraform fmt -check -recursive + - name: Init + run: terraform init -backend=false + - name: Validate + run: terraform validate + - name: Setup TFLint + uses: terraform-linters/setup-tflint@v2 + with: + tflint_version: latest + - name: Run TFLint + run: tflint -f compact \ No newline at end of file -- cgit v1.2.3 From 4eef583c0ad16f938bcec1eaaf83ace974b8c667 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 15:59:34 +0100 Subject: ci: add python quality checks (separately) --- .github/workflows/python.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/python.yml diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml new file mode 100644 index 0000000..4f752b7 --- /dev/null +++ b/.github/workflows/python.yml @@ -0,0 +1,33 @@ +name: python-quality-checks + +on: + push: + branches: [development] + pull_request: + branches: [development, staging] + +jobs: + quality-checks: + runs-on: ubuntu-latest + steps: + - uses : actions/checkout@v2 + - name : Setup + uses : actions/setup-python@v2 + with: + python-version: 3.11 + - name : Dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pylint black bandit safety + - name : Linting + run: | + flake8 . + find . -name "*.py" | xargs pylint + - name : Formatting + run: | + black --check . + - name: Security + run: | + bandit -r . + safety check + \ No newline at end of file -- cgit v1.2.3 From 20c7b0c647b455958bca02b263fb50e330fe6775 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 16:00:07 +0100 Subject: ci: rm unused deploy.yml --- .github/workflows/deploy.yml | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 71b2871..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,33 +0,0 @@ -name: de-project-bentley - -on: - push: - branches: [development, staging] - pull_request: - branches: [main, development, staging] - -jobs: - quality-checks: - runs-on: ubuntu-latest - steps: - - uses : actions/checkout@v2 - - name : Setup - uses : actions/setup-python@v2 - with: - python-version: 3.11 - - name : Dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pylint black bandit safety - - name : Linting - run: | - flake8 . - find . -name "*.py" | xargs pylint - - name : Formatting - run: | - black --check . - - name: Security - run: | - bandit -r . - safety check - \ No newline at end of file -- cgit v1.2.3 From c849807a008f738b0e3b9230521b4112ae8a71b8 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 16:01:39 +0100 Subject: ci: set cwd to ./terraform.yml --- .github/workflows/terraform.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/terraform.yml b/.github/workflows/terraform.yml index d28a357..c349756 100644 --- a/.github/workflows/terraform.yml +++ b/.github/workflows/terraform.yml @@ -14,6 +14,9 @@ on: jobs: terraform-validation: runs-on: ubuntu-latest + defaults: + run: + working-directory: ./terraform steps: - uses: actions/checkout@v2 - name: Setup Terraform -- cgit v1.2.3 From 6f5759ceccb7f84cbd7898a057504f49a0cbb790 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 13 Aug 2024 16:05:20 +0100 Subject: ci: check if python files exist --- .github/workflows/python.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 4f752b7..7d5b5b1 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,7 +7,24 @@ on: branches: [development, staging] jobs: + + check-if-py-files-exist: + runs-on: ubuntu-latest + outputs: + py_files_exist: ${{ steps.check.outputs.py_files_exist }} + steps: + - uses: actions/checkout@v2 + - id: check_files + run: | + if [ -n "$(find . -name '*.py')" ]; then + echo "::set-output name=py_files_exist::true" + else + echo "::set-output name=py_files_exist::false" + fi + quality-checks: + needs: check-if-py-files-exist + if: ${{ needs.check-if-py-files-exist.outputs.py_files_exist == 'true' }} runs-on: ubuntu-latest steps: - uses : actions/checkout@v2 -- cgit v1.2.3 From bb1665fd08d8abf10930875272bdc2b7f8a4c681 Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 13 Aug 2024 16:25:06 +0100 Subject: rds.tf file with some placehodlers. Additional set up is required --- terraform/lambda.tf | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++ terraform/rds.tf | 78 +++++++++++++++++++++++++++++++++++++++++++++++++++++ terraform/vars.tf | 10 +++++++ 3 files changed, 162 insertions(+) create mode 100644 terraform/lambda.tf create mode 100644 terraform/rds.tf diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..09d6697 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,74 @@ +### EXTRACT LAMBDA SET UP +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} + +resource "aws_lambda_function" "extract_lambda" { + function_name = "${var.extract_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "extract_lambda/extract_function.zip" + role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.extract_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn +} + + +### TRANSFORM LAMBDA SET UP +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} + +resource "aws_lambda_function" "transform_lambda" { + function_name = "${var.transform_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "transform_lambda/transform_function.zip" + role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.transform_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn +} + + +### LOAD LAMBDA SET UP +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} + +resource "aws_lambda_function" "load_lambda" { + function_name = "${var.load_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "load_lambda/load_function.zip" + role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" +} + diff --git a/terraform/rds.tf b/terraform/rds.tf new file mode 100644 index 0000000..4b25c5f --- /dev/null +++ b/terraform/rds.tf @@ -0,0 +1,78 @@ +data "aws_availability_zones" "available" {} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "2.77.0" + + name = "${var.project_name}" + cidr = "10.0.0.0/16" + azs = data.aws_availability_zones.available.names + public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] + enable_dns_hostnames = true + enable_dns_support = true +} + +resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { + name = "TT-db-subnet" + subnet_ids = module.vpc.public_subnets + + tags = { + Name = "${var.project_name}" + } +} + +resource "aws_security_group" "rds" { + name = "${var.project_name}-rds" + vpc_id = module.vpc.vpc_id + + ingress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "${var.project_name}-rds" + } +} + +resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { + name = "TT-db-param" + family = "postgres14" + + parameter { + name = "log_connections" + value = "1" + } +} + +resource "aws_db_instance" "Terrific-Totes-rds" { + db_name = "${var.project_name}" + instance_class = "db.t3.micro" + allocated_storage = 5 + engine = "postgres" + engine_version = "14.1" + username = "user credentials for the root user" # we could use .env here + password = "user password for the root user" # we could use .env here + ### alternatively to providing username nad password we can specify: +# resource "aws_kms_key" "example_key" { +# description = "Example KMS Key" +# } +# within the resource: +# manage_master_user_password = true +# master_user_secret_kms_key_id = aws_kms_key.example.key_id +# } + db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name + vpc_security_group_ids = [aws_security_group.rds.id] + parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name + publicly_accessible = false + skip_final_snapshot = true +} \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf index fa84222..350c2c6 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -23,6 +23,16 @@ variable "transform_lambda_name" { default = "transform-lambda" } +variable "load_lambda_name" { + type = string + default = "load-lambda" +} + +variable "project_name" { + type = string + default = "Terrific-Totes" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} \ No newline at end of file -- cgit v1.2.3 From 6c7914a9d33fbaa962cf1f083c2ee79ace62f401 Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 13 Aug 2024 16:33:56 +0100 Subject: restore load_lambda script --- src/load_lambda.py | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/load_lambda.py b/src/load_lambda.py index e69de29..5c6718c 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -0,0 +1,52 @@ +### Example taken from https://medium.com/@pranay1001090/how-to-load-data-from-amazon-s3-csv-parquet-to-aws-rds-using-python-3dc51dd2186e + +### THIS IS AN EXAMPLE CODE WE CAN PICK FROM, NONE OF THIS HAS BEEN CUSTOMISED YET + +import boto3 +import pandas as pd +import pyarrow.parquet as pq +from io import BytesIO +from sqlalchemy import create_engine + +# AWS credentials and region +aws_access_key = '' +aws_secret_key = '' +region_name = '' + +# S3 bucket and file details +bucket_name = '' +file_prefix = '' +s3_client = boto3.client('s3', aws_access_key_id=aws_access_key, aws_secret_access_key=aws_secret_key, region_name=region_name) + +# RDS connection details +database_name = '' +table_name = '' +rds_host = '' +rds_port = '' +rds_user = '' +rds_password = '' +# Function to load Parquet files into a Pandas DataFrame +def load_parquet_data(s3_bucket, s3_prefix): + file_objects = s3_client.list_objects_v2(Bucket=s3_bucket, Prefix=s3_prefix)['Contents'] + dfs = [] + for file_object in file_objects: + file_key = file_object['Key'] + file_obj = s3_client.get_object(Bucket=s3_bucket, Key=file_key) + parquet_file = pq.ParquetFile(BytesIO(file_obj['Body'].read())) + df = parquet_file.read().to_pandas() + dfs.append(df) + return pd.concat(dfs) + +# Load Parquet data from S3 into a Pandas DataFrame +df = load_parquet_data(bucket_name, file_prefix) +# Connect to RDS +conn_str = f'mysql+pymysql://{rds_user}:{rds_password}@{rds_host}:{rds_port}/{database_name}' +engine = create_engine(conn_str) + +# Write the DataFrame to RDS +df.to_sql(table_name, con=engine, if_exists='replace', index=False) + +# Closing the connection +engine.dispose() + +print('Data loaded successfully!') \ No newline at end of file -- cgit v1.2.3 From 68a0b4740e1aab2c507547ab985c7c1dc436d9c9 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Tue, 13 Aug 2024 17:16:12 +0100 Subject: wip: running terraform apply to continue fixing terraform infrastructure --- .gitignore | 5 +++++ terraform/events.tf | 4 ++-- terraform/iam.tf | 64 ++++++++++++++++++++++++++++------------------------- terraform/lambda.tf | 13 +++++------ terraform/s3.tf | 63 ++++++++++++++++++++++++++++++---------------------- 5 files changed, 84 insertions(+), 65 deletions(-) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5861f48 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +*.tfstate +*.tfstate.* +*.tfvars +*.tfvars.json +.terraform.tfstate.lock.info \ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 0196dc3..7a6b0ad 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -50,7 +50,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ###### -resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { +resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder @@ -67,5 +67,5 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" { lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } - depends_on = [aws_lambda_permission.allow_s3_transform] + depends_on = [aws_lambda_permission.allow_s3_transform_bucket] } \ No newline at end of file diff --git a/terraform/iam.tf b/terraform/iam.tf index bb8d932..f34d58a 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -4,7 +4,7 @@ ######################################################################## # DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events) -resource "aws_iam_role" "bentley_multi_service_role" { +resource "aws_iam_role" "multi_service_role" { name = "multi_service_role" assume_role_policy = jsonencode({ @@ -16,7 +16,7 @@ resource "aws_iam_role" "bentley_multi_service_role" { Principal = { Service = [ "lambda.amazonaws.com", - "states.amazonaws.com", + "cloudwatch.amazonaws.com", "events.amazonaws.com", "s3.amazonaws.com" ] @@ -27,7 +27,6 @@ resource "aws_iam_role" "bentley_multi_service_role" { } - ######################################################################## # S3 SETUP # Description: allows allows retention/tagging/access control settings @@ -35,32 +34,23 @@ resource "aws_iam_role" "bentley_multi_service_role" { ######################################################################## # S3 DEFINE POLICY -resource "aws_iam_policy" "s3_access_policy" { - name = "s3_access_policy" - path = "/" - description = "IAM policy for S3 access" - - policy = jsonencode({ - Version = "2012-10-17" - Statement = [ - { - Effect = "Allow" - Action = [ - "s3:PutObject", - "s3:GetObject", - "s3:ListBucket" - ] - resources = [ - "${aws_s3_bucket.extract_bucket.arn}/*", - "${aws_s3_bucket.transform_bucket.arn}/*", - "${aws_s3_bucket.lambda_bucket.arn}/*" - ] - } - ] - } - ) +data "aws_iam_policy_document" "s3_data_policy_doc" { + statement { + actions = [ + "s3:PutObject", + "s3:PutObjectRetention", + "s3:PutObjectTagging", + "s3:PutObjectAcl" + ] + resources = [ + "${aws_s3_bucket.extract_bucket.arn}/*", + "${aws_s3_bucket.transform_bucket.arn}/*", + "${aws_s3_bucket.lambda_code_bucket.arn}/*", + ] + } } + ######################################################################## # LAMBDA SETUP # Description: Allows Lambda permission to write to Cloudwatch logs @@ -112,6 +102,11 @@ data "aws_iam_policy_document" "cw_document" { } } +resource "aws_iam_policy" "cw_policy" { + name = "cw_policy" + policy = data.aws_iam_policy_document.cw_document.json +} + ######################################################################## # POLICY WRITE & ATTACH ######################################################################## @@ -123,6 +118,15 @@ resource "aws_iam_policy" "s3_write_policy" { # S3 ATTACH POLICY resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { - role = aws_iam_role.lambda_role.name - policy_arn = aws_iam_policy.s3_write_policy.arn -} \ No newline at end of file + for_each = toset([ + aws_iam_policy.s3_write_policy.arn, + aws_iam_policy.lambda_execution_policy.arn, + aws_iam_policy.cw_policy.arn + ]) + role = aws_iam_role.multi_service_role.name + policy_arn = each.value +} + +################ +# RDS POLICIES # +################ diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 09d6697..bcbf394 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -7,9 +7,9 @@ data "archive_file" "extract_lambda_zip" { resource "aws_lambda_function" "extract_lambda" { function_name = "${var.extract_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "extract_lambda/extract_function.zip" - role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" environment { @@ -36,9 +36,9 @@ data "archive_file" "transform_lambda_zip" { resource "aws_lambda_function" "transform_lambda" { function_name = "${var.transform_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "transform_lambda/transform_function.zip" - role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" environment { @@ -55,7 +55,6 @@ resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { source_arn = aws_s3_bucket.transform_bucket.arn } - ### LOAD LAMBDA SET UP data "archive_file" "load_lambda_zip" { type = "zip" @@ -65,9 +64,9 @@ data "archive_file" "load_lambda_zip" { resource "aws_lambda_function" "load_lambda" { function_name = "${var.load_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "load_lambda/load_function.zip" - role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" } diff --git a/terraform/s3.tf b/terraform/s3.tf index 8cb65ef..8ab5622 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,40 +1,51 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { - bucket = "${var.s3_extract_bucket_name}" + bucket_prefix = "${var.s3_extract_bucket_name}-" } -resource "aws_s3_object" "extract_lambda_code" { - bucket = aws_s3_bucket.s3_code_bucket_name.bucket - key = "${var.extract_lambda_name}/extract_function.zip" - source = "${path.module}/../extract_function.zip" -} # << can't figure out how this is being used but we seem to need it - -resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract_bucket.id - lambda_function { - lambda_function_arn = aws_lambda_function.extract_lambda.arn - events = ["s3:ObjectCreated:*"] - } - depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] -} # << is this the correct permission dependency? - +# resource "aws_s3_bucket_notification" "extract_bucket_notification" { +# bucket = aws_s3_bucket.extract_bucket.id +# lambda_function { +# lambda_function_arn = aws_lambda_function.extract_lambda.arn +# events = ["s3:ObjectCreated:*"] +# } +# depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +# } # << is this the correct permission dependency? ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { - bucket = "${var.s3_transform_bucket_name}" + bucket_prefix = "${var.s3_transform_bucket_name}-" } +# resource "aws_s3_bucket_notification" "transform_bucket_notification" { +# bucket = aws_s3_bucket.transform_bucket.id +# lambda_function { +# lambda_function_arn = aws_lambda_function.transform_lambda.arn +# events = ["s3:ObjectCreated:*"] +# } +# depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +# } # << is this the correct permission dependency? + + +### LAMBDA BUCKET +resource "aws_s3_bucket" "lambda_code_bucket" { + bucket_prefix = "${var.s3_code_bucket_name}-" +} + +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + resource "aws_s3_object" "transform_lambda_code" { - bucket = aws_s3_bucket.s3_code_bucket_name.bucket + bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.transform_lambda_name}/transform_function.zip" source = "${path.module}/../transform_function.zip" } # << can't figure out how this is being used but we seem to need it -resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform_bucket.id - lambda_function { - lambda_function_arn = aws_lambda_function.transform_lambda.arn - events = ["s3:ObjectCreated:*"] - } - depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] -} # << is this the correct permission dependency? +resource "aws_s3_object" "load_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.load_lambda_name}/load_function.zip" + source = "${path.module}/../load_function.zip" +} \ No newline at end of file -- cgit v1.2.3 From 0e3faa19dec148fe05f6b2f58fc0331230ec41b7 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:34:42 +0100 Subject: delete events TBD --- events - TBD?.tf | 52 ---------------------------------------------------- 1 file changed, 52 deletions(-) delete mode 100644 events - TBD?.tf diff --git a/events - TBD?.tf b/events - TBD?.tf deleted file mode 100644 index 25fb35b..0000000 --- a/events - TBD?.tf +++ /dev/null @@ -1,52 +0,0 @@ -resource "aws_cloudwatch_event_rule" "lambda_trigger" { - name = "lambda-scheduled-trigger" - description = "Schedule to trigger the Lambda function" - schedule_expression = "rate(30 minutes)" - -# event_pattern = jsonencode({ -# detail-type = [ -# "AWS Console Sign In via CloudTrail" -# ] -# }) -} - - -resource "aws_cloudwatch_event_target" "lambda" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn -} - - - -resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name - principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn -} - - -# below is step function 1 -resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform" - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn -} - - -resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id - - lambda_function { - events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn - } - - depends_on = [aws_lambda_permission.allow_s3_ingestion] -} - -# need to duplicate and replace "2" with "3" \ No newline at end of file -- cgit v1.2.3 From 9966fcff43cc5e748bc3eb406c270d63fa51ca61 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:35:25 +0100 Subject: change the s3-key --- terraform/lambda.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bcbf394..a5da972 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -8,7 +8,7 @@ data "archive_file" "extract_lambda_zip" { resource "aws_lambda_function" "extract_lambda" { function_name = "${var.extract_lambda_name}" s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "extract_lambda/extract_function.zip" + s3_key = "extract-lambda/extract_function.zip" role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" @@ -37,7 +37,7 @@ data "archive_file" "transform_lambda_zip" { resource "aws_lambda_function" "transform_lambda" { function_name = "${var.transform_lambda_name}" s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "transform_lambda/transform_function.zip" + s3_key = "transform-lambda/transform_function.zip" role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" @@ -65,7 +65,7 @@ data "archive_file" "load_lambda_zip" { resource "aws_lambda_function" "load_lambda" { function_name = "${var.load_lambda_name}" s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "load_lambda/load_function.zip" + s3_key = "load-lambda/load_function.zip" role = aws_iam_role.multi_service_role.arn # << lambda role placehodler handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler runtime = "python3.11" -- cgit v1.2.3 From 6caff95389a6054f8fe1d1c9c23db9a251b6b355 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:35:59 +0100 Subject: change the name of bucket in backend bucket --- terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/main.tf b/terraform/main.tf index 3ca9a3d..2a048b9 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -6,7 +6,7 @@ terraform { } } backend "s3" { - bucket = "bentley-secrets" + bucket = "bentley-project-secrets" key = "bentley-project/terraform.tfstate" region = "eu-west-2" } -- cgit v1.2.3 From 4d52771ceee6841febc6179e0b9608fb35a792d9 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:36:22 +0100 Subject: change the attachment/policies, individual --- terraform/iam.tf | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index f34d58a..cf4902a 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -117,14 +117,29 @@ resource "aws_iam_policy" "s3_write_policy" { } # S3 ATTACH POLICY -resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { - for_each = toset([ - aws_iam_policy.s3_write_policy.arn, - aws_iam_policy.lambda_execution_policy.arn, - aws_iam_policy.cw_policy.arn - ]) - role = aws_iam_role.multi_service_role.name - policy_arn = each.value +# resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { +# for_each = toset([ +# aws_iam_policy.s3_write_policy.arn, +# aws_iam_policy.lambda_execution_policy.arn, +# aws_iam_policy.cw_policy.arn +# ]) +# role = aws_iam_role.multi_service_role.name +# policy_arn = each.value +# } + +resource "aws_iam_role_policy_attachment" "s3_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.s3_write_policy.arn +} + +resource "aws_iam_role_policy_attachment" "lambda_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.lambda_execution_policy.arn +} + +resource "aws_iam_role_policy_attachment" "cw_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.cw_policy.arn } ################ -- cgit v1.2.3 From f2c0429698edafccb0846a58e4020bf419e7e824 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:37:31 +0100 Subject: style change --- terraform/events.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/events.tf b/terraform/events.tf index 7a6b0ad..6744085 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -4,7 +4,7 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { schedule_expression = "rate(30 minutes)" # event_pattern = jsonencode({ -# detail-type = [ +# detail-type = # "AWS Console Sign In via CloudTrail" # ] # }) -- cgit v1.2.3 From 46673b671bef834dc2e043e7845e8a5b8fee9d34 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 12:38:08 +0100 Subject: update gitignore --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 5861f48..239c7e0 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.tfstate.* *.tfvars *.tfvars.json -.terraform.tfstate.lock.info \ No newline at end of file +.terraform.tfstate.lock.info +*.zip \ No newline at end of file -- cgit v1.2.3 From 101e1e24cb38b6a45661b723881e2b2d6dd2fb07 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Wed, 14 Aug 2024 14:35:05 +0100 Subject: wip: terraform debugging --- .gitignore | 5 ++++- src/load_lambda.py | 2 ++ src/transform_lambda.py | 2 ++ terraform/events.tf | 18 ++++++++++-------- terraform/s3.tf | 34 +++++++++++++++++----------------- 5 files changed, 35 insertions(+), 26 deletions(-) diff --git a/.gitignore b/.gitignore index 239c7e0..d759665 100644 --- a/.gitignore +++ b/.gitignore @@ -3,4 +3,7 @@ *.tfvars *.tfvars.json .terraform.tfstate.lock.info -*.zip \ No newline at end of file +*.zip +.terraform/ +.terraform* +log* \ No newline at end of file diff --git a/src/load_lambda.py b/src/load_lambda.py index e69de29..6ee681f 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -0,0 +1,2 @@ +def lambda_handler(): + pass \ No newline at end of file diff --git a/src/transform_lambda.py b/src/transform_lambda.py index e69de29..6ee681f 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -0,0 +1,2 @@ +def lambda_handler(): + pass \ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 6744085..9fd89e4 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,7 +1,17 @@ +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + force_destroy = true +} + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" schedule_expression = "rate(30 minutes)" + force_destroy = true + # depends_on = [ + # aws_cloudwatch_event_target.extract_lambda_cw_event] # event_pattern = jsonencode({ # detail-type = @@ -10,14 +20,6 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { # }) } - -resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder -} - - resource "aws_lambda_permission" "allow_eventbridge" { statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" diff --git a/terraform/s3.tf b/terraform/s3.tf index 8ab5622..4c06b8e 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -32,20 +32,20 @@ resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } -resource "aws_s3_object" "extract_lambda_code" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "${var.extract_lambda_name}/extract_function.zip" - source = "${path.module}/../extract_function.zip" -} # << can't figure out how this is being used but we seem to need it - -resource "aws_s3_object" "transform_lambda_code" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "${var.transform_lambda_name}/transform_function.zip" - source = "${path.module}/../transform_function.zip" -} # << can't figure out how this is being used but we seem to need it - -resource "aws_s3_object" "load_lambda_code" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "${var.load_lambda_name}/load_function.zip" - source = "${path.module}/../load_function.zip" -} \ No newline at end of file +# resource "aws_s3_object" "extract_lambda_code" { +# bucket = aws_s3_bucket.lambda_code_bucket.bucket +# key = "${var.extract_lambda_name}/extract_function.zip" +# source = "${path.module}/../extract_function.zip" +# } # << can't figure out how this is being used but we seem to need it + +# resource "aws_s3_object" "transform_lambda_code" { +# bucket = aws_s3_bucket.lambda_code_bucket.bucket +# key = "${var.transform_lambda_name}/transform_function.zip" +# source = "${path.module}/../transform_function.zip" +# } # << can't figure out how this is being used but we seem to need it + +# resource "aws_s3_object" "load_lambda_code" { +# bucket = aws_s3_bucket.lambda_code_bucket.bucket +# key = "${var.load_lambda_name}/load_function.zip" +# source = "${path.module}/../load_function.zip" +# } \ No newline at end of file -- cgit v1.2.3 From e95e9e59d2a36a2d700bfa199f6b5ece3c49233d Mon Sep 17 00:00:00 2001 From: Ellie Date: Wed, 14 Aug 2024 16:35:32 +0100 Subject: chage rate 30 seconds --- terraform/events.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/events.tf b/terraform/events.tf index 9fd89e4..ef01feb 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -8,7 +8,7 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" - schedule_expression = "rate(30 minutes)" + schedule_expression = "rate(30 seconds)" force_destroy = true # depends_on = [ # aws_cloudwatch_event_target.extract_lambda_cw_event] -- cgit v1.2.3 From 911b2a4ba08e59f46a53b5252a044a5853796f78 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 16:38:36 +0100 Subject: test(tf): attempt to solve ResourceConflictException --- terraform/events.tf | 35 ++++++++--------- terraform/iam.tf | 60 +++++++++++++++++++++-------- terraform/lambda.tf | 109 +++++++++++++++++++++++++--------------------------- terraform/main.tf | 26 ++++++------- terraform/s3.tf | 43 ++------------------- terraform/vars.tf | 26 ++++++------- 6 files changed, 139 insertions(+), 160 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index 9fd89e4..d2e2eb5 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,23 +1,18 @@ -resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { - rule = aws_cloudwatch_event_rule.lambda_trigger.name - target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder - force_destroy = true -} - resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" schedule_expression = "rate(30 minutes)" - force_destroy = true - # depends_on = [ - # aws_cloudwatch_event_target.extract_lambda_cw_event] - -# event_pattern = jsonencode({ -# detail-type = -# "AWS Console Sign In via CloudTrail" -# ] -# }) + + lifecycle { + create_before_destroy = true + } +} + +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + depends_on = [aws_lambda_permission.allow_eventbridge] } resource "aws_lambda_permission" "allow_eventbridge" { @@ -25,7 +20,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder principal = "events.amazonaws.com" - source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn } @@ -43,7 +38,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder lambda_function { - events = ["s3:ObjectCreated:*"] + events = ["s3:ObjectCreated:*"] lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } @@ -65,9 +60,9 @@ resource "aws_s3_bucket_notification" "transform_bucket_notification" { bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder lambda_function { - events = ["s3:ObjectCreated:*"] + events = ["s3:ObjectCreated:*"] lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_transform_bucket] -} \ No newline at end of file +} diff --git a/terraform/iam.tf b/terraform/iam.tf index cf4902a..6c6b4fc 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -16,9 +16,7 @@ resource "aws_iam_role" "multi_service_role" { Principal = { Service = [ "lambda.amazonaws.com", - "cloudwatch.amazonaws.com", - "events.amazonaws.com", - "s3.amazonaws.com" + "scheduler.amazonaws.com" ] } } @@ -57,22 +55,22 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { ######################################################################## resource "aws_iam_policy" "lambda_execution_policy" { - name = "lambda_execution_policy" - path = "/" + name = "lambda_execution_policy" + path = "/" description = "IAM policy for Lambda execution" policy = jsonencode({ Version = "2012-10-17" Statement = [ - { + { Effect = "Allow" Action = [ "lambda:InvokeFunction", "lambda:GetFunction" ] Resource = "*" - } - ] + } + ] } ) } @@ -87,7 +85,7 @@ data "aws_iam_policy_document" "cw_document" { actions = ["logs:CreateLogGroup"] resources = [ "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" - ] + ] } statement { @@ -95,15 +93,15 @@ data "aws_iam_policy_document" "cw_document" { "logs:CreateLogStream", "logs:CreateLogGroup", "logs:PutLogEvents" - ] - resources = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" - ] + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] } } resource "aws_iam_policy" "cw_policy" { - name = "cw_policy" + name = "cw_policy" policy = data.aws_iam_policy_document.cw_document.json } @@ -128,20 +126,48 @@ resource "aws_iam_policy" "s3_write_policy" { # } resource "aws_iam_role_policy_attachment" "s3_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.s3_write_policy.arn } resource "aws_iam_role_policy_attachment" "lambda_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.lambda_execution_policy.arn } resource "aws_iam_role_policy_attachment" "cw_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.cw_policy.arn } ################ # RDS POLICIES # ################ + +################### +# EVENTS POLICIES # +################### + +data "aws_iam_policy_document" "cloudwatch_events_policy" { + statement { + actions = [ + "events:PutRule", + "events:PutTargets", + "events:RemoveTargets", + "events:DeleteRule", + "events:PutEvents" + ] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "cloudwatch_events_policy" { + name = "cloudwatch_events_policy" + policy = data.aws_iam_policy_document.cloudwatch_events_policy.json +} + +resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.cloudwatch_events_policy.arn +} diff --git a/terraform/lambda.tf b/terraform/lambda.tf index a5da972..fb0a666 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,73 +1,68 @@ -### EXTRACT LAMBDA SET UP -data "archive_file" "extract_lambda_zip" { - type = "zip" - source_file = "${path.module}/../src/extract_lambda.py" - output_path = "${path.module}/../extract_function.zip" +# Extract Lambda Function +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" + etag = filemd5("${path.module}/../extract_function.zip") } resource "aws_lambda_function" "extract_lambda" { - function_name = "${var.extract_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "extract-lambda/extract_function.zip" - role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler - handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler - runtime = "python3.11" - environment { - variables = { - output = aws_s3_bucket.extract_bucket.bucket - } - } -} + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.extract" + runtime = "python3.11" -resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.extract_lambda.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract_bucket.arn -} + lifecycle { + create_before_destroy = true + } + depends_on = [aws_s3_object.extract_lambda_code] +} -### TRANSFORM LAMBDA SET UP -data "archive_file" "transform_lambda_zip" { - type = "zip" - source_file = "${path.module}/../src/transform_lambda.py" - output_path = "${path.module}/../transform_function.zip" +# Transform Lambda Function +resource "aws_s3_object" "transform_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" + etag = filemd5("${path.module}/../transform_function.zip") } resource "aws_lambda_function" "transform_lambda" { - function_name = "${var.transform_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "transform-lambda/transform_function.zip" - role = aws_iam_role.multi_service_role.arn # << lambda role placehodler - handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler - runtime = "python3.11" - environment { - variables = { - output = aws_s3_bucket.transform_bucket.bucket - } - } -} + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.transform" + runtime = "python3.11" -resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { - action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name - principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform_bucket.arn + lifecycle { + create_before_destroy = true + } + + depends_on = [aws_s3_object.transform_lambda_code] } -### LOAD LAMBDA SET UP -data "archive_file" "load_lambda_zip" { - type = "zip" - source_file = "${path.module}/../src/load_lambda.py" - output_path = "${path.module}/../load_function.zip" +# Load Lambda Function +resource "aws_s3_object" "load_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.load_lambda_name}/load_function.zip" + source = "${path.module}/../load_function.zip" + etag = filemd5("${path.module}/../load_function.zip") } resource "aws_lambda_function" "load_lambda" { - function_name = "${var.load_lambda_name}" - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = "load-lambda/load_function.zip" - role = aws_iam_role.multi_service_role.arn # << lambda role placehodler - handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler - runtime = "python3.11" -} + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.load" + runtime = "python3.11" + lifecycle { + create_before_destroy = true + } + + depends_on = [aws_s3_object.load_lambda_code] +} diff --git a/terraform/main.tf b/terraform/main.tf index 2a048b9..3b06701 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,26 +1,26 @@ terraform { required_providers { aws = { - source = "hashicorp/aws" - version = "~>5.0" + source = "hashicorp/aws" + version = "~>5.0" } } backend "s3" { bucket = "bentley-project-secrets" - key = "bentley-project/terraform.tfstate" + key = "bentley-project/terraform.tfstate" region = "eu-west-2" } } provider "aws" { - region = "eu-west-2" - default_tags { - tags = { - ProjectName = "Terrific-Totes" - Team = "Team-Bentley" - Environment = "Dev" - GitHubRepo = "de-project-bentley" - ManagedBy = "Terraform" - } + region = "eu-west-2" + default_tags { + tags = { + ProjectName = "Terrific-Totes" + Team = "Team-Bentley" + Environment = "Dev" + GitHubRepo = "de-project-bentley" + ManagedBy = "Terraform" } -} \ No newline at end of file + } +} diff --git a/terraform/s3.tf b/terraform/s3.tf index 4c06b8e..d5cdee3 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,51 +1,14 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { - bucket_prefix = "${var.s3_extract_bucket_name}-" + bucket_prefix = "${var.s3_extract_bucket_name}-" } -# resource "aws_s3_bucket_notification" "extract_bucket_notification" { -# bucket = aws_s3_bucket.extract_bucket.id -# lambda_function { -# lambda_function_arn = aws_lambda_function.extract_lambda.arn -# events = ["s3:ObjectCreated:*"] -# } -# depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] -# } # << is this the correct permission dependency? - ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { - bucket_prefix = "${var.s3_transform_bucket_name}-" + bucket_prefix = "${var.s3_transform_bucket_name}-" } -# resource "aws_s3_bucket_notification" "transform_bucket_notification" { -# bucket = aws_s3_bucket.transform_bucket.id -# lambda_function { -# lambda_function_arn = aws_lambda_function.transform_lambda.arn -# events = ["s3:ObjectCreated:*"] -# } -# depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] -# } # << is this the correct permission dependency? - - ### LAMBDA BUCKET resource "aws_s3_bucket" "lambda_code_bucket" { - bucket_prefix = "${var.s3_code_bucket_name}-" + bucket_prefix = "${var.s3_code_bucket_name}-" } - -# resource "aws_s3_object" "extract_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.extract_lambda_name}/extract_function.zip" -# source = "${path.module}/../extract_function.zip" -# } # << can't figure out how this is being used but we seem to need it - -# resource "aws_s3_object" "transform_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.transform_lambda_name}/transform_function.zip" -# source = "${path.module}/../transform_function.zip" -# } # << can't figure out how this is being used but we seem to need it - -# resource "aws_s3_object" "load_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.load_lambda_name}/load_function.zip" -# source = "${path.module}/../load_function.zip" -# } \ No newline at end of file diff --git a/terraform/vars.tf b/terraform/vars.tf index cc9348a..84824ce 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -1,33 +1,33 @@ variable "s3_extract_bucket_name" { - type = string - default = "extract-bucket" + type = string + default = "extract-bucket" } variable "s3_transform_bucket_name" { - type = string - default = "transform-bucket" + type = string + default = "transform-bucket" } variable "s3_code_bucket_name" { - type = string - default = "lambda-bucket" + type = string + default = "lambda-bucket" } variable "extract_lambda_name" { - type = string - default = "extract-lambda" + type = string + default = "extract-lambda" } variable "transform_lambda_name" { - type = string - default = "transform-lambda" + type = string + default = "transform-lambda" } variable "load_lambda_name" { - type = string - default = "load-lambda" + type = string + default = "load-lambda" } data "aws_caller_identity" "current" {} -data "aws_region" "current" {} \ No newline at end of file +data "aws_region" "current" {} -- cgit v1.2.3 From 43b7bca661a4c42acbfd6c27ff26fee9c25e911e Mon Sep 17 00:00:00 2001 From: Ellie Date: Wed, 14 Aug 2024 16:39:12 +0100 Subject: amend s3 handler to extract --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index a5da972..c190ea8 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -10,7 +10,7 @@ resource "aws_lambda_function" "extract_lambda" { s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = "extract-lambda/extract_function.zip" role = aws_iam_role.multi_service_role.arn #<< lambda role placehodler - handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + handler = "extract_lambda.extract" # << check that the function is called lambda handler runtime = "python3.11" environment { variables = { -- cgit v1.2.3 From 51eb46bbeb8fe5cb7b8750c37c776e8c9b4ae7f6 Mon Sep 17 00:00:00 2001 From: Ellie Date: Wed, 14 Aug 2024 16:41:02 +0100 Subject: uncomment s3 objects --- terraform/s3.tf | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/terraform/s3.tf b/terraform/s3.tf index 4c06b8e..8ab5622 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -32,20 +32,20 @@ resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } -# resource "aws_s3_object" "extract_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.extract_lambda_name}/extract_function.zip" -# source = "${path.module}/../extract_function.zip" -# } # << can't figure out how this is being used but we seem to need it - -# resource "aws_s3_object" "transform_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.transform_lambda_name}/transform_function.zip" -# source = "${path.module}/../transform_function.zip" -# } # << can't figure out how this is being used but we seem to need it - -# resource "aws_s3_object" "load_lambda_code" { -# bucket = aws_s3_bucket.lambda_code_bucket.bucket -# key = "${var.load_lambda_name}/load_function.zip" -# source = "${path.module}/../load_function.zip" -# } \ No newline at end of file +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_object" "transform_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_object" "load_lambda_code" { + bucket = aws_s3_bucket.lambda_code_bucket.bucket + key = "${var.load_lambda_name}/load_function.zip" + source = "${path.module}/../load_function.zip" +} \ No newline at end of file -- cgit v1.2.3 From 5cdcbd64e9f4dba5f3ed8e8eb9f6e91e1adde0ba Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 22:51:32 +0100 Subject: chore(gitignore): ignore .DS_Store --- .gitignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index d759665..d1df545 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ *.zip .terraform/ .terraform* -log* \ No newline at end of file +log* +.DS_Store -- cgit v1.2.3 From 9ff947c167932bb9ff93f05c8adf2ffcd98b91cc Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 22:52:33 +0100 Subject: infra(tf): simplify multi_service_role --- terraform/iam.tf | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index cf4902a..20aeab3 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -16,9 +16,7 @@ resource "aws_iam_role" "multi_service_role" { Principal = { Service = [ "lambda.amazonaws.com", - "cloudwatch.amazonaws.com", - "events.amazonaws.com", - "s3.amazonaws.com" + "scheduler.amazonaws.com" ] } } @@ -57,22 +55,22 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { ######################################################################## resource "aws_iam_policy" "lambda_execution_policy" { - name = "lambda_execution_policy" - path = "/" + name = "lambda_execution_policy" + path = "/" description = "IAM policy for Lambda execution" policy = jsonencode({ Version = "2012-10-17" Statement = [ - { + { Effect = "Allow" Action = [ "lambda:InvokeFunction", "lambda:GetFunction" ] Resource = "*" - } - ] + } + ] } ) } @@ -87,7 +85,7 @@ data "aws_iam_policy_document" "cw_document" { actions = ["logs:CreateLogGroup"] resources = [ "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" - ] + ] } statement { @@ -95,15 +93,15 @@ data "aws_iam_policy_document" "cw_document" { "logs:CreateLogStream", "logs:CreateLogGroup", "logs:PutLogEvents" - ] - resources = [ - "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" - ] + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] } } resource "aws_iam_policy" "cw_policy" { - name = "cw_policy" + name = "cw_policy" policy = data.aws_iam_policy_document.cw_document.json } @@ -128,17 +126,17 @@ resource "aws_iam_policy" "s3_write_policy" { # } resource "aws_iam_role_policy_attachment" "s3_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.s3_write_policy.arn } resource "aws_iam_role_policy_attachment" "lambda_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.lambda_execution_policy.arn } resource "aws_iam_role_policy_attachment" "cw_attachment" { - role = aws_iam_role.multi_service_role.name + role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.cw_policy.arn } -- cgit v1.2.3 From 3f89444cb09f0372b6a7621913944e372acd826c Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 22:58:50 +0100 Subject: infra(tf): temporarily remove events policies --- terraform/iam.tf | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 6c6b4fc..20aeab3 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -143,31 +143,3 @@ resource "aws_iam_role_policy_attachment" "cw_attachment" { ################ # RDS POLICIES # ################ - -################### -# EVENTS POLICIES # -################### - -data "aws_iam_policy_document" "cloudwatch_events_policy" { - statement { - actions = [ - "events:PutRule", - "events:PutTargets", - "events:RemoveTargets", - "events:DeleteRule", - "events:PutEvents" - ] - resources = ["*"] - effect = "Allow" - } -} - -resource "aws_iam_policy" "cloudwatch_events_policy" { - name = "cloudwatch_events_policy" - policy = data.aws_iam_policy_document.cloudwatch_events_policy.json -} - -resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { - role = aws_iam_role.multi_service_role.name - policy_arn = aws_iam_policy.cloudwatch_events_policy.arn -} -- cgit v1.2.3 From 8b975ba92d10c8034bdb35fee19c42b6187ea24f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 23:03:14 +0100 Subject: chore(cleanup): remove commented out code --- terraform/iam.tf | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 20aeab3..195b07c 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -114,17 +114,6 @@ resource "aws_iam_policy" "s3_write_policy" { policy = data.aws_iam_policy_document.s3_data_policy_doc.json } -# S3 ATTACH POLICY -# resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { -# for_each = toset([ -# aws_iam_policy.s3_write_policy.arn, -# aws_iam_policy.lambda_execution_policy.arn, -# aws_iam_policy.cw_policy.arn -# ]) -# role = aws_iam_role.multi_service_role.name -# policy_arn = each.value -# } - resource "aws_iam_role_policy_attachment" "s3_attachment" { role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.s3_write_policy.arn -- cgit v1.2.3 From 8ce75bcc1a89000fc0a9fea9b78ad31cd316ca06 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 23:04:32 +0100 Subject: chore(cleanup): remove unused rds section --- terraform/iam.tf | 3 --- 1 file changed, 3 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 195b07c..acb98f4 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -129,6 +129,3 @@ resource "aws_iam_role_policy_attachment" "cw_attachment" { policy_arn = aws_iam_policy.cw_policy.arn } -################ -# RDS POLICIES # -################ -- cgit v1.2.3 From c5f840e96b8c1696a6ed506e9260c4f1c26db10d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Wed, 14 Aug 2024 23:05:26 +0100 Subject: feat(tf): add cloudwatch events iam policies --- terraform/iam.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/terraform/iam.tf b/terraform/iam.tf index acb98f4..0e5fa6d 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -129,3 +129,30 @@ resource "aws_iam_role_policy_attachment" "cw_attachment" { policy_arn = aws_iam_policy.cw_policy.arn } +################### +# EVENTS POLICIES # +################### + +data "aws_iam_policy_document" "cloudwatch_events_policy" { + statement { + actions = [ + "events:PutRule", + "events:PutTargets", + "events:RemoveTargets", + "events:DeleteRule", + "events:PutEvents" + ] + resources = ["*"] + effect = "Allow" + } +} + +resource "aws_iam_policy" "cloudwatch_events_policy" { + name = "cloudwatch_events_policy" + policy = data.aws_iam_policy_document.cloudwatch_events_policy.json +} + +resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.cloudwatch_events_policy.arn +} -- cgit v1.2.3 From d78db8a0f99c778b1e4b401c7cb4a094a5f2b103 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 15 Aug 2024 00:24:41 +0100 Subject: fix(infra/tf): add unique id to permissions This was done to resolve issues with `ResourceConflictException` - add random_string resources for every lambda permission to add as suffixes - also add lifecycle options to force recreation whenever the random strings change --- terraform/events.tf | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index d2e2eb5..263141f 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,11 +1,25 @@ +resource "random_string" "eventbridge_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_ingestion_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_transform_suffix" { + length = 8 + special = false + upper = false +} + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" schedule_expression = "rate(30 minutes)" - - lifecycle { - create_before_destroy = true - } } resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { @@ -16,21 +30,28 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge" + statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn -} + lifecycle { + replace_triggered_by = [random_string.eventbridge_suffix] + } +} # below is step function 1 resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder + + lifecycle { + replace_triggered_by = [random_string.s3_ingestion_suffix] + } } @@ -45,14 +66,16 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { depends_on = [aws_lambda_permission.allow_s3_ingestion] } -###### - resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder + + lifecycle { + replace_triggered_by = [random_string.s3_transform_suffix] + } } -- cgit v1.2.3 From 79d230f3c76609d32a3d8c553d64f37ce9fe6d09 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Thu, 15 Aug 2024 10:15:31 +0100 Subject: fix(tf): create lambda zips before referencing --- terraform/lambda.tf | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index fb0a666..72d1306 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,9 +1,14 @@ # Extract Lambda Function +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} resource "aws_s3_object" "extract_lambda_code" { bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.extract_lambda_name}/extract_function.zip" - source = "${path.module}/../extract_function.zip" - etag = filemd5("${path.module}/../extract_function.zip") + source = data.archive_file.extract_lambda_zip.output_path + etag = filemd5(data.archive_file.extract_lambda_zip.output_path) } resource "aws_lambda_function" "extract_lambda" { @@ -22,11 +27,16 @@ resource "aws_lambda_function" "extract_lambda" { } # Transform Lambda Function +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} resource "aws_s3_object" "transform_lambda_code" { bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.transform_lambda_name}/transform_function.zip" - source = "${path.module}/../transform_function.zip" - etag = filemd5("${path.module}/../transform_function.zip") + source = data.archive_file.transform_lambda_zip.output_path + etag = filemd5(data.archive_file.transform_lambda_zip.output_path) } resource "aws_lambda_function" "transform_lambda" { @@ -45,11 +55,16 @@ resource "aws_lambda_function" "transform_lambda" { } # Load Lambda Function +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} resource "aws_s3_object" "load_lambda_code" { bucket = aws_s3_bucket.lambda_code_bucket.bucket key = "${var.load_lambda_name}/load_function.zip" - source = "${path.module}/../load_function.zip" - etag = filemd5("${path.module}/../load_function.zip") + source = data.archive_file.load_lambda_zip.output_path + etag = filemd5(data.archive_file.load_lambda_zip.output_path) } resource "aws_lambda_function" "load_lambda" { -- cgit v1.2.3 From fe548561acc5e133e3bee4026aab85db2e511bcd Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Thu, 15 Aug 2024 13:51:53 +0100 Subject: wip: secrets manager pushing to merge with extract_lambda --- .gitignore | 1 + src/extract_lambda.py | 1 + src/secrets_manager.py | 48 ++++++++++++++++++++++++++++++++++++++++++++ test/test_secrets_manager.py | 34 +++++++++++++++++++++++++++++++ 4 files changed, 84 insertions(+) create mode 100644 src/secrets_manager.py create mode 100644 test/test_secrets_manager.py diff --git a/.gitignore b/.gitignore index d1df545..d164c3f 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ .terraform* log* .DS_Store +venv \ No newline at end of file diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 7d56c66..faa1d30 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -2,6 +2,7 @@ from pg8000.native import Connection, Error, DatabaseError, InterfaceError from dotenv import load_dotenv import os + load_dotenv() def extract(): diff --git a/src/secrets_manager.py b/src/secrets_manager.py new file mode 100644 index 0000000..c0fb61e --- /dev/null +++ b/src/secrets_manager.py @@ -0,0 +1,48 @@ +import boto3 +from botocore.exceptions import ClientError +import json + + +def sm_client(): + sm_client = boto3.client('secretsmanager') + yield sm_client + +def create_secret(sm_client, secret_name, cohort_id, user, password, host, database, port): + secret = { + "cohort_id": cohort_id, + "user": user, + "password": password, + "host": host, + "database": database, + "port": port + } + + response = sm_client.create_secret( + Name = secret_name, + SecretString = json.dumps(secret) + ) + + print(response) + return response + +def list_secret(sm_client): + response = sm_client.list_secrets() + secret_dict = response['SecretList'] + secret_names = [] + for items in secret_dict: + secret_names.append(items['Name']) + print(f'{len(secret_names)} secret(s) available') + for name in secret_names: + print(name) + return secret_names + +def retrieve_secrets(sm_client): + response = sm_client.get_secrets( + + ) + + + +#retrieve secret +#so lambda can access totesy db +#so lambda connect to the db and then retrieve the data \ No newline at end of file diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py new file mode 100644 index 0000000..86533bc --- /dev/null +++ b/test/test_secrets_manager.py @@ -0,0 +1,34 @@ +from src.secrets_manager import sm_client, create_secret, list_secret +import boto3 +from moto import mock_aws +import json +import pytest +import os + +pytest.fixture(scope='class') +def mock_aws_credentials(): + """Mocked AWS Credentials for moto.""" + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + +@pytest.fixture(scope='class') +def mock_sm_client(mock_aws_credentials): + with mock_aws(): + yield boto3.client('secretsmanager') + + +def test_create_secret_stores_secrets(mock_sm_client): + cohort_id = "test_cohort_id" + user = "test_user_id" + password = "test_password" + host = "test_host" + database = "test_database" + port = "test_port" + + secret_name = "test_secret" + response = create_secret(mock_sm_client, secret_name, cohort_id, user, password, host, database, port) + + assert response['Name'] == secret_name \ No newline at end of file -- cgit v1.2.3