diff options
Diffstat (limited to 'terraform')
| -rw-r--r-- | terraform/events.tf | 23 | ||||
| -rw-r--r-- | terraform/lambda.tf | 74 | ||||
| -rw-r--r-- | terraform/s3.tf | 39 | ||||
| -rw-r--r-- | terraform/vars.tf | 5 |
4 files changed, 121 insertions, 20 deletions
diff --git a/terraform/events.tf b/terraform/events.tf index 4d68a23..0196dc3 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -11,18 +11,17 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { } -resource "aws_cloudwatch_event_target" "lambda" { +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder } - resource "aws_lambda_permission" "allow_eventbridge" { statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name + function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn } @@ -32,18 +31,18 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn + source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id + bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -54,18 +53,18 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform.arn + source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform.id + bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_transform] diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..09d6697 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,74 @@ +### EXTRACT LAMBDA SET UP +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} + +resource "aws_lambda_function" "extract_lambda" { + function_name = "${var.extract_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "extract_lambda/extract_function.zip" + role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.extract_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn +} + + +### TRANSFORM LAMBDA SET UP +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} + +resource "aws_lambda_function" "transform_lambda" { + function_name = "${var.transform_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "transform_lambda/transform_function.zip" + role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.transform_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn +} + + +### LOAD LAMBDA SET UP +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} + +resource "aws_lambda_function" "load_lambda" { + function_name = "${var.load_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "load_lambda/load_function.zip" + role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" +} + diff --git a/terraform/s3.tf b/terraform/s3.tf index bfe891e..8cb65ef 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,17 +1,40 @@ +### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket = "${var.s3_extract_bucket_name}" } +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.extract_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +} # << is this the correct permission dependency? + + +### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket = "${var.s3_transform_bucket_name}" } -resource "aws_s3_bucket" "lambda_bucket" { - bucket = "${var.s3_code_bucket_name}" -} - -resource "aws_s3_object" "extract_lambda_code" { +resource "aws_s3_object" "transform_lambda_code" { bucket = aws_s3_bucket.s3_code_bucket_name.bucket - key = "${var.extract_lambda_name}/function_e.zip" - source = "${path.module}/../function_e.zip" -}
\ No newline at end of file + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.transform_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +} # << is this the correct permission dependency? diff --git a/terraform/vars.tf b/terraform/vars.tf index fa84222..cc9348a 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -23,6 +23,11 @@ variable "transform_lambda_name" { default = "transform-lambda" } +variable "load_lambda_name" { + type = string + default = "load-lambda" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {}
\ No newline at end of file |