diff options
| -rw-r--r-- | events - TBD?.tf (renamed from events.tf) | 0 | ||||
| -rw-r--r-- | src/extract_lambda.py | 32 | ||||
| -rw-r--r-- | terraform/events.tf | 23 | ||||
| -rw-r--r-- | terraform/iam.tf | 128 | ||||
| -rw-r--r-- | terraform/s3.tf | 39 |
5 files changed, 202 insertions, 20 deletions
diff --git a/events.tf b/events - TBD?.tf index 25fb35b..25fb35b 100644 --- a/events.tf +++ b/events - TBD?.tf diff --git a/src/extract_lambda.py b/src/extract_lambda.py index e69de29..7d56c66 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -0,0 +1,32 @@ +from pg8000.native import Connection, Error, DatabaseError, InterfaceError +from dotenv import load_dotenv +import os + +load_dotenv() + +def extract(): + +# temporary credentials for dev- will not have access when uploaded + + database = os.getenv('database') + user = os.getenv('user') + password = os.getenv('password') + host = os.getenv('host') + port = os.getenv('port') + + + try: + db = Connection.run( + database=database, + user=user, + password=password, + host=host, + port=port + ) + except DatabaseError as e: + print(e) + except InterfaceError as i: + print(i) + + +
\ No newline at end of file diff --git a/terraform/events.tf b/terraform/events.tf index 4d68a23..0196dc3 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -11,18 +11,17 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { } -resource "aws_cloudwatch_event_target" "lambda" { +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.my_lambda_function.arn + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder } - resource "aws_lambda_permission" "allow_eventbridge" { statement_id = "AllowExecutionFromEventBridge" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.my_lambda_function.function_name + function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn } @@ -32,18 +31,18 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract.arn + source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract.id + bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -54,18 +53,18 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { statement_id = "AllowS3InvokeLambdaTransform" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.lambda_transform.function_name + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform.arn + source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder } resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform.id + bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.lambda_transform.arn + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder } depends_on = [aws_lambda_permission.allow_s3_transform] diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..bb8d932 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,128 @@ +# Description: This file contains the IAM roles and policies for the lambda functions +######################################################################## +# IAM MULTI-ROLE SETUP +######################################################################## + +# DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events) +resource "aws_iam_role" "bentley_multi_service_role" { + name = "multi_service_role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = [ + "lambda.amazonaws.com", + "states.amazonaws.com", + "events.amazonaws.com", + "s3.amazonaws.com" + ] + } + } + ] + }) +} + + + +######################################################################## +# S3 SETUP +# Description: allows allows retention/tagging/access control settings +# Lambda IAM Policy for S3 Write +######################################################################## + +# S3 DEFINE POLICY +resource "aws_iam_policy" "s3_access_policy" { + name = "s3_access_policy" + path = "/" + description = "IAM policy for S3 access" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ] + resources = [ + "${aws_s3_bucket.extract_bucket.arn}/*", + "${aws_s3_bucket.transform_bucket.arn}/*", + "${aws_s3_bucket.lambda_bucket.arn}/*" + ] + } + ] + } + ) +} + +######################################################################## +# LAMBDA SETUP +# Description: Allows Lambda permission to write to Cloudwatch logs +######################################################################## + +resource "aws_iam_policy" "lambda_execution_policy" { + name = "lambda_execution_policy" + path = "/" + description = "IAM policy for Lambda execution" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "lambda:InvokeFunction", + "lambda:GetFunction" + ] + Resource = "*" + } + ] + } + ) +} + +######################################################################## +# CLOUDWATCH SETUP +# Description: Give permission for Lambda to write to CloudWatch logs +######################################################################## + +data "aws_iam_policy_document" "cw_document" { + statement { + actions = ["logs:CreateLogGroup"] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" + ] + } + + statement { + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] + } +} + +######################################################################## +# POLICY WRITE & ATTACH +######################################################################## + +# S3 WRITE POLICY +resource "aws_iam_policy" "s3_write_policy" { + policy = data.aws_iam_policy_document.s3_data_policy_doc.json +} + +# S3 ATTACH POLICY +resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.s3_write_policy.arn +}
\ No newline at end of file diff --git a/terraform/s3.tf b/terraform/s3.tf index bfe891e..8cb65ef 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,17 +1,40 @@ +### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket = "${var.s3_extract_bucket_name}" } +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.extract_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +} # << is this the correct permission dependency? + + +### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket = "${var.s3_transform_bucket_name}" } -resource "aws_s3_bucket" "lambda_bucket" { - bucket = "${var.s3_code_bucket_name}" -} - -resource "aws_s3_object" "extract_lambda_code" { +resource "aws_s3_object" "transform_lambda_code" { bucket = aws_s3_bucket.s3_code_bucket_name.bucket - key = "${var.extract_lambda_name}/function_e.zip" - source = "${path.module}/../function_e.zip" -}
\ No newline at end of file + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.transform_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +} # << is this the correct permission dependency? |