diff options
| -rw-r--r-- | DEVNOTES.md | 117 | ||||
| -rw-r--r-- | events - TBD?.tf | 52 | ||||
| -rw-r--r-- | src/extract_lambda.py | 32 | ||||
| -rw-r--r-- | src/load_lambda.py | 0 | ||||
| -rw-r--r-- | src/transform_lambda.py | 0 | ||||
| -rw-r--r-- | terraform/events.tf | 71 | ||||
| -rw-r--r-- | terraform/iam.tf | 128 | ||||
| -rw-r--r-- | terraform/lambda.tf | 74 | ||||
| -rw-r--r-- | terraform/main.tf | 26 | ||||
| -rw-r--r-- | terraform/s3.tf | 40 | ||||
| -rw-r--r-- | terraform/vars.tf | 33 |
11 files changed, 537 insertions, 36 deletions
diff --git a/DEVNOTES.md b/DEVNOTES.md index 00f06ec..00b4ddd 100644 --- a/DEVNOTES.md +++ b/DEVNOTES.md @@ -1,55 +1,100 @@ # Workflow -## Commits +## References -### Make small and focused commits -- Please avoid mixing unrelated changes in a single commit -- Commit at regular points to revert changes easily if needed +https://nvie.com/posts/a-successful-git-branching-model/ \ +https://learn.microsoft.com/en-us/azure/devops/repos/git/merging-with-squash?view=azure-devops -### Write clear commit messages -- Limit subject line to 50 characters -- Provide more detailed explainations in the commit body (if required) -- Use the imperative mood in the subject line (e.g. 'add' instead of 'added') -``` -$ ~ git commit -``` +## Branching + +*Based off GitFlow but slightly modified* + +- There are two main branches + - `main` - production-ready code + - `development` - integration branch for features + - `staging` - represents the current staging state +- In addition, there are additional branches + - Feature branches - for new features and non-urgent bugfixes + - Hotfix branches - probably won't be used but for critical bugs in production (this is what testing should prevent) + - Release branches - for preparation of production releases + +- Feature branches - e.g. `feature/short-description` +- Bugfix branches - e.g. `bugfix/short-description` +- Hotfix branches - e.g. `hotfix/short-description` +- Release branches - e.g. `release/vX.Y.Z` +### Examples ``` -[Type]: [Short Subject] ----[Blank Line]--- -[Body, Limit to 72 Characters] +feature/add-data-extractor +bugfix/fix-s3-upload-error +hotfix/security-patch +release/v1.0.0 ``` -- Types: feat, fix, docs, style, refactor, test, chore, ci, perf - - See [here](https://eagerworks.com/blog/conventional-commits) for more information -## Branches +## Environments + +1. Development - where active development and initial testing occur +2. Staging - for integration testing and final checks before production +3. Production - live and stable environment + +## Deployment + +1. `main` - represents the current production state +2. `develop` - represents the integration branch for features and non-urgent fixes +3. `staging` - represents the current staging state + +## Staging Flow + +1. Create feature branches from `develop` & merge completed features back into `develop` +2. When the `develop` branch is ready for testing, create a `staging` branch from `develop` +3. Deploy the `staging` branch to the staging environment and perform our unit-tests +4. If staging tests pass, create a `release/vX.Y.Z` branch from `staging` +5. Make any final adjustments in the `release/vX.Y.Z` branch +6. Once we have approved the changes in the `release/vX.Y.Z` branch, merge into `main` +7. Tag the release in `main` -### Naming Conventions +### Notes -- Use lowercase with hyphens -- Include type and change with small description +- No new features should be included in the release branches and any new features should be merged into `develop` for the next release cycle +## Commit Messages + +Please follow the conventional commits specification: + ``` -[type]/[brief-description] :: e.g. feature/api +<type>[optional scope]: <description> + +<optional body> + +[optional footer(s)] ``` -### Base Branch -- Branch from `develop` for features and non-urgent fixes -- Branch off from `main` for urgent changes (project deadline) - this should be rarely used +### Types +- feat: new features +- fix: bugfixes +- docs: documentation-only changes +- style: changes that do not affect the meaning of the code +- refactor: code changes that neither fix bugs nor adds features +- perf: code changes that improve performance +- test: adding tests or correcting existing tests +- chore: changes to build process or tools/libraries (probably not needed) +- infra: changes to infrastructure configuration (e.g. Terraform) + +### Examples +``` +feat(extract): add automatic scheduling for data ingestion +docs: update README with project setup instructions +``` -### Keep branches updated +Configuration files for things such as Terraform isn't native to Conventional Commits, but we can add our own: -- Regularly merge and also delete branches when stale +``` +infra(tf): update S3 bucket policy +``` -## PRs +If the Terraform change involves a fix, you may combine `fix` and `infra`: -1. Create a pull request for each feature or fix (link to related issues) -2. Write a clear description which... - 1. Summarises the changes - 2. Explains the reasoning behind the changes - 3. Lists any areas of concerns (i.e. breaking changes) -3. Keep PRs focused - split changes into multiple PRs if needed -4. Assign someone to review -5. Merge ONLY after team approval - resolve conflicts & ensure CI checks pass -6. Use [squash and merge](https://learn.microsoft.com/en-us/azure/devops/repos/git/merging-with-squash?view=azure-devops) when needed to keep main branch history clean
\ No newline at end of file +``` +fix(infra): ... +``` diff --git a/events - TBD?.tf b/events - TBD?.tf new file mode 100644 index 0000000..25fb35b --- /dev/null +++ b/events - TBD?.tf @@ -0,0 +1,52 @@ +resource "aws_cloudwatch_event_rule" "lambda_trigger" { + name = "lambda-scheduled-trigger" + description = "Schedule to trigger the Lambda function" + schedule_expression = "rate(30 minutes)" + +# event_pattern = jsonencode({ +# detail-type = [ +# "AWS Console Sign In via CloudTrail" +# ] +# }) +} + + +resource "aws_cloudwatch_event_target" "lambda" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.my_lambda_function.arn +} + + + +resource "aws_lambda_permission" "allow_eventbridge" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.my_lambda_function.function_name + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn +} + + +# below is step function 1 +resource "aws_lambda_permission" "allow_s3_ingestion" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.lambda_transform.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract.arn +} + + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract.id + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.lambda_transform.arn + } + + depends_on = [aws_lambda_permission.allow_s3_ingestion] +} + +# need to duplicate and replace "2" with "3"
\ No newline at end of file diff --git a/src/extract_lambda.py b/src/extract_lambda.py new file mode 100644 index 0000000..7d56c66 --- /dev/null +++ b/src/extract_lambda.py @@ -0,0 +1,32 @@ +from pg8000.native import Connection, Error, DatabaseError, InterfaceError +from dotenv import load_dotenv +import os + +load_dotenv() + +def extract(): + +# temporary credentials for dev- will not have access when uploaded + + database = os.getenv('database') + user = os.getenv('user') + password = os.getenv('password') + host = os.getenv('host') + port = os.getenv('port') + + + try: + db = Connection.run( + database=database, + user=user, + password=password, + host=host, + port=port + ) + except DatabaseError as e: + print(e) + except InterfaceError as i: + print(i) + + +
\ No newline at end of file diff --git a/src/load_lambda.py b/src/load_lambda.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/load_lambda.py diff --git a/src/transform_lambda.py b/src/transform_lambda.py new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/src/transform_lambda.py diff --git a/terraform/events.tf b/terraform/events.tf new file mode 100644 index 0000000..0196dc3 --- /dev/null +++ b/terraform/events.tf @@ -0,0 +1,71 @@ +resource "aws_cloudwatch_event_rule" "lambda_trigger" { + name = "lambda-scheduled-trigger" + description = "Schedule to trigger the Lambda function" + schedule_expression = "rate(30 minutes)" + +# event_pattern = jsonencode({ +# detail-type = [ +# "AWS Console Sign In via CloudTrail" +# ] +# }) +} + + +resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { + rule = aws_cloudwatch_event_rule.lambda_trigger.name + target_id = "TargetFunctionV1" + arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder +} + + +resource "aws_lambda_permission" "allow_eventbridge" { + statement_id = "AllowExecutionFromEventBridge" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name #replaced lambda name placeholder + principal = "events.amazonaws.com" + source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn +} + + +# below is step function 1 +resource "aws_lambda_permission" "allow_s3_ingestion" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder +} + + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + } + + depends_on = [aws_lambda_permission.allow_s3_ingestion] +} + +###### + +resource "aws_lambda_permission" "allow_s3_transfrom_bucket" { + statement_id = "AllowS3InvokeLambdaTransform" + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder +} + + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder + + lambda_function { + events = ["s3:ObjectCreated:*"] + lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + } + + depends_on = [aws_lambda_permission.allow_s3_transform] +}
\ No newline at end of file diff --git a/terraform/iam.tf b/terraform/iam.tf new file mode 100644 index 0000000..bb8d932 --- /dev/null +++ b/terraform/iam.tf @@ -0,0 +1,128 @@ +# Description: This file contains the IAM roles and policies for the lambda functions +######################################################################## +# IAM MULTI-ROLE SETUP +######################################################################## + +# DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events) +resource "aws_iam_role" "bentley_multi_service_role" { + name = "multi_service_role" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Action = "sts:AssumeRole" + Effect = "Allow" + Principal = { + Service = [ + "lambda.amazonaws.com", + "states.amazonaws.com", + "events.amazonaws.com", + "s3.amazonaws.com" + ] + } + } + ] + }) +} + + + +######################################################################## +# S3 SETUP +# Description: allows allows retention/tagging/access control settings +# Lambda IAM Policy for S3 Write +######################################################################## + +# S3 DEFINE POLICY +resource "aws_iam_policy" "s3_access_policy" { + name = "s3_access_policy" + path = "/" + description = "IAM policy for S3 access" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ] + resources = [ + "${aws_s3_bucket.extract_bucket.arn}/*", + "${aws_s3_bucket.transform_bucket.arn}/*", + "${aws_s3_bucket.lambda_bucket.arn}/*" + ] + } + ] + } + ) +} + +######################################################################## +# LAMBDA SETUP +# Description: Allows Lambda permission to write to Cloudwatch logs +######################################################################## + +resource "aws_iam_policy" "lambda_execution_policy" { + name = "lambda_execution_policy" + path = "/" + description = "IAM policy for Lambda execution" + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Action = [ + "lambda:InvokeFunction", + "lambda:GetFunction" + ] + Resource = "*" + } + ] + } + ) +} + +######################################################################## +# CLOUDWATCH SETUP +# Description: Give permission for Lambda to write to CloudWatch logs +######################################################################## + +data "aws_iam_policy_document" "cw_document" { + statement { + actions = ["logs:CreateLogGroup"] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*" + ] + } + + statement { + actions = [ + "logs:CreateLogStream", + "logs:CreateLogGroup", + "logs:PutLogEvents" + ] + resources = [ + "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*" + ] + } +} + +######################################################################## +# POLICY WRITE & ATTACH +######################################################################## + +# S3 WRITE POLICY +resource "aws_iam_policy" "s3_write_policy" { + policy = data.aws_iam_policy_document.s3_data_policy_doc.json +} + +# S3 ATTACH POLICY +resource "aws_iam_role_policy_attachment" "lambda_s3_policy_attachment" { + role = aws_iam_role.lambda_role.name + policy_arn = aws_iam_policy.s3_write_policy.arn +}
\ No newline at end of file diff --git a/terraform/lambda.tf b/terraform/lambda.tf new file mode 100644 index 0000000..09d6697 --- /dev/null +++ b/terraform/lambda.tf @@ -0,0 +1,74 @@ +### EXTRACT LAMBDA SET UP +data "archive_file" "extract_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/extract_lambda.py" + output_path = "${path.module}/../extract_function.zip" +} + +resource "aws_lambda_function" "extract_lambda" { + function_name = "${var.extract_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "extract_lambda/extract_function.zip" + role = aws_iam_role.PLACEHOLDER_extract_lambda_role.arn # << lambda role placehodler + handler = "extract_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.extract_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_extract_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.extract_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.extract_bucket.arn +} + + +### TRANSFORM LAMBDA SET UP +data "archive_file" "transform_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/transform_lambda.py" + output_path = "${path.module}/../transform_function.zip" +} + +resource "aws_lambda_function" "transform_lambda" { + function_name = "${var.transform_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "transform_lambda/transform_function.zip" + role = aws_iam_role.PLACEHOLDER_transform_lambda_role.arn # << lambda role placehodler + handler = "transform_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" + environment { + variables = { + output = aws_s3_bucket.transform_bucket.bucket + } + } +} + +resource "aws_lambda_permission" "allow_to_write_to_s3_transform_bucket" { + action = "lambda:InvokeFunction" + function_name = aws_lambda_function.transform_lambda.function_name + principal = "s3.amazonaws.com" + source_arn = aws_s3_bucket.transform_bucket.arn +} + + +### LOAD LAMBDA SET UP +data "archive_file" "load_lambda_zip" { + type = "zip" + source_file = "${path.module}/../src/load_lambda.py" + output_path = "${path.module}/../load_function.zip" +} + +resource "aws_lambda_function" "load_lambda" { + function_name = "${var.load_lambda_name}" + s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_key = "load_lambda/load_function.zip" + role = aws_iam_role.PLACEHOLDER_load_lambda_role.arn # << lambda role placehodler + handler = "load_lambda.lambda_handler" # << check that the function is called lambda handler + runtime = "python3.11" +} + diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..3ca9a3d --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,26 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~>5.0" + } + } + backend "s3" { + bucket = "bentley-secrets" + key = "bentley-project/terraform.tfstate" + region = "eu-west-2" + } +} + +provider "aws" { + region = "eu-west-2" + default_tags { + tags = { + ProjectName = "Terrific-Totes" + Team = "Team-Bentley" + Environment = "Dev" + GitHubRepo = "de-project-bentley" + ManagedBy = "Terraform" + } + } +}
\ No newline at end of file diff --git a/terraform/s3.tf b/terraform/s3.tf new file mode 100644 index 0000000..8cb65ef --- /dev/null +++ b/terraform/s3.tf @@ -0,0 +1,40 @@ +### EXTRACT BUCKET SET-UP +resource "aws_s3_bucket" "extract_bucket" { + bucket = "${var.s3_extract_bucket_name}" +} + +resource "aws_s3_object" "extract_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.extract_lambda_name}/extract_function.zip" + source = "${path.module}/../extract_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "extract_bucket_notification" { + bucket = aws_s3_bucket.extract_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.extract_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_extract_bucket] +} # << is this the correct permission dependency? + + +### TRANSFORM BUCKET SET-UP +resource "aws_s3_bucket" "transform_bucket" { + bucket = "${var.s3_transform_bucket_name}" +} + +resource "aws_s3_object" "transform_lambda_code" { + bucket = aws_s3_bucket.s3_code_bucket_name.bucket + key = "${var.transform_lambda_name}/transform_function.zip" + source = "${path.module}/../transform_function.zip" +} # << can't figure out how this is being used but we seem to need it + +resource "aws_s3_bucket_notification" "transform_bucket_notification" { + bucket = aws_s3_bucket.transform_bucket.id + lambda_function { + lambda_function_arn = aws_lambda_function.transform_lambda.arn + events = ["s3:ObjectCreated:*"] + } + depends_on = [aws_lambda_permission.allow_to_write_to_s3_transform_bucket] +} # << is this the correct permission dependency? diff --git a/terraform/vars.tf b/terraform/vars.tf new file mode 100644 index 0000000..cc9348a --- /dev/null +++ b/terraform/vars.tf @@ -0,0 +1,33 @@ +variable "s3_extract_bucket_name" { + type = string + default = "extract-bucket" +} + +variable "s3_transform_bucket_name" { + type = string + default = "transform-bucket" +} + +variable "s3_code_bucket_name" { + type = string + default = "lambda-bucket" +} + +variable "extract_lambda_name" { + type = string + default = "extract-lambda" +} + +variable "transform_lambda_name" { + type = string + default = "transform-lambda" +} + +variable "load_lambda_name" { + type = string + default = "load-lambda" +} + +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {}
\ No newline at end of file |