Merge pull request #76 from ajschofield/development

pr: pull development into main

6 files changed, 607 insertions, 0 deletions

diff --git a/terraform/events.tf b/terraform/events.tf
new file mode 100644
index 0000000..53ae10a
--- /dev/null
+++ b/terraform/events.tf
@@ -0,0 +1,109 @@
+#################
+# Random String #
+#################
+
+resource "random_string" "eventbridge_suffix" {
+  length  = 8
+  special = false
+  upper   = false
+}
+
+resource "random_string" "s3_ingestion_suffix" {
+  length  = 8
+  special = false
+  upper   = false
+}
+
+resource "random_string" "s3_transform_suffix" {
+  length  = 8
+  special = false
+  upper   = false
+}
+
+#############################
+# EventBridge Configuration #
+#############################
+
+resource "aws_cloudwatch_event_rule" "lambda_trigger" {
+  name                = "lambda-scheduled-trigger"
+  description         = "Schedule to trigger the Lambda function"
+  schedule_expression = "rate(30 minutes)"
+}
+
+resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" {
+  rule       = aws_cloudwatch_event_rule.lambda_trigger.name
+  target_id  = "TargetFunctionV1"
+  arn        = aws_lambda_function.extract_lambda.arn
+  depends_on = [aws_lambda_permission.allow_eventbridge]
+}
+
+resource "aws_lambda_permission" "allow_eventbridge" {
+  statement_id  = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.extract_lambda.function_name
+  principal     = "events.amazonaws.com"
+  source_arn    = aws_cloudwatch_event_rule.lambda_trigger.arn
+
+  lifecycle {
+    create_before_destroy = true
+    replace_triggered_by  = [random_string.eventbridge_suffix]
+  }
+}
+
+########################################
+# S3 Extract Bucket Notification Setup #
+########################################
+
+resource "aws_lambda_permission" "allow_s3_ingestion" {
+  statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.transform_lambda.function_name
+  principal     = "s3.amazonaws.com"
+  source_arn    = aws_s3_bucket.extract_bucket.arn
+
+  lifecycle {
+    create_before_destroy = true
+    replace_triggered_by  = [random_string.s3_ingestion_suffix]
+  }
+}
+
+
+resource "aws_s3_bucket_notification" "extract_bucket_notification" {
+  bucket = aws_s3_bucket.extract_bucket.id
+
+  lambda_function {
+    events              = ["s3:ObjectCreated:*"]
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn
+  }
+
+  depends_on = [aws_lambda_permission.allow_s3_ingestion]
+}
+
+##########################################
+# S3 Transform Bucket Notification Setup #
+##########################################
+
+resource "aws_lambda_permission" "allow_s3_transform_bucket" {
+  statement_id  = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}"
+  action        = "lambda:InvokeFunction"
+  function_name = aws_lambda_function.transform_lambda.function_name
+  principal     = "s3.amazonaws.com"
+  source_arn    = aws_s3_bucket.transform_bucket.arn
+
+  lifecycle {
+    create_before_destroy = true
+    replace_triggered_by  = [random_string.s3_transform_suffix]
+  }
+}
+
+
+resource "aws_s3_bucket_notification" "transform_bucket_notification" {
+  bucket = aws_s3_bucket.transform_bucket.id
+
+  lambda_function {
+    events              = ["s3:ObjectCreated:*"]
+    lambda_function_arn = aws_lambda_function.transform_lambda.arn
+  }
+
+  depends_on = [aws_lambda_permission.allow_s3_transform_bucket]
+}
diff --git a/terraform/iam.tf b/terraform/iam.tf
new file mode 100644
index 0000000..3d62b69
--- /dev/null
+++ b/terraform/iam.tf
@@ -0,0 +1,202 @@
+# Description: This file contains the IAM roles and policies for the lambda functions
+########################################################################
+# IAM MULTI-ROLE SETUP
+########################################################################
+
+# DEFINE MULTI-SERVICE ROLE (lambda, s3, cloudwatch, events)
+resource "aws_iam_role" "multi_service_role" {
+  name = "multi_service_role"
+
+  assume_role_policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Action = "sts:AssumeRole"
+        Effect = "Allow"
+        Principal = {
+          Service = [
+            "lambda.amazonaws.com",
+            "scheduler.amazonaws.com"
+          ]
+        }
+      }
+    ]
+  })
+}
+
+
+########################################################################
+# S3 SETUP                                                        
+# Description: allows allows retention/tagging/access control settings
+# Lambda IAM Policy for S3
+########################################################################
+
+# S3 DEFINE POLICY
+data "aws_iam_policy_document" "s3_data_policy_doc" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:PutObject",
+      "s3:PutObjectRetention",
+      "s3:PutObjectTagging",
+      "s3:PutObjectAcl",
+      "s3:ListObjects",
+      "s3:ListObjectsV2",
+      "s3:GetObject"
+    ]
+    resources = [
+      "${aws_s3_bucket.extract_bucket.arn}/*",
+      "${aws_s3_bucket.transform_bucket.arn}/*",
+      "${aws_s3_bucket.lambda_code_bucket.arn}/*",
+    ]
+  }
+
+  statement {
+    effect = "Allow"
+    actions = [
+      "s3:ListBucket",
+      "s3:ListAllMyBuckets",
+      "s3:ListObjectsV2",
+      "s3:ListObjects"
+    ]
+    resources = [
+      "arn:aws:s3:::*",
+    ]
+  }
+}
+
+
+########################################################################
+# LAMBDA SETUP
+# Description: Allows Lambda permission to write to Cloudwatch logs
+########################################################################
+
+resource "aws_iam_policy" "lambda_execution_policy" {
+  name        = "lambda_execution_policy"
+  path        = "/"
+  description = "IAM policy for Lambda execution"
+
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [
+      {
+        Effect = "Allow"
+        Action = [
+          "lambda:InvokeFunction",
+          "lambda:GetFunction"
+        ]
+        Resource = "*"
+      }
+    ]
+    }
+  )
+}
+
+########################################################################
+# CLOUDWATCH SETUP
+# Description: Give permission for Lambda to write to CloudWatch logs
+########################################################################
+
+data "aws_iam_policy_document" "cw_document" {
+  statement {
+    actions = ["logs:CreateLogGroup"]
+    resources = [
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:*"
+    ]
+  }
+
+  statement {
+    actions = [
+      "logs:CreateLogStream",
+      "logs:CreateLogGroup",
+      "logs:PutLogEvents"
+    ]
+    resources = [
+      "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/lambda/*"
+    ]
+  }
+}
+
+resource "aws_iam_policy" "cw_policy" {
+  name   = "cw_policy"
+  policy = data.aws_iam_policy_document.cw_document.json
+}
+
+########################################################################
+# POLICY WRITE & ATTACH
+########################################################################
+
+# S3 WRITE POLICY
+resource "aws_iam_policy" "s3_write_policy" {
+  policy = data.aws_iam_policy_document.s3_data_policy_doc.json
+}
+
+resource "aws_iam_role_policy_attachment" "s3_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.s3_write_policy.arn
+}
+
+resource "aws_iam_role_policy_attachment" "lambda_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.lambda_execution_policy.arn
+}
+
+resource "aws_iam_role_policy_attachment" "cw_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.cw_policy.arn
+}
+
+###################
+# EVENTS POLICIES #
+###################
+
+data "aws_iam_policy_document" "cloudwatch_events_policy" {
+  statement {
+    actions = [
+      "events:PutRule",
+      "events:PutTargets",
+      "events:RemoveTargets",
+      "events:DeleteRule",
+      "events:PutEvents"
+    ]
+    resources = ["*"]
+    effect    = "Allow"
+  }
+}
+
+resource "aws_iam_policy" "cloudwatch_events_policy" {
+  name   = "cloudwatch_events_policy"
+  policy = data.aws_iam_policy_document.cloudwatch_events_policy.json
+}
+
+resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.cloudwatch_events_policy.arn
+}
+
+#########################
+# SECRETS MANAGER SETUP #
+#########################
+
+# Policy Doc
+data "aws_iam_policy_document" "secrets_manager_policy_doc" {
+  statement {
+    effect = "Allow"
+    actions = [
+      "secretsmanager:GetSecretValue"
+    ]
+    resources = ["arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:bentley-secrets-Na0yc8"]
+  }
+}
+
+# SM Policy Resource
+resource "aws_iam_policy" "secrets_manager_policy" {
+  name   = "secrets_manager_policy"
+  policy = data.aws_iam_policy_document.secrets_manager_policy_doc.json
+}
+
+# Attach SM Policy to Role
+resource "aws_iam_role_policy_attachment" "secrets_manager_attachment" {
+  role       = aws_iam_role.multi_service_role.name
+  policy_arn = aws_iam_policy.secrets_manager_policy.arn
+}
diff --git a/terraform/lambda.tf b/terraform/lambda.tf
new file mode 100644
index 0000000..f8e7515
--- /dev/null
+++ b/terraform/lambda.tf
@@ -0,0 +1,146 @@
+####################
+# Common Variables #
+####################
+
+locals {
+  layer_dir      = "../"
+  layer_zip      = "layer.zip"
+  layer_name     = "lambda_layer"
+  script_dir     = "../scripts"
+  layer_zip_path = "${local.layer_dir}/${local.layer_zip}"
+}
+
+######################
+# Lambda Layer Setup #
+######################
+
+resource "null_resource" "prepare_layer" {
+
+  # New change: only run the script if the layer zip does not exist
+
+  triggers = {
+    layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists"
+  }
+
+  provisioner "local-exec" {
+    command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi"
+  }
+}
+
+resource "aws_s3_object" "lambda_layer_zip" {
+  bucket     = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id
+  key        = "${local.layer_name}/${local.layer_zip}"
+  source     = "${local.layer_dir}/${local.layer_zip}"
+  depends_on = [null_resource.prepare_layer]
+  etag       = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null
+}
+
+resource "aws_lambda_layer_version" "lambda_layer" {
+  layer_name          = local.layer_name
+  compatible_runtimes = ["python3.11"]
+  s3_bucket           = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key              = aws_s3_object.lambda_layer_zip.key
+  source_code_hash    = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null
+  skip_destroy        = true
+  depends_on          = [aws_s3_object.lambda_layer_zip]
+}
+
+###########################
+# Extract Lambda Function #
+###########################
+
+data "archive_file" "extract_lambda_zip" {
+  type        = "zip"
+  source_file = "${path.module}/../src/extract_lambda.py"
+  output_path = "${path.module}/../extract_function.zip"
+}
+resource "aws_s3_object" "extract_lambda_code" {
+  bucket = aws_s3_bucket.lambda_code_bucket.bucket
+  key    = "${var.extract_lambda_name}/extract_function.zip"
+  source = data.archive_file.extract_lambda_zip.output_path
+  etag   = filemd5(data.archive_file.extract_lambda_zip.output_path)
+}
+
+resource "aws_lambda_function" "extract_lambda" {
+  function_name    = var.extract_lambda_name
+  s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key           = aws_s3_object.extract_lambda_code.key
+  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  role             = aws_iam_role.multi_service_role.arn
+  handler          = "extract_lambda.lambda_handler"
+  runtime          = "python3.11"
+  source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  depends_on = [aws_s3_object.extract_lambda_code]
+}
+
+#############################
+# Transform Lambda Function #
+#############################
+
+data "archive_file" "transform_lambda_zip" {
+  type        = "zip"
+  source_file = "${path.module}/../src/transform_lambda.py"
+  output_path = "${path.module}/../transform_function.zip"
+}
+resource "aws_s3_object" "transform_lambda_code" {
+  bucket = aws_s3_bucket.lambda_code_bucket.bucket
+  key    = "${var.transform_lambda_name}/transform_function.zip"
+  source = data.archive_file.transform_lambda_zip.output_path
+  etag   = filemd5(data.archive_file.transform_lambda_zip.output_path)
+}
+
+resource "aws_lambda_function" "transform_lambda" {
+  function_name    = var.transform_lambda_name
+  s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key           = aws_s3_object.transform_lambda_code.key
+  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  role             = aws_iam_role.multi_service_role.arn
+  handler          = "transform_lambda.lambda_handler"
+  runtime          = "python3.11"
+  source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  depends_on = [aws_s3_object.transform_lambda_code]
+}
+
+########################
+# Load Lambda Function #
+########################
+
+data "archive_file" "load_lambda_zip" {
+  type        = "zip"
+  source_file = "${path.module}/../src/load_lambda.py"
+  output_path = "${path.module}/../load_function.zip"
+}
+resource "aws_s3_object" "load_lambda_code" {
+  bucket = aws_s3_bucket.lambda_code_bucket.bucket
+  key    = "${var.load_lambda_name}/load_function.zip"
+  source = data.archive_file.load_lambda_zip.output_path
+  etag   = filemd5(data.archive_file.load_lambda_zip.output_path)
+}
+
+resource "aws_lambda_function" "load_lambda" {
+  function_name    = var.load_lambda_name
+  s3_bucket        = aws_s3_bucket.lambda_code_bucket.bucket
+  s3_key           = aws_s3_object.load_lambda_code.key
+  layers           = [aws_lambda_layer_version.lambda_layer.arn]
+  role             = aws_iam_role.multi_service_role.arn
+  handler          = "load_lambda.lambda_handler"
+  runtime          = "python3.11"
+  source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256
+
+  lifecycle {
+    create_before_destroy = true
+  }
+
+  depends_on = [aws_s3_object.load_lambda_code]
+}
+
diff --git a/terraform/main.tf b/terraform/main.tf
new file mode 100644
index 0000000..6577b70
--- /dev/null
+++ b/terraform/main.tf
@@ -0,0 +1,40 @@
+terraform {
+  required_version = ">= 1.8.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "~>5.0"
+    }
+    null = {
+      source  = "hashicorp/null"
+      version = "~>3.2.2"
+    }
+    archive = {
+      source  = "hashicorp/archive"
+      version = "~>2.5.0"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~>3.6.2"
+    }
+  }
+  backend "s3" {
+    bucket  = "bentley-project-secrets"
+    key     = "bentley-project/terraform.tfstate"
+    region  = "eu-west-2"
+    encrypt = true
+  }
+}
+
+provider "aws" {
+  region = "eu-west-2"
+  default_tags {
+    tags = {
+      ProjectName = var.project_name
+      Environment = var.environment
+      ManagedBy   = "Terraform"
+      GitHubRepo  = var.github_repo
+      Team        = var.team_name
+    }
+  }
+}
diff --git a/terraform/s3.tf b/terraform/s3.tf
new file mode 100644
index 0000000..14e8835
--- /dev/null
+++ b/terraform/s3.tf
@@ -0,0 +1,57 @@
+########################
+# EXTRACT BUCKET SETUP #
+########################
+
+resource "aws_s3_bucket" "extract_bucket" {
+  bucket_prefix = "${var.s3_extract_bucket_name}-"
+  force_destroy = true
+  tags = {
+    Name = "Ingestion Bucket"
+  }
+}
+
+resource "aws_s3_bucket_versioning" "extract_bucket_versioning" {
+  bucket = aws_s3_bucket.extract_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+##########################
+# TRANSFORM BUCKET SETUP #
+##########################
+
+resource "aws_s3_bucket" "transform_bucket" {
+  bucket_prefix = "${var.s3_transform_bucket_name}-"
+  force_destroy = true
+  tags = {
+    Name = "Transform Bucket"
+  }
+}
+
+
+resource "aws_s3_bucket_versioning" "transform_bucket_versioning" {
+  bucket = aws_s3_bucket.transform_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
+
+#######################
+# LAMBDA BUCKET SETUP #
+#######################
+
+resource "aws_s3_bucket" "lambda_code_bucket" {
+  bucket_prefix = "${var.s3_code_bucket_name}-"
+  force_destroy = true
+  tags = {
+    Name = "Lambda Bucket"
+  }
+}
+
+resource "aws_s3_bucket_versioning" "lambda_bucket_versioning" {
+  bucket = aws_s3_bucket.lambda_code_bucket.id
+  versioning_configuration {
+    status = "Enabled"
+  }
+}
diff --git a/terraform/vars.tf b/terraform/vars.tf
new file mode 100644
index 0000000..b3e3e47
--- /dev/null
+++ b/terraform/vars.tf
@@ -0,0 +1,53 @@
+variable "s3_extract_bucket_name" {
+  type    = string
+  default = "extract-bucket"
+}
+
+variable "s3_transform_bucket_name" {
+  type    = string
+  default = "transform-bucket"
+}
+
+variable "s3_code_bucket_name" {
+  type    = string
+  default = "lambda-bucket"
+}
+
+variable "extract_lambda_name" {
+  type    = string
+  default = "extract-lambda"
+}
+
+variable "transform_lambda_name" {
+  type    = string
+  default = "transform-lambda"
+}
+
+variable "load_lambda_name" {
+  type    = string
+  default = "load-lambda"
+}
+
+variable "project_name" {
+  type    = string
+  default = "tt"
+}
+
+variable "environment" {
+  type    = string
+  default = "dev"
+}
+
+variable "github_repo" {
+  type    = string
+  default = "de-project-bentley"
+}
+
+variable "team_name" {
+  type    = string
+  default = "Team-Bentley"
+}
+
+data "aws_caller_identity" "current" {}
+
+data "aws_region" "current" {}