diff options
| -rwxr-xr-x | scripts/make_layer_zip.sh | 8 | ||||
| -rw-r--r-- | src/extract_lambda.py | 1 | ||||
| -rw-r--r-- | terraform/iam.tf | 17 | ||||
| -rw-r--r-- | terraform/lambda.tf | 22 | ||||
| -rw-r--r-- | terraform/s3.tf | 5 | ||||
| -rw-r--r-- | test/test_secrets_manager.py | 39 | ||||
| -rw-r--r-- | tests/test_extract_lambda.py | 34 |
7 files changed, 49 insertions, 77 deletions
diff --git a/scripts/make_layer_zip.sh b/scripts/make_layer_zip.sh new file mode 100755 index 0000000..eabe301 --- /dev/null +++ b/scripts/make_layer_zip.sh @@ -0,0 +1,8 @@ +# Description: Make the zip file for the layer + +cd "$(dirname "$0")/.." +mkdir -p python/lib/python3.11/site-packages +pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages +rm layer.zip +zip -r layer.zip python +rm -r python/ diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 9b17ef2..15fe785 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -90,6 +90,7 @@ def extract_bucket(client=boto3.client("s3")): extract_bucket_filter = [ bucket["Name"] for bucket in response["Buckets"] if "extract" in bucket["Name"] ] + return extract_bucket_filter[0] diff --git a/terraform/iam.tf b/terraform/iam.tf index 0e5fa6d..7585ff8 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -28,17 +28,19 @@ resource "aws_iam_role" "multi_service_role" { ######################################################################## # S3 SETUP # Description: allows allows retention/tagging/access control settings -# Lambda IAM Policy for S3 Write +# Lambda IAM Policy for S3 ######################################################################## # S3 DEFINE POLICY data "aws_iam_policy_document" "s3_data_policy_doc" { statement { + effect = "Allow" actions = [ "s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging", - "s3:PutObjectAcl" + "s3:PutObjectAcl", + "s3:ListObjects" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -46,6 +48,17 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "${aws_s3_bucket.lambda_code_bucket.arn}/*", ] } + + statement { + effect = "Allow" + actions = [ + "s3:ListBuckets", + "s3:ListAllMyBuckets" + ] + resources = [ + "arn:aws:s3:::*", + ] + } } diff --git a/terraform/lambda.tf b/terraform/lambda.tf index e33bc79..72aae04 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,27 +90,21 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "lambda_layer" - requirements = "requirements.txt" - layer_zip = "layer.zip" - layer_name = "lambda_layer_dev" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { provisioner "local-exec" { - command = <<EOT - cd ${local.layer_dir} - rm -rf python - mkdir python - pip3 install -r ${local.requirements} -t python/ - zip -r ${local.layer_zip} python - EOT - } #removed / at the end of python in line 99 + command = "bash ${local.script_dir}/make_layer_zip.sh" + } } resource "aws_s3_object" "lambda_layer_zip" { bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "lambda_layer/${local.layer_name}/${local.layer_zip}" + key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } @@ -118,7 +112,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] diff --git a/terraform/s3.tf b/terraform/s3.tf index b3a863c..d5cdee3 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,8 +12,3 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } - -### LAMBDA LAYER BUCKET -resource "aws_s3_bucket" "lambda_layer_bucket" { - bucket_prefix = "lambda-layer-dev-" -}
\ No newline at end of file diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py deleted file mode 100644 index cb4ec15..0000000 --- a/test/test_secrets_manager.py +++ /dev/null @@ -1,39 +0,0 @@ -from src.secrets_manager import sm_client, create_secret, list_secret -import boto3 -from moto import mock_aws -import json -import pytest -import os - -pytest.fixture(scope="class") - - -def mock_aws_credentials(): - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - os.environ["AWS_SECURITY_TOKEN"] = "testing" - os.environ["AWS_SESSION_TOKEN"] = "testing" - os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" - - -@pytest.fixture(scope="class") -def mock_sm_client(mock_aws_credentials): - with mock_aws(): - yield boto3.client("secretsmanager") - - -def test_create_secret_stores_secrets(mock_sm_client): - cohort_id = "test_cohort_id" - user = "test_user_id" - password = "test_password" - host = "test_host" - database = "test_database" - port = "test_port" - - secret_name = "test_secret" - response = create_secret( - mock_sm_client, secret_name, cohort_id, user, password, host, database, port - ) - - assert response["Name"] == secret_name diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 5a1c5b2..347ef22 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -1,3 +1,5 @@ +import boto3.exceptions +import botocore.exceptions import pytest import boto3 from moto import mock_aws @@ -35,7 +37,7 @@ def mock_config(): def aws_credentials(): os.environ["AWS_ACCESS_KEY_ID"] = "testing" os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - os.environ["AWS_SECURIT_TOKEN"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" os.environ["AWS_SESSION_TOKEN"] = "testing" os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" @@ -46,6 +48,15 @@ def s3_client(aws_credentials): yield boto3.client("s3") +@pytest.fixture(scope="class") +def s3_mock_bucket(s3_client): + bucket = s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + return bucket + + class TestLambdaHandler: def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): mock_db = MagicMock() @@ -138,27 +149,18 @@ class TestListExistingS3Files: list_existing_s3_files(client=s3_client) assert "Error listing S3 objects" in caplog.text - def test_error_if_bucket_is_empty(self, s3_client, caplog): - s3_client.create_bucket( - Bucket="extract_bucket", - CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, - ) + def test_error_if_bucket_is_empty(self, s3_client, caplog, s3_mock_bucket): list_existing_s3_files("extract_bucket", client=s3_client) assert "The bucket is empty" in caplog.text - def test_error_retrieving_object(self, s3_client, caplog): + def test_retrieves_file_content(self, s3_client, caplog, s3_mock_bucket): s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") - list_existing_s3_files(bucket_name="test_bucket", client=s3_client) - - assert "Error retrieving S3 object " in caplog.text - - def test_retrieves_file_content(self, s3_client, caplog): - result = list_existing_s3_files(client=s3_client) - + result = list_existing_s3_files("extract_bucket", client=s3_client) assert list(result.values()) == ["This is a test file."] class TestConnectToDatabase: + # had mock_config in param def test_connect_to_database(mock_conn, mock_config): with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() @@ -166,7 +168,7 @@ class TestConnectToDatabase: host="abc", user="def", port="5432", password="password", database="db" ) - def test_database_error(self, mock_config): + def test_database_error(self, mock_config): # had mock_config in param with pytest.raises(DBConnectionException): connect_to_database() @@ -178,7 +180,6 @@ class TestConnectToDatabase: connect_to_database() assert "Interface error" in caplog.text - class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): caplog.set_level(logging.INFO) @@ -218,4 +219,3 @@ class TestProcessAndUploadTables: # Assert that the log contains "No new data" assert "No new data" in caplog.text - # process and upload tables needs more tests |