From 0727dab70cb56521b73c04ab8e378b7f165fc224 Mon Sep 17 00:00:00 2001 From: T-Aji Date: Fri, 16 Aug 2024 14:07:05 +0100 Subject: test: passing lambda_handler both no_changes and with changes to files --- tests/test_extract_lambda.py | 46 +++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index e94a8a4..4b61b83 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -3,9 +3,10 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables +from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, lambda_handler, process_and_upload_tables import os import logging +import json @pytest.fixture(scope='class') def mock_config(): @@ -33,6 +34,49 @@ def s3_client(aws_credentials): with mock_aws(): yield boto3.client('s3') +class TestLambdaHandler: + def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): + mock_db = MagicMock() + mock_db.run.side_effect = [ + [['Fruits']], + [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], + [['Food_type'], ['Flavour'], ['Colour']] + ] + mock_db.columns.return_value = [{'name': 'Food_type'}, {'name': 'Flavour'}, {'name': 'Colour'}] + with patch("src.extract_lambda.connect_to_database", return_value=mock_db): + mock_process_and_upload_tables = mocker.patch("src.extract_lambda.process_and_upload_tables", return_value=mock_db) + mock_list_existing_s3_files = mocker.patch("src.extract_lambda.list_existing_s3_files", return_value={}) + event = {} + context = {} + response = lambda_handler(event, context) + assert response['statusCode'] == 200 + assert json.loads(response['body']) == 'CSV files processed and uploaded successfully.' + mock_list_existing_s3_files.assert_called_once() + mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) + mock_db.close.assert_called_once() + + def test_lambda_handler_no_changes_detected_no_files_uploaded(self, mocker): + mock_db = MagicMock() + mock_db.run.side_effect = [ + [['Fruits']], + [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], + [['Food_type'], ['Flavour'], ['Colour']] + ] + mock_db.columns.return_value = [{'name': 'Food_type'}, {'name': 'Flavour'}, {'name': 'Colour'}] + + with patch("src.extract_lambda.connect_to_database", return_value=mock_db): + mock_process_and_upload_tables = mocker.patch("src.extract_lambda.process_and_upload_tables", return_value=False) + mock_list_existing_s3_files = mocker.patch("src.extract_lambda.list_existing_s3_files", return_value={}) + event = {} + context = {} + response = lambda_handler(event, context) + assert response['statusCode'] == 200 + assert json.loads(response['body']) == 'No changes detected, no CSV files were uploaded.' + mock_list_existing_s3_files.assert_called_once() + mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) + mock_db.close.assert_called_once() + + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): -- cgit v1.2.3 From 1e27974ecc48d8611b87af1b9cd51e29afa8c792 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Fri, 16 Aug 2024 17:15:59 +0100 Subject: test(fx): fix prepare_layer - broken --- terraform/lambda.tf | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 67fd6eb..27e6266 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -89,14 +89,13 @@ locals { } resource "null_resource" "prepare_layer" { - triggers = { - requirements_hash = filesha1(local.requirements) - } provisioner "local-exec" { command = < Date: Fri, 16 Aug 2024 21:06:51 +0100 Subject: docs: add badges to README --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 6bc75dc..cbb446c 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,13 @@ # ToteSys - Data Engineering Project +[![Python](https://img.shields.io/badge/Python-FFD43B?style=for-the-badge&logo=python&logoColor=blue)](https://www.python.org/) +[![AWS](https://img.shields.io/badge/Amazon_AWS-FF9900?style=for-the-badge&logo=amazonaws&logoColor=white)](https://aws.amazon.com/) +[![Terraform](https://img.shields.io/badge/Terraform-7B42BC?style=for-the-badge&logo=terraform&logoColor=white)](https://www.terraform.io/) +[![Postgresql](https://img.shields.io/badge/PostgreSQL-316192?style=for-the-badge&logo=postgresql&logoColor=white)](https://www.postgresql.org/) +[![GitHub Actions](https://img.shields.io/badge/GitHub_Actions-2088FF?style=for-the-badge&logo=github-actions&logoColor=white)](https://github.com/features/actions) + +[![Terraform Main Deployment Workflow Status](https://img.shields.io/github/actions/workflow/status/ajschofield/de-project-bentley/deploy.yml?branch=main&style=flat-square&label=deploy)](https://github.com/ajschofield/de-project-bentley/actions/workflows/deploy.yml?query=branch%3Amain) +[![Production Environment Status](https://img.shields.io/github/deployments/ajschofield/de-project-bentley/production?style=flat-square&label=env)](https://github.com/ajschofield/de-project-bentley/deployments/production) # Summary The project aims to implement a data platform that can extract data from an operational database, archive it in a data lake, and make it easily accessible -- cgit v1.2.3 From 95ad71be4315f5ae3f9183f66049ae8b8cf914fc Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 16 Aug 2024 20:07:43 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 9dabc89 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/52 --- src/load_lambda.py | 2 +- src/secrets_manager.py | 31 +++++++++---------- src/transform_lambda.py | 2 +- test/test_secrets_manager.py | 19 +++++++----- tests/test_extract_lambda.py | 69 ++++++++++++++++++++++++------------------- tests/test_secrets_manager.py | 37 +++++++++++++++-------- 6 files changed, 93 insertions(+), 67 deletions(-) diff --git a/src/load_lambda.py b/src/load_lambda.py index 6ee681f..c6a8e60 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -1,2 +1,2 @@ def lambda_handler(): - pass \ No newline at end of file + pass diff --git a/src/secrets_manager.py b/src/secrets_manager.py index c0fb61e..3484688 100644 --- a/src/secrets_manager.py +++ b/src/secrets_manager.py @@ -4,45 +4,46 @@ import json def sm_client(): - sm_client = boto3.client('secretsmanager') + sm_client = boto3.client("secretsmanager") yield sm_client -def create_secret(sm_client, secret_name, cohort_id, user, password, host, database, port): + +def create_secret( + sm_client, secret_name, cohort_id, user, password, host, database, port +): secret = { "cohort_id": cohort_id, "user": user, "password": password, "host": host, "database": database, - "port": port + "port": port, } response = sm_client.create_secret( - Name = secret_name, - SecretString = json.dumps(secret) + Name=secret_name, SecretString=json.dumps(secret) ) print(response) return response + def list_secret(sm_client): response = sm_client.list_secrets() - secret_dict = response['SecretList'] + secret_dict = response["SecretList"] secret_names = [] for items in secret_dict: - secret_names.append(items['Name']) - print(f'{len(secret_names)} secret(s) available') + secret_names.append(items["Name"]) + print(f"{len(secret_names)} secret(s) available") for name in secret_names: print(name) return secret_names -def retrieve_secrets(sm_client): - response = sm_client.get_secrets( - - ) +def retrieve_secrets(sm_client): + response = sm_client.get_secrets() -#retrieve secret -#so lambda can access totesy db -#so lambda connect to the db and then retrieve the data \ No newline at end of file +# retrieve secret +# so lambda can access totesy db +# so lambda connect to the db and then retrieve the data diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 6ee681f..c6a8e60 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -1,2 +1,2 @@ def lambda_handler(): - pass \ No newline at end of file + pass diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py index 86533bc..cb4ec15 100644 --- a/test/test_secrets_manager.py +++ b/test/test_secrets_manager.py @@ -2,10 +2,12 @@ from src.secrets_manager import sm_client, create_secret, list_secret import boto3 from moto import mock_aws import json -import pytest +import pytest import os -pytest.fixture(scope='class') +pytest.fixture(scope="class") + + def mock_aws_credentials(): """Mocked AWS Credentials for moto.""" os.environ["AWS_ACCESS_KEY_ID"] = "testing" @@ -14,10 +16,11 @@ def mock_aws_credentials(): os.environ["AWS_SESSION_TOKEN"] = "testing" os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_sm_client(mock_aws_credentials): with mock_aws(): - yield boto3.client('secretsmanager') + yield boto3.client("secretsmanager") def test_create_secret_stores_secrets(mock_sm_client): @@ -29,6 +32,8 @@ def test_create_secret_stores_secrets(mock_sm_client): port = "test_port" secret_name = "test_secret" - response = create_secret(mock_sm_client, secret_name, cohort_id, user, password, host, database, port) - - assert response['Name'] == secret_name \ No newline at end of file + response = create_secret( + mock_sm_client, secret_name, cohort_id, user, password, host, database, port + ) + + assert response["Name"] == secret_name diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index e94a8a4..877e36a 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -3,11 +3,17 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables -import os +from src.extract_lambda import ( + list_existing_s3_files, + connect_to_database, + DBConnectionException, + process_and_upload_tables, +) +import os import logging -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_config(): env_vars = { "host": "abc", @@ -20,54 +26,55 @@ def mock_config(): yield mock_config -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def aws_credentials(): - os.environ["AWS_ACCESS_KEY_ID"] = 'testing' - os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURIT_TOKEN"] = 'testing' - os.environ["AWS_SESSION_TOKEN"] = 'testing' - os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURIT_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_client(aws_credentials): with mock_aws(): - yield boto3.client('s3') + yield boto3.client("s3") + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) list_existing_s3_files(client=s3_client) - assert 'Error listing S3 objects' in caplog.text + assert "Error listing S3 objects" in caplog.text def test_error_if_bucket_is_empty(self, s3_client, caplog): - - s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) + s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) list_existing_s3_files(client=s3_client) - assert 'The bucket is empty' in caplog.text + assert "The bucket is empty" in caplog.text def test_error_retrieving_object(self, s3_client, caplog): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - list_existing_s3_files(bucket_name='test_bucket', client=s3_client) + s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") + list_existing_s3_files(bucket_name="test_bucket", client=s3_client) - assert 'Error retrieving S3 object ' in caplog.text + assert "Error retrieving S3 object " in caplog.text def test_retrieves_file_content(self, s3_client, caplog): result = list_existing_s3_files(client=s3_client) - assert list(result.values()) == ['This is a test file.'] + assert list(result.values()) == ["This is a test file."] + class TestConnectToDatabase: def test_connect_to_database(mock_conn, mock_config): - with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: + with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( - host="abc", user="def", port="5432", password="password", database="db" + host="abc", user="def", port="5432", password="password", database="db" ) def test_database_error(self, mock_config): @@ -76,12 +83,14 @@ class TestConnectToDatabase: def test_logs_interface_error(self, caplog): logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert 'Interface error' in caplog.text -''' + assert "Interface error" in caplog.text + + +""" class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): logger = logging.getLogger() @@ -106,4 +115,4 @@ class TestProcessAndUploadTables: s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) assert 'No new data.' in caplog.text -''' \ No newline at end of file +""" diff --git a/tests/test_secrets_manager.py b/tests/test_secrets_manager.py index a30be86..609c572 100644 --- a/tests/test_secrets_manager.py +++ b/tests/test_secrets_manager.py @@ -3,10 +3,11 @@ import boto3 import botocore.exceptions from moto import mock_aws import json -import pytest +import pytest import os -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def aws_credentials(): """Mocked AWS Credentials for moto.""" os.environ["AWS_ACCESS_KEY_ID"] = "testing" @@ -15,12 +16,14 @@ def aws_credentials(): os.environ["AWS_SESSION_TOKEN"] = "testing" os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def mock_sm_client(aws_credentials): with mock_aws(): yield boto3.client("secretsmanager") -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def mock_store_secret(mock_sm_client): secret = { "cohort_id": "test_cohort_id", @@ -28,15 +31,18 @@ def mock_store_secret(mock_sm_client): "password": "test_password", "host": "test_host", "database": "test_database", - "port": "test_port" + "port": "test_port", } secret_name = "test_secret" - response = mock_sm_client.create_secret(Name=secret_name, SecretString=json.dumps(secret)) + response = mock_sm_client.create_secret( + Name=secret_name, SecretString=json.dumps(secret) + ) return response + def test_retrieves_secrets_returns_dictionary(mock_sm_client, mock_store_secret): secret_name = "test_secret" @@ -44,8 +50,10 @@ def test_retrieves_secrets_returns_dictionary(mock_sm_client, mock_store_secret) assert isinstance(result, dict) -def test_retrieves_secrets_returns_correct_keys_and_values(mock_sm_client, mock_store_secret): +def test_retrieves_secrets_returns_correct_keys_and_values( + mock_sm_client, mock_store_secret +): secret_name = "test_secret" result = retrieve_secrets(mock_sm_client, secret_name) @@ -57,17 +65,20 @@ def test_retrieves_secrets_returns_correct_keys_and_values(mock_sm_client, mock_ assert result["database"] == "test_database" assert result["port"] == "test_port" -def test_retrieves_secrets_raises_error_if_secret_name_incorrect_data_type(mock_sm_client): - secret_name = [1, 2, 3] +def test_retrieves_secrets_raises_error_if_secret_name_incorrect_data_type( + mock_sm_client, +): + secret_name = [1, 2, 3] with pytest.raises(botocore.exceptions.ParamValidationError) as error: retrieve_secrets(mock_sm_client, secret_name) -def test_retrieves_secrets_raises_error_if_secret_name_does_not_exist(mock_sm_client, mock_store_secret): - secret_name = 'test_secret_2' - +def test_retrieves_secrets_raises_error_if_secret_name_does_not_exist( + mock_sm_client, mock_store_secret +): + secret_name = "test_secret_2" with pytest.raises(botocore.exceptions.ClientError) as error: - retrieve_secrets(mock_sm_client, secret_name) \ No newline at end of file + retrieve_secrets(mock_sm_client, secret_name) -- cgit v1.2.3 From afc889de865e6ce42b19ce89c57e9bfed98d6757 Mon Sep 17 00:00:00 2001 From: T-Aji Date: Mon, 19 Aug 2024 09:27:20 +0100 Subject: test: handler exception test failing --- tests/test_extract_lambda.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 4b61b83..bc40df1 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -76,6 +76,17 @@ class TestLambdaHandler: mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) mock_db.close.assert_called_once() + def test_lambda_handler_exception_error(self, mocker): + with patch("src.extract_lambda.connect_to_database", side_effect=Exception("Database connection error")): + mock_process_and_upload_tables = mocker.patch("src.extract_lambda.process_and_upload_tables") + mock_list_existing_s3_files = mocker.patch("src.extract_lambda.list_existing_s3_files") + event = {} + context = {} + response = lambda_handler(event, context) + assert response['statusCode'] == 500 + assert json.loads(response['body']) == 'Internal server error.' + mock_list_existing_s3_files.assert_not_called() + mock_process_and_upload_tables.assert_not_called() class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): -- cgit v1.2.3 From dd536c3209fc37423af4219a941c006bdb6b3c4f Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Mon, 19 Aug 2024 10:32:57 +0100 Subject: deleted the test folder --- test/test_secrets_manager.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 test/test_secrets_manager.py diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py deleted file mode 100644 index 86533bc..0000000 --- a/test/test_secrets_manager.py +++ /dev/null @@ -1,34 +0,0 @@ -from src.secrets_manager import sm_client, create_secret, list_secret -import boto3 -from moto import mock_aws -import json -import pytest -import os - -pytest.fixture(scope='class') -def mock_aws_credentials(): - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - os.environ["AWS_SECURITY_TOKEN"] = "testing" - os.environ["AWS_SESSION_TOKEN"] = "testing" - os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" - -@pytest.fixture(scope='class') -def mock_sm_client(mock_aws_credentials): - with mock_aws(): - yield boto3.client('secretsmanager') - - -def test_create_secret_stores_secrets(mock_sm_client): - cohort_id = "test_cohort_id" - user = "test_user_id" - password = "test_password" - host = "test_host" - database = "test_database" - port = "test_port" - - secret_name = "test_secret" - response = create_secret(mock_sm_client, secret_name, cohort_id, user, password, host, database, port) - - assert response['Name'] == secret_name \ No newline at end of file -- cgit v1.2.3 From 3ab3164c2e6f0e7a7ae6755a58914522bf3390a6 Mon Sep 17 00:00:00 2001 From: "deepsource-io[bot]" <42547082+deepsource-io[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:34:32 +0000 Subject: ci: update .deepsource.toml --- .deepsource.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.deepsource.toml b/.deepsource.toml index a840b78..42d0973 100644 --- a/.deepsource.toml +++ b/.deepsource.toml @@ -1,5 +1,7 @@ version = 1 +test_patterns = ["tests/**"] + [[analyzers]] name = "sql" @@ -22,6 +24,4 @@ name = "black" name = "autopep8" [[transformers]] -name = "ruff" - - +name = "ruff" \ No newline at end of file -- cgit v1.2.3 From 5cc511d2afeea262db0db7039c8f83c123da77ea Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 09:55:43 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in afc889d according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/54 --- tests/test_extract_lambda.py | 144 +++++++++++++++++++++++++++---------------- 1 file changed, 92 insertions(+), 52 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index bc40df1..67cb6d3 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -3,12 +3,19 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, lambda_handler, process_and_upload_tables -import os +from src.extract_lambda import ( + list_existing_s3_files, + connect_to_database, + DBConnectionException, + lambda_handler, + process_and_upload_tables, +) +import os import logging import json -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_config(): env_vars = { "host": "abc", @@ -21,36 +28,49 @@ def mock_config(): yield mock_config -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def aws_credentials(): - os.environ["AWS_ACCESS_KEY_ID"] = 'testing' - os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURIT_TOKEN"] = 'testing' - os.environ["AWS_SESSION_TOKEN"] = 'testing' - os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURIT_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_client(aws_credentials): with mock_aws(): - yield boto3.client('s3') + yield boto3.client("s3") + class TestLambdaHandler: def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): mock_db = MagicMock() mock_db.run.side_effect = [ - [['Fruits']], - [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], - [['Food_type'], ['Flavour'], ['Colour']] + [["Fruits"]], + [["Vegetable", "Sour", "Green"], ["Berry", "Sweet", "Red"]], + [["Food_type"], ["Flavour"], ["Colour"]], + ] + mock_db.columns.return_value = [ + {"name": "Food_type"}, + {"name": "Flavour"}, + {"name": "Colour"}, ] - mock_db.columns.return_value = [{'name': 'Food_type'}, {'name': 'Flavour'}, {'name': 'Colour'}] with patch("src.extract_lambda.connect_to_database", return_value=mock_db): - mock_process_and_upload_tables = mocker.patch("src.extract_lambda.process_and_upload_tables", return_value=mock_db) - mock_list_existing_s3_files = mocker.patch("src.extract_lambda.list_existing_s3_files", return_value={}) + mock_process_and_upload_tables = mocker.patch( + "src.extract_lambda.process_and_upload_tables", return_value=mock_db + ) + mock_list_existing_s3_files = mocker.patch( + "src.extract_lambda.list_existing_s3_files", return_value={} + ) event = {} context = {} response = lambda_handler(event, context) - assert response['statusCode'] == 200 - assert json.loads(response['body']) == 'CSV files processed and uploaded successfully.' + assert response["statusCode"] == 200 + assert ( + json.loads(response["body"]) + == "CSV files processed and uploaded successfully." + ) mock_list_existing_s3_files.assert_called_once() mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) mock_db.close.assert_called_once() @@ -58,71 +78,89 @@ class TestLambdaHandler: def test_lambda_handler_no_changes_detected_no_files_uploaded(self, mocker): mock_db = MagicMock() mock_db.run.side_effect = [ - [['Fruits']], - [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], - [['Food_type'], ['Flavour'], ['Colour']] + [["Fruits"]], + [["Vegetable", "Sour", "Green"], ["Berry", "Sweet", "Red"]], + [["Food_type"], ["Flavour"], ["Colour"]], + ] + mock_db.columns.return_value = [ + {"name": "Food_type"}, + {"name": "Flavour"}, + {"name": "Colour"}, ] - mock_db.columns.return_value = [{'name': 'Food_type'}, {'name': 'Flavour'}, {'name': 'Colour'}] with patch("src.extract_lambda.connect_to_database", return_value=mock_db): - mock_process_and_upload_tables = mocker.patch("src.extract_lambda.process_and_upload_tables", return_value=False) - mock_list_existing_s3_files = mocker.patch("src.extract_lambda.list_existing_s3_files", return_value={}) + mock_process_and_upload_tables = mocker.patch( + "src.extract_lambda.process_and_upload_tables", return_value=False + ) + mock_list_existing_s3_files = mocker.patch( + "src.extract_lambda.list_existing_s3_files", return_value={} + ) event = {} context = {} response = lambda_handler(event, context) - assert response['statusCode'] == 200 - assert json.loads(response['body']) == 'No changes detected, no CSV files were uploaded.' + assert response["statusCode"] == 200 + assert ( + json.loads(response["body"]) + == "No changes detected, no CSV files were uploaded." + ) mock_list_existing_s3_files.assert_called_once() mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) mock_db.close.assert_called_once() def test_lambda_handler_exception_error(self, mocker): - with patch("src.extract_lambda.connect_to_database", side_effect=Exception("Database connection error")): - mock_process_and_upload_tables = mocker.patch("src.extract_lambda.process_and_upload_tables") - mock_list_existing_s3_files = mocker.patch("src.extract_lambda.list_existing_s3_files") + with patch( + "src.extract_lambda.connect_to_database", + side_effect=Exception("Database connection error"), + ): + mock_process_and_upload_tables = mocker.patch( + "src.extract_lambda.process_and_upload_tables" + ) + mock_list_existing_s3_files = mocker.patch( + "src.extract_lambda.list_existing_s3_files" + ) event = {} context = {} response = lambda_handler(event, context) - assert response['statusCode'] == 500 - assert json.loads(response['body']) == 'Internal server error.' + assert response["statusCode"] == 500 + assert json.loads(response["body"]) == "Internal server error." mock_list_existing_s3_files.assert_not_called() - mock_process_and_upload_tables.assert_not_called() + mock_process_and_upload_tables.assert_not_called() + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) list_existing_s3_files(client=s3_client) - assert 'Error listing S3 objects' in caplog.text + assert "Error listing S3 objects" in caplog.text def test_error_if_bucket_is_empty(self, s3_client, caplog): - - s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) + s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) list_existing_s3_files(client=s3_client) - assert 'The bucket is empty' in caplog.text + assert "The bucket is empty" in caplog.text def test_error_retrieving_object(self, s3_client, caplog): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - list_existing_s3_files(bucket_name='test_bucket', client=s3_client) + s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") + list_existing_s3_files(bucket_name="test_bucket", client=s3_client) - assert 'Error retrieving S3 object ' in caplog.text + assert "Error retrieving S3 object " in caplog.text def test_retrieves_file_content(self, s3_client, caplog): result = list_existing_s3_files(client=s3_client) - assert list(result.values()) == ['This is a test file.'] + assert list(result.values()) == ["This is a test file."] + class TestConnectToDatabase: def test_connect_to_database(mock_conn, mock_config): - with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: + with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( - host="abc", user="def", port="5432", password="password", database="db" + host="abc", user="def", port="5432", password="password", database="db" ) def test_database_error(self, mock_config): @@ -131,12 +169,14 @@ class TestConnectToDatabase: def test_logs_interface_error(self, caplog): logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert 'Interface error' in caplog.text -''' + assert "Interface error" in caplog.text + + +""" class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): logger = logging.getLogger() @@ -161,4 +201,4 @@ class TestProcessAndUploadTables: s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) assert 'No new data.' in caplog.text -''' \ No newline at end of file +""" -- cgit v1.2.3 From e27c6b48897a48f8462b8a0f40deb0ddaf301b63 Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Mon, 19 Aug 2024 11:21:58 +0100 Subject: layers block update, function resources to inlcude attributes: layers, correct handler and source_code_hash --- terraform/lambda.tf | 70 +++++++++++++++++++++++++++++------------------------ terraform/s3.tf | 5 ++++ 2 files changed, 44 insertions(+), 31 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 27e6266..e33bc79 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -12,12 +12,14 @@ resource "aws_s3_object" "extract_lambda_code" { } resource "aws_lambda_function" "extract_lambda" { - function_name = var.extract_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.extract_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "extract_lambda.extract" - runtime = "python3.11" + function_name = var.extract_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.extract_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "extract_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -40,12 +42,14 @@ resource "aws_s3_object" "transform_lambda_code" { } resource "aws_lambda_function" "transform_lambda" { - function_name = var.transform_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.transform_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "transform_lambda.transform" - runtime = "python3.11" + function_name = var.transform_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.transform_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "transform_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -68,12 +72,14 @@ resource "aws_s3_object" "load_lambda_code" { } resource "aws_lambda_function" "load_lambda" { - function_name = var.load_lambda_name - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.load_lambda_code.key - role = aws_iam_role.multi_service_role.arn - handler = "load_lambda.load" - runtime = "python3.11" + function_name = var.load_lambda_name + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.load_lambda_code.key + layers = [aws_lambda_layer_version.lambda_layer.arn] + role = aws_iam_role.multi_service_role.arn + handler = "load_lambda.lambda_handler" + runtime = "python3.11" + source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 lifecycle { create_before_destroy = true @@ -82,10 +88,12 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } +# Lambda Layer Specification locals { - layer_dir = "${path.module}/.." - requirements = "${path.module}/../requirements.txt" - layer_zip = "${path.module}/../layer.zip" + layer_dir = "lambda_layer" + requirements = "requirements.txt" + layer_zip = "layer.zip" + layer_name = "lambda_layer_dev" } resource "null_resource" "prepare_layer" { @@ -95,23 +103,23 @@ resource "null_resource" "prepare_layer" { rm -rf python mkdir python pip3 install -r ${local.requirements} -t python/ - zip -r ${local.layer_zip} python/ - EOT - } + zip -r ${local.layer_zip} python + EOT + } #removed / at the end of python in line 99 } -resource "aws_s3_object" "layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.bucket - key = "layer.zip" +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "lambda_layer/${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = "lambda_layer" + layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.layer_zip.key + s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true - depends_on = [aws_s3_object.layer_zip] + depends_on = [aws_s3_object.lambda_layer_zip] } diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..b3a863c 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,3 +12,8 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } + +### LAMBDA LAYER BUCKET +resource "aws_s3_bucket" "lambda_layer_bucket" { + bucket_prefix = "lambda-layer-dev-" +} \ No newline at end of file -- cgit v1.2.3 From 43df5dd9c6bd21f33a7fccbc9b81ad3677637da5 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 10:23:19 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in e27c6b4 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/55 --- src/load_lambda.py | 2 +- src/secrets_manager.py | 31 +++++++++---------- src/transform_lambda.py | 2 +- test/test_secrets_manager.py | 19 +++++++----- tests/test_extract_lambda.py | 69 ++++++++++++++++++++++++------------------- tests/test_secrets_manager.py | 37 +++++++++++++++-------- 6 files changed, 93 insertions(+), 67 deletions(-) diff --git a/src/load_lambda.py b/src/load_lambda.py index 6ee681f..c6a8e60 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -1,2 +1,2 @@ def lambda_handler(): - pass \ No newline at end of file + pass diff --git a/src/secrets_manager.py b/src/secrets_manager.py index c0fb61e..3484688 100644 --- a/src/secrets_manager.py +++ b/src/secrets_manager.py @@ -4,45 +4,46 @@ import json def sm_client(): - sm_client = boto3.client('secretsmanager') + sm_client = boto3.client("secretsmanager") yield sm_client -def create_secret(sm_client, secret_name, cohort_id, user, password, host, database, port): + +def create_secret( + sm_client, secret_name, cohort_id, user, password, host, database, port +): secret = { "cohort_id": cohort_id, "user": user, "password": password, "host": host, "database": database, - "port": port + "port": port, } response = sm_client.create_secret( - Name = secret_name, - SecretString = json.dumps(secret) + Name=secret_name, SecretString=json.dumps(secret) ) print(response) return response + def list_secret(sm_client): response = sm_client.list_secrets() - secret_dict = response['SecretList'] + secret_dict = response["SecretList"] secret_names = [] for items in secret_dict: - secret_names.append(items['Name']) - print(f'{len(secret_names)} secret(s) available') + secret_names.append(items["Name"]) + print(f"{len(secret_names)} secret(s) available") for name in secret_names: print(name) return secret_names -def retrieve_secrets(sm_client): - response = sm_client.get_secrets( - - ) +def retrieve_secrets(sm_client): + response = sm_client.get_secrets() -#retrieve secret -#so lambda can access totesy db -#so lambda connect to the db and then retrieve the data \ No newline at end of file +# retrieve secret +# so lambda can access totesy db +# so lambda connect to the db and then retrieve the data diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 6ee681f..c6a8e60 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -1,2 +1,2 @@ def lambda_handler(): - pass \ No newline at end of file + pass diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py index 86533bc..cb4ec15 100644 --- a/test/test_secrets_manager.py +++ b/test/test_secrets_manager.py @@ -2,10 +2,12 @@ from src.secrets_manager import sm_client, create_secret, list_secret import boto3 from moto import mock_aws import json -import pytest +import pytest import os -pytest.fixture(scope='class') +pytest.fixture(scope="class") + + def mock_aws_credentials(): """Mocked AWS Credentials for moto.""" os.environ["AWS_ACCESS_KEY_ID"] = "testing" @@ -14,10 +16,11 @@ def mock_aws_credentials(): os.environ["AWS_SESSION_TOKEN"] = "testing" os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_sm_client(mock_aws_credentials): with mock_aws(): - yield boto3.client('secretsmanager') + yield boto3.client("secretsmanager") def test_create_secret_stores_secrets(mock_sm_client): @@ -29,6 +32,8 @@ def test_create_secret_stores_secrets(mock_sm_client): port = "test_port" secret_name = "test_secret" - response = create_secret(mock_sm_client, secret_name, cohort_id, user, password, host, database, port) - - assert response['Name'] == secret_name \ No newline at end of file + response = create_secret( + mock_sm_client, secret_name, cohort_id, user, password, host, database, port + ) + + assert response["Name"] == secret_name diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index e94a8a4..877e36a 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -3,11 +3,17 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables -import os +from src.extract_lambda import ( + list_existing_s3_files, + connect_to_database, + DBConnectionException, + process_and_upload_tables, +) +import os import logging -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_config(): env_vars = { "host": "abc", @@ -20,54 +26,55 @@ def mock_config(): yield mock_config -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def aws_credentials(): - os.environ["AWS_ACCESS_KEY_ID"] = 'testing' - os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURIT_TOKEN"] = 'testing' - os.environ["AWS_SESSION_TOKEN"] = 'testing' - os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURIT_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_client(aws_credentials): with mock_aws(): - yield boto3.client('s3') + yield boto3.client("s3") + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) list_existing_s3_files(client=s3_client) - assert 'Error listing S3 objects' in caplog.text + assert "Error listing S3 objects" in caplog.text def test_error_if_bucket_is_empty(self, s3_client, caplog): - - s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) + s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) list_existing_s3_files(client=s3_client) - assert 'The bucket is empty' in caplog.text + assert "The bucket is empty" in caplog.text def test_error_retrieving_object(self, s3_client, caplog): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - list_existing_s3_files(bucket_name='test_bucket', client=s3_client) + s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") + list_existing_s3_files(bucket_name="test_bucket", client=s3_client) - assert 'Error retrieving S3 object ' in caplog.text + assert "Error retrieving S3 object " in caplog.text def test_retrieves_file_content(self, s3_client, caplog): result = list_existing_s3_files(client=s3_client) - assert list(result.values()) == ['This is a test file.'] + assert list(result.values()) == ["This is a test file."] + class TestConnectToDatabase: def test_connect_to_database(mock_conn, mock_config): - with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: + with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( - host="abc", user="def", port="5432", password="password", database="db" + host="abc", user="def", port="5432", password="password", database="db" ) def test_database_error(self, mock_config): @@ -76,12 +83,14 @@ class TestConnectToDatabase: def test_logs_interface_error(self, caplog): logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert 'Interface error' in caplog.text -''' + assert "Interface error" in caplog.text + + +""" class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): logger = logging.getLogger() @@ -106,4 +115,4 @@ class TestProcessAndUploadTables: s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) assert 'No new data.' in caplog.text -''' \ No newline at end of file +""" diff --git a/tests/test_secrets_manager.py b/tests/test_secrets_manager.py index a30be86..609c572 100644 --- a/tests/test_secrets_manager.py +++ b/tests/test_secrets_manager.py @@ -3,10 +3,11 @@ import boto3 import botocore.exceptions from moto import mock_aws import json -import pytest +import pytest import os -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def aws_credentials(): """Mocked AWS Credentials for moto.""" os.environ["AWS_ACCESS_KEY_ID"] = "testing" @@ -15,12 +16,14 @@ def aws_credentials(): os.environ["AWS_SESSION_TOKEN"] = "testing" os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def mock_sm_client(aws_credentials): with mock_aws(): yield boto3.client("secretsmanager") -@pytest.fixture(scope='function') + +@pytest.fixture(scope="function") def mock_store_secret(mock_sm_client): secret = { "cohort_id": "test_cohort_id", @@ -28,15 +31,18 @@ def mock_store_secret(mock_sm_client): "password": "test_password", "host": "test_host", "database": "test_database", - "port": "test_port" + "port": "test_port", } secret_name = "test_secret" - response = mock_sm_client.create_secret(Name=secret_name, SecretString=json.dumps(secret)) + response = mock_sm_client.create_secret( + Name=secret_name, SecretString=json.dumps(secret) + ) return response + def test_retrieves_secrets_returns_dictionary(mock_sm_client, mock_store_secret): secret_name = "test_secret" @@ -44,8 +50,10 @@ def test_retrieves_secrets_returns_dictionary(mock_sm_client, mock_store_secret) assert isinstance(result, dict) -def test_retrieves_secrets_returns_correct_keys_and_values(mock_sm_client, mock_store_secret): +def test_retrieves_secrets_returns_correct_keys_and_values( + mock_sm_client, mock_store_secret +): secret_name = "test_secret" result = retrieve_secrets(mock_sm_client, secret_name) @@ -57,17 +65,20 @@ def test_retrieves_secrets_returns_correct_keys_and_values(mock_sm_client, mock_ assert result["database"] == "test_database" assert result["port"] == "test_port" -def test_retrieves_secrets_raises_error_if_secret_name_incorrect_data_type(mock_sm_client): - secret_name = [1, 2, 3] +def test_retrieves_secrets_raises_error_if_secret_name_incorrect_data_type( + mock_sm_client, +): + secret_name = [1, 2, 3] with pytest.raises(botocore.exceptions.ParamValidationError) as error: retrieve_secrets(mock_sm_client, secret_name) -def test_retrieves_secrets_raises_error_if_secret_name_does_not_exist(mock_sm_client, mock_store_secret): - secret_name = 'test_secret_2' - +def test_retrieves_secrets_raises_error_if_secret_name_does_not_exist( + mock_sm_client, mock_store_secret +): + secret_name = "test_secret_2" with pytest.raises(botocore.exceptions.ClientError) as error: - retrieve_secrets(mock_sm_client, secret_name) \ No newline at end of file + retrieve_secrets(mock_sm_client, secret_name) -- cgit v1.2.3 From 1ea59ed0d92d5bbbd1ffe46ca7a1e296aa55fb1f Mon Sep 17 00:00:00 2001 From: T-Aji Date: Mon, 19 Aug 2024 11:29:45 +0100 Subject: all tests added --- tests/test_extract_lambda.py | 155 ++++++++++++++++++++++++++++++++++--------- 1 file changed, 125 insertions(+), 30 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index e94a8a4..67cb6d3 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -3,11 +3,19 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables -import os +from src.extract_lambda import ( + list_existing_s3_files, + connect_to_database, + DBConnectionException, + lambda_handler, + process_and_upload_tables, +) +import os import logging +import json -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_config(): env_vars = { "host": "abc", @@ -20,54 +28,139 @@ def mock_config(): yield mock_config -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def aws_credentials(): - os.environ["AWS_ACCESS_KEY_ID"] = 'testing' - os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURIT_TOKEN"] = 'testing' - os.environ["AWS_SESSION_TOKEN"] = 'testing' - os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURIT_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_client(aws_credentials): with mock_aws(): - yield boto3.client('s3') + yield boto3.client("s3") + + +class TestLambdaHandler: + def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): + mock_db = MagicMock() + mock_db.run.side_effect = [ + [["Fruits"]], + [["Vegetable", "Sour", "Green"], ["Berry", "Sweet", "Red"]], + [["Food_type"], ["Flavour"], ["Colour"]], + ] + mock_db.columns.return_value = [ + {"name": "Food_type"}, + {"name": "Flavour"}, + {"name": "Colour"}, + ] + with patch("src.extract_lambda.connect_to_database", return_value=mock_db): + mock_process_and_upload_tables = mocker.patch( + "src.extract_lambda.process_and_upload_tables", return_value=mock_db + ) + mock_list_existing_s3_files = mocker.patch( + "src.extract_lambda.list_existing_s3_files", return_value={} + ) + event = {} + context = {} + response = lambda_handler(event, context) + assert response["statusCode"] == 200 + assert ( + json.loads(response["body"]) + == "CSV files processed and uploaded successfully." + ) + mock_list_existing_s3_files.assert_called_once() + mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) + mock_db.close.assert_called_once() + + def test_lambda_handler_no_changes_detected_no_files_uploaded(self, mocker): + mock_db = MagicMock() + mock_db.run.side_effect = [ + [["Fruits"]], + [["Vegetable", "Sour", "Green"], ["Berry", "Sweet", "Red"]], + [["Food_type"], ["Flavour"], ["Colour"]], + ] + mock_db.columns.return_value = [ + {"name": "Food_type"}, + {"name": "Flavour"}, + {"name": "Colour"}, + ] + + with patch("src.extract_lambda.connect_to_database", return_value=mock_db): + mock_process_and_upload_tables = mocker.patch( + "src.extract_lambda.process_and_upload_tables", return_value=False + ) + mock_list_existing_s3_files = mocker.patch( + "src.extract_lambda.list_existing_s3_files", return_value={} + ) + event = {} + context = {} + response = lambda_handler(event, context) + assert response["statusCode"] == 200 + assert ( + json.loads(response["body"]) + == "No changes detected, no CSV files were uploaded." + ) + mock_list_existing_s3_files.assert_called_once() + mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) + mock_db.close.assert_called_once() + + def test_lambda_handler_exception_error(self, mocker): + with patch( + "src.extract_lambda.connect_to_database", + side_effect=Exception("Database connection error"), + ): + mock_process_and_upload_tables = mocker.patch( + "src.extract_lambda.process_and_upload_tables" + ) + mock_list_existing_s3_files = mocker.patch( + "src.extract_lambda.list_existing_s3_files" + ) + event = {} + context = {} + response = lambda_handler(event, context) + assert response["statusCode"] == 500 + assert json.loads(response["body"]) == "Internal server error." + mock_list_existing_s3_files.assert_not_called() + mock_process_and_upload_tables.assert_not_called() + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) list_existing_s3_files(client=s3_client) - assert 'Error listing S3 objects' in caplog.text + assert "Error listing S3 objects" in caplog.text def test_error_if_bucket_is_empty(self, s3_client, caplog): - - s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) + s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) list_existing_s3_files(client=s3_client) - assert 'The bucket is empty' in caplog.text + assert "The bucket is empty" in caplog.text def test_error_retrieving_object(self, s3_client, caplog): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - list_existing_s3_files(bucket_name='test_bucket', client=s3_client) + s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") + list_existing_s3_files(bucket_name="test_bucket", client=s3_client) - assert 'Error retrieving S3 object ' in caplog.text + assert "Error retrieving S3 object " in caplog.text def test_retrieves_file_content(self, s3_client, caplog): result = list_existing_s3_files(client=s3_client) - assert list(result.values()) == ['This is a test file.'] + assert list(result.values()) == ["This is a test file."] + class TestConnectToDatabase: def test_connect_to_database(mock_conn, mock_config): - with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: + with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( - host="abc", user="def", port="5432", password="password", database="db" + host="abc", user="def", port="5432", password="password", database="db" ) def test_database_error(self, mock_config): @@ -76,12 +169,14 @@ class TestConnectToDatabase: def test_logs_interface_error(self, caplog): logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert 'Interface error' in caplog.text -''' + assert "Interface error" in caplog.text + + +""" class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): logger = logging.getLogger() @@ -106,4 +201,4 @@ class TestProcessAndUploadTables: s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) assert 'No new data.' in caplog.text -''' \ No newline at end of file +""" -- cgit v1.2.3 From 24a4573d6cf64ec0383ae16bfba09a0ffdb8c129 Mon Sep 17 00:00:00 2001 From: T-Aji Date: Mon, 19 Aug 2024 11:49:08 +0100 Subject: update .gitignore --- .gitignore | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index bceab93..6aa03fc 100644 --- a/.gitignore +++ b/.gitignore @@ -14,5 +14,4 @@ __pycache__/ # OS-Related Files .DS_Store - -*venv* +venv \ No newline at end of file -- cgit v1.2.3 From 444bb270fc8f758f33b0477c992b6a8e873bcd89 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 11:06:02 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 0eff70f according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/59 --- tests/test_extract_lambda.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index fc68a4a..7707cbf 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -42,6 +42,7 @@ def s3_client(aws_credentials): with mock_aws(): yield boto3.client("s3") + class TestLambdaHandler: def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): mock_db = MagicMock() @@ -125,6 +126,7 @@ class TestLambdaHandler: mock_list_existing_s3_files.assert_not_called() mock_process_and_upload_tables.assert_not_called() + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): logger = logging.getLogger() -- cgit v1.2.3 From 81cba7c5bc4bed060901d6e19c84d5acee054b3e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 14:53:43 +0100 Subject: feat: create shell script for creating lambda layer zip --- scripts/make_layer_zip.sh | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100755 scripts/make_layer_zip.sh diff --git a/scripts/make_layer_zip.sh b/scripts/make_layer_zip.sh new file mode 100755 index 0000000..0e7560f --- /dev/null +++ b/scripts/make_layer_zip.sh @@ -0,0 +1,7 @@ +# Description: Make the zip file for the layer + +cd "$(dirname "$0")/.." +mkdir tmp_python +pip3 install --upgrade -r requirements.txt -t tmp_python/ +zip -r layer.zip tmp_python +rm -r tmp_python/ -- cgit v1.2.3 From 57d1e1ee5a13269f1bef6c3b754cb8374a657202 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 14:55:39 +0100 Subject: style: remove redundant comment --- terraform/lambda.tf | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index e33bc79..714ffa5 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -99,13 +99,9 @@ locals { resource "null_resource" "prepare_layer" { provisioner "local-exec" { command = < Date: Mon, 19 Aug 2024 15:02:39 +0100 Subject: infra(tf): modify variables & remove past zip creation --- terraform/lambda.tf | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 714ffa5..986170f 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,17 +90,16 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "lambda_layer" + layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" layer_name = "lambda_layer_dev" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { provisioner "local-exec" { - command = < Date: Mon, 19 Aug 2024 15:03:50 +0100 Subject: wip: amend extract_lambda test --- tests/test_extract_lambda.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 7707cbf..4a5157b 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -12,6 +12,7 @@ from src.extract_lambda import ( DBConnectionException, lambda_handler, process_and_upload_tables, + retrieve_secrets ) @@ -24,7 +25,7 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.get_config", return_value=env_vars) as mock_config: + with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: yield mock_config @@ -140,7 +141,7 @@ class TestListExistingS3Files: Bucket="extract_bucket", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, ) - list_existing_s3_files(client=s3_client) + list_existing_s3_files("extract_bucket", client=s3_client) assert "The bucket is empty" in caplog.text def test_error_retrieving_object(self, s3_client, caplog): @@ -176,9 +177,8 @@ class TestConnectToDatabase: assert "Interface error" in caplog.text -""" class TestProcessAndUploadTables: - def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): + def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): logger = logging.getLogger() logger.info('Testing now.') caplog.set_level(logging.ERROR) @@ -188,17 +188,17 @@ class TestProcessAndUploadTables: "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'"] return_values = [[['Fruits']], [['Vegetable','Sour','Green'],['Berry','Sweet','Red']], - [['Food_type'],['Flavour'],['Colour']]] + [['Food_type'],['Flavour'],['Colour']]] # why are individual column names in lists vals = dict(zip(queries,return_values)) + # {"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';": [['Fruits']], 'SELECT * FROM Fruits;': [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'": [['Food_type'], ['Flavour'], ['Colour']]} - #### - with patch('src.extract_lambda.connect_to_database') as mock_db: - mock_db().run.side_effects = return_values + with patch('src.extract_lambda.Connection') as mock_db: + mock_db().run.side_effect = return_values s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' existing_files = {s3_key: 'Food_type,Flavour,Colour\nFruit,Sour,Green\nBerry,Sweet,Red'} - s3_client.create_bucket(Bucket='extract_bucket', + s3_client.create_bucket(Bucket='test_extract_bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) + print(s3_client.list_buckets) s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) - assert 'No new data.' in caplog.text -""" + assert 'No new data.' in caplog.text \ No newline at end of file -- cgit v1.2.3 From 7b46fec037830648f6f356219f9df7fdbbbd181c Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:06:35 +0100 Subject: infra(tf): remove lambda layer dev reference --- terraform/s3.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/terraform/s3.tf b/terraform/s3.tf index b3a863c..d5cdee3 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -12,8 +12,3 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" } - -### LAMBDA LAYER BUCKET -resource "aws_s3_bucket" "lambda_layer_bucket" { - bucket_prefix = "lambda-layer-dev-" -} \ No newline at end of file -- cgit v1.2.3 From 284a52df866c34d925b85ccd4f06d6141e67ce70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:12:56 +0100 Subject: fix(tf): correct layer.zip output path --- terraform/lambda.tf | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 986170f..8a4207d 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -93,7 +93,7 @@ locals { layer_dir = "../" requirements = "requirements.txt" layer_zip = "layer.zip" - layer_name = "lambda_layer_dev" + layer_name = "lambda_layer" script_dir = "../scripts" } @@ -105,7 +105,7 @@ resource "null_resource" "prepare_layer" { resource "aws_s3_object" "lambda_layer_zip" { bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "lambda_layer/${local.layer_name}/${local.layer_zip}" + key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] } @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_layer_bucket.id #bucket instead of id + s3_bucket = aws_s3_bucket.lambda_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From cbf1d083dc0bf4d78da83cb169da49731f8ace65 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:18:22 +0100 Subject: fix(tf): correct s3_bucket value for lambda_layer --- terraform/lambda.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 8a4207d..bf96747 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -113,7 +113,7 @@ resource "aws_s3_object" "lambda_layer_zip" { resource "aws_lambda_layer_version" "lambda_layer" { layer_name = local.layer_name compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_bucket.bucket + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] -- cgit v1.2.3 From 024de7d7947f46cf6c0c829dc29eb8298e029576 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:37:54 +0100 Subject: fix(make_layer_zip): change folder structure of layer.zip --- scripts/make_layer_zip.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/scripts/make_layer_zip.sh b/scripts/make_layer_zip.sh index 0e7560f..eabe301 100755 --- a/scripts/make_layer_zip.sh +++ b/scripts/make_layer_zip.sh @@ -1,7 +1,8 @@ # Description: Make the zip file for the layer cd "$(dirname "$0")/.." -mkdir tmp_python -pip3 install --upgrade -r requirements.txt -t tmp_python/ -zip -r layer.zip tmp_python -rm -r tmp_python/ +mkdir -p python/lib/python3.11/site-packages +pip3 install --upgrade -r requirements.txt -t python/lib/python3.11/site-packages +rm layer.zip +zip -r layer.zip python +rm -r python/ -- cgit v1.2.3 From 4b3b80a2f2177456ed6c2857a7ae0987d7304360 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 15:40:01 +0100 Subject: chore(tf): remove unused requirements variable --- terraform/lambda.tf | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index bf96747..72aae04 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -90,11 +90,10 @@ resource "aws_lambda_function" "load_lambda" { # Lambda Layer Specification locals { - layer_dir = "../" - requirements = "requirements.txt" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" } resource "null_resource" "prepare_layer" { -- cgit v1.2.3 From b9f3576771c8af8933d23e95f7863f63e2bbc6aa Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Mon, 19 Aug 2024 15:43:28 +0100 Subject: wip: fixed broken tests; hashed out test_error_retrieving_object --- src/extract_lambda.py | 1 + tests/test_extract_lambda.py | 49 ++++++++++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 4168e27..217efdb 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -90,6 +90,7 @@ def extract_bucket(client=boto3.client("s3")): extract_bucket_filter = [ bucket["Name"] for bucket in response["Buckets"] if "extract" in bucket["Name"] ] + return extract_bucket_filter[0] diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index e94a8a4..665e419 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -1,11 +1,13 @@ +import boto3.exceptions +import botocore.exceptions import pytest import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables -import os +from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables, extract_bucket import logging +import os @pytest.fixture(scope='class') def mock_config(): @@ -16,7 +18,7 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.get_config", return_value=env_vars) as mock_config: + with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: yield mock_config @@ -24,7 +26,7 @@ def mock_config(): def aws_credentials(): os.environ["AWS_ACCESS_KEY_ID"] = 'testing' os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURIT_TOKEN"] = 'testing' + os.environ["AWS_SECURITY_TOKEN"] = 'testing' os.environ["AWS_SESSION_TOKEN"] = 'testing' os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' @@ -33,6 +35,14 @@ def s3_client(aws_credentials): with mock_aws(): yield boto3.client('s3') +@pytest.fixture(scope='class') +def s3_mock_bucket(s3_client): + bucket = s3_client.create_bucket(Bucket='extract_bucket', + CreateBucketConfiguration={ + 'LocationConstraint': 'eu-west-2' + }) + return bucket + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): @@ -42,35 +52,34 @@ class TestListExistingS3Files: list_existing_s3_files(client=s3_client) assert 'Error listing S3 objects' in caplog.text - def test_error_if_bucket_is_empty(self, s3_client, caplog): + def test_error_if_bucket_is_empty(self, s3_client, caplog, s3_mock_bucket): + list_existing_s3_files('extract_bucket', client=s3_client) + assert 'The bucket is empty' in caplog.text - s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) - list_existing_s3_files(client=s3_client) - assert 'The bucket is empty' in caplog.text - def test_error_retrieving_object(self, s3_client, caplog): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - list_existing_s3_files(bucket_name='test_bucket', client=s3_client) + # def test_error_retrieving_object(self, s3_client, caplog, s3_mock_bucket): + # s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - assert 'Error retrieving S3 object ' in caplog.text + # list_existing_s3_files(bucket_name='extract_bucket', client=s3_client) - def test_retrieves_file_content(self, s3_client, caplog): - result = list_existing_s3_files(client=s3_client) + # assert 'Error retrieving S3 object dummy.txt: ClientError' in caplog.text + + + def test_retrieves_file_content(self, s3_client, caplog, s3_mock_bucket): + s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') + result = list_existing_s3_files('extract_bucket', client=s3_client) - assert list(result.values()) == ['This is a test file.'] + assert list(result.values()) == ['This is a test file.'] class TestConnectToDatabase: - def test_connect_to_database(mock_conn, mock_config): + def test_connect_to_database(mock_conn, mock_config): ##had mock_config in param with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( host="abc", user="def", port="5432", password="password", database="db" ) - def test_database_error(self, mock_config): + def test_database_error(self, mock_config): ##had mock_config in param with pytest.raises(DBConnectionException): connect_to_database() -- cgit v1.2.3 From c3c45c0d133ce32d48f1c72a0ac54f291038b1e7 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 15:56:48 +0100 Subject: wip: fixing last test --- src/extract_lambda.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 4168e27..533bf82 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -147,12 +147,13 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): WHERE table_schema='public' AND table_type='BASE TABLE';""" ) for table in tables: + print(tables) table_name = table[0] rows = db.run( f"SELECT * FROM {identifier(table_name)} " "WHERE last_updated >= :latest;", latest={datetime.strftime(latest_timestamp, "%H-%m-%d %H:%M:%S")}, ) - + print('rows', rows) # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" @@ -183,6 +184,6 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): else: load_status["no change"].append(table_name) logger.info( - f"No new data in {table_name} name. Latest data retrieved is from {latest_timestamp}." + f"No new data" ) return load_status -- cgit v1.2.3 From e4b66476a174edb68992b00b37bef2d0e0be3969 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 15:57:14 +0100 Subject: wip: fixing last test --- tests/test_extract_lambda.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 4a5157b..01d7add 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -181,24 +181,27 @@ class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): logger = logging.getLogger() logger.info('Testing now.') - caplog.set_level(logging.ERROR) + caplog.set_level(logging.INFO) #### - queries = ["SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", - "SELECT * FROM Fruits;", - "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'"] + queries = [ + "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", + "SELECT * FROM Fruits;", + "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'" + ] return_values = [[['Fruits']], - [['Vegetable','Sour','Green'],['Berry','Sweet','Red']], - [['Food_type'],['Flavour'],['Colour']]] # why are individual column names in lists + [['Vegetable','Sour','Green','2022-11-03 14:20:49.962'],['Berry','Sweet','Red','2022-11-03 14:20:49.962']], + [['Food_type'],['Flavour'],['Colour'],['last_updated']]] # why are individual column names in lists vals = dict(zip(queries,return_values)) # {"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';": [['Fruits']], 'SELECT * FROM Fruits;': [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'": [['Food_type'], ['Flavour'], ['Colour']]} with patch('src.extract_lambda.Connection') as mock_db: mock_db().run.side_effect = return_values s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' - existing_files = {s3_key: 'Food_type,Flavour,Colour\nFruit,Sour,Green\nBerry,Sweet,Red'} + existing_files = {s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red, 2022-11-03 14:20:49.962'} s3_client.create_bucket(Bucket='test_extract_bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) - print(s3_client.list_buckets) - s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) + s3_client.upload_file('tests/dummy_identical.csv', 'test_extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) - assert 'No new data.' in caplog.text \ No newline at end of file + print('logger', logger.info('hello')) + print('our test', caplog.text) + assert 'No new data' in caplog.text \ No newline at end of file -- cgit v1.2.3 From ec3523a20d5ece3ce1d7b59072f5948f4fa40810 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 15:57:40 +0100 Subject: amend dummy_identical --- tests/dummy_identical.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/dummy_identical.csv b/tests/dummy_identical.csv index fdd8993..e44e9fc 100644 --- a/tests/dummy_identical.csv +++ b/tests/dummy_identical.csv @@ -1,4 +1,4 @@ -Food_type,Flavour,Colour -Vegetable,Sour,Green -Berry,Sweet,Red +Food_type,Flavour,Colour,last_updated +Vegetable,Sour,Green,2022-11-03 14:20:49.962 +Berry,Sweet,Red,2022-11-03 14:20:49.962 -- cgit v1.2.3 From 333822a70640712ac57036d37f7d8ac0787e9cc0 Mon Sep 17 00:00:00 2001 From: HastarTara Date: Mon, 19 Aug 2024 16:19:16 +0100 Subject: bugfixing --- tests/test_extract_lambda.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 01d7add..a4e8f2b 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -179,29 +179,36 @@ class TestConnectToDatabase: class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') caplog.set_level(logging.INFO) - #### + + # Mock return values for database queries queries = [ "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", - "SELECT * FROM Fruits;", - "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'" - ] - return_values = [[['Fruits']], - [['Vegetable','Sour','Green','2022-11-03 14:20:49.962'],['Berry','Sweet','Red','2022-11-03 14:20:49.962']], - [['Food_type'],['Flavour'],['Colour'],['last_updated']]] # why are individual column names in lists - vals = dict(zip(queries,return_values)) - # {"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';": [['Fruits']], 'SELECT * FROM Fruits;': [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'": [['Food_type'], ['Flavour'], ['Colour']]} + "SELECT * FROM Fruits WHERE last_updated > :latest;", + "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits';" + ] + return_values = [ + [['Fruits']], + [], # No new rows with a more recent last_updated timestamp + [['Food_type'], ['Flavour'], ['Colour'], ['last_updated']] + ] + vals = dict(zip(queries, return_values)) + # Patch the database connection and set return values for queries with patch('src.extract_lambda.Connection') as mock_db: mock_db().run.side_effect = return_values s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' - existing_files = {s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red, 2022-11-03 14:20:49.962'} + existing_files = { + s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red,2022-11-03 14:20:49.962' + } + + # Simulate S3 bucket and file setup s3_client.create_bucket(Bucket='test_extract_bucket', - CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) + CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) s3_client.upload_file('tests/dummy_identical.csv', 'test_extract_bucket', s3_key) + + # Run the process_and_upload_tables function process_and_upload_tables(mock_db(), existing_files, client=s3_client) - print('logger', logger.info('hello')) - print('our test', caplog.text) - assert 'No new data' in caplog.text \ No newline at end of file + + # Assert that the log contains "No new data" + assert 'No new data' in caplog.text -- cgit v1.2.3 From 8b4e78b781617f68554efebcda75d982a382f650 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 16:31:50 +0100 Subject: fix(tf): fix permissions for bucket/object access --- terraform/iam.tf | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 0e5fa6d..7585ff8 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -28,17 +28,19 @@ resource "aws_iam_role" "multi_service_role" { ######################################################################## # S3 SETUP # Description: allows allows retention/tagging/access control settings -# Lambda IAM Policy for S3 Write +# Lambda IAM Policy for S3 ######################################################################## # S3 DEFINE POLICY data "aws_iam_policy_document" "s3_data_policy_doc" { statement { + effect = "Allow" actions = [ "s3:PutObject", "s3:PutObjectRetention", "s3:PutObjectTagging", - "s3:PutObjectAcl" + "s3:PutObjectAcl", + "s3:ListObjects" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -46,6 +48,17 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "${aws_s3_bucket.lambda_code_bucket.arn}/*", ] } + + statement { + effect = "Allow" + actions = [ + "s3:ListBuckets", + "s3:ListAllMyBuckets" + ] + resources = [ + "arn:aws:s3:::*", + ] + } } -- cgit v1.2.3 From 982b8fa318c9065bd9037d14c56abcd126252978 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 16:33:26 +0100 Subject: add working process and upload tables test --- src/extract_lambda.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 533bf82..5a5a631 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -150,8 +150,8 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): print(tables) table_name = table[0] rows = db.run( - f"SELECT * FROM {identifier(table_name)} " "WHERE last_updated >= :latest;", - latest={datetime.strftime(latest_timestamp, "%H-%m-%d %H:%M:%S")}, + f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", + latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) print('rows', rows) # Creating a temporary file path and writing the column name to it followed by each row of data -- cgit v1.2.3 From 4f629e532a1e989096985dc9cd9e6f03f7b44354 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 16:33:46 +0100 Subject: add working process and upload tables test --- tests/test_extract_lambda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index a4e8f2b..3405743 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -209,6 +209,7 @@ class TestProcessAndUploadTables: # Run the process_and_upload_tables function process_and_upload_tables(mock_db(), existing_files, client=s3_client) - # Assert that the log contains "No new data" assert 'No new data' in caplog.text + + # process and upload tables needs more tests \ No newline at end of file -- cgit v1.2.3 From 3e35364cc425db8738fb247a18f91c052c49fa8f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 16:36:39 +0100 Subject: chore: remove redundant test folder --- test/test_secrets_manager.py | 39 --------------------------------------- 1 file changed, 39 deletions(-) delete mode 100644 test/test_secrets_manager.py diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py deleted file mode 100644 index cb4ec15..0000000 --- a/test/test_secrets_manager.py +++ /dev/null @@ -1,39 +0,0 @@ -from src.secrets_manager import sm_client, create_secret, list_secret -import boto3 -from moto import mock_aws -import json -import pytest -import os - -pytest.fixture(scope="class") - - -def mock_aws_credentials(): - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - os.environ["AWS_SECURITY_TOKEN"] = "testing" - os.environ["AWS_SESSION_TOKEN"] = "testing" - os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" - - -@pytest.fixture(scope="class") -def mock_sm_client(mock_aws_credentials): - with mock_aws(): - yield boto3.client("secretsmanager") - - -def test_create_secret_stores_secrets(mock_sm_client): - cohort_id = "test_cohort_id" - user = "test_user_id" - password = "test_password" - host = "test_host" - database = "test_database" - port = "test_port" - - secret_name = "test_secret" - response = create_secret( - mock_sm_client, secret_name, cohort_id, user, password, host, database, port - ) - - assert response["Name"] == secret_name -- cgit v1.2.3 From 91d2e615a6af595898de2e329299c9cf42fc74f7 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:00:10 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in b9f3576 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/64 --- tests/test_extract_lambda.py | 81 +++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 665e419..02e3d3c 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -5,11 +5,18 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables, extract_bucket +from src.extract_lambda import ( + list_existing_s3_files, + connect_to_database, + DBConnectionException, + process_and_upload_tables, + extract_bucket, +) import logging import os -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_config(): env_vars = { "host": "abc", @@ -18,44 +25,47 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: + with patch( + "src.extract_lambda.retrieve_secrets", return_value=env_vars + ) as mock_config: yield mock_config -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def aws_credentials(): - os.environ["AWS_ACCESS_KEY_ID"] = 'testing' - os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURITY_TOKEN"] = 'testing' - os.environ["AWS_SESSION_TOKEN"] = 'testing' - os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_client(aws_credentials): with mock_aws(): - yield boto3.client('s3') + yield boto3.client("s3") + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_mock_bucket(s3_client): - bucket = s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) + bucket = s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) return bucket + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) list_existing_s3_files(client=s3_client) - assert 'Error listing S3 objects' in caplog.text + assert "Error listing S3 objects" in caplog.text def test_error_if_bucket_is_empty(self, s3_client, caplog, s3_mock_bucket): - list_existing_s3_files('extract_bucket', client=s3_client) - assert 'The bucket is empty' in caplog.text - + list_existing_s3_files("extract_bucket", client=s3_client) + assert "The bucket is empty" in caplog.text # def test_error_retrieving_object(self, s3_client, caplog, s3_mock_bucket): # s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') @@ -64,33 +74,36 @@ class TestListExistingS3Files: # assert 'Error retrieving S3 object dummy.txt: ClientError' in caplog.text - def test_retrieves_file_content(self, s3_client, caplog, s3_mock_bucket): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - result = list_existing_s3_files('extract_bucket', client=s3_client) + s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") + result = list_existing_s3_files("extract_bucket", client=s3_client) + + assert list(result.values()) == ["This is a test file."] - assert list(result.values()) == ['This is a test file.'] class TestConnectToDatabase: - def test_connect_to_database(mock_conn, mock_config): ##had mock_config in param - with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: + # had mock_config in param + def test_connect_to_database(mock_conn, mock_config): + with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( - host="abc", user="def", port="5432", password="password", database="db" + host="abc", user="def", port="5432", password="password", database="db" ) - def test_database_error(self, mock_config): ##had mock_config in param + def test_database_error(self, mock_config): # had mock_config in param with pytest.raises(DBConnectionException): connect_to_database() def test_logs_interface_error(self, caplog): logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert 'Interface error' in caplog.text -''' + assert "Interface error" in caplog.text + + +""" class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): logger = logging.getLogger() @@ -115,4 +128,4 @@ class TestProcessAndUploadTables: s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) assert 'No new data.' in caplog.text -''' \ No newline at end of file +""" -- cgit v1.2.3 From b80ad74122609fca98597d9a04518df855b58aed Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:10:22 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 4a23069 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/64 --- tests/test_extract_lambda.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index b1894cc..a43ae0a 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -55,6 +55,7 @@ def s3_mock_bucket(s3_client): ) return bucket + class TestLambdaHandler: def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): mock_db = MagicMock() @@ -138,6 +139,7 @@ class TestLambdaHandler: mock_list_existing_s3_files.assert_not_called() mock_process_and_upload_tables.assert_not_called() + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): logger = logging.getLogger() @@ -175,4 +177,4 @@ class TestConnectToDatabase: caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert "Interface error" in caplog.text \ No newline at end of file + assert "Interface error" in caplog.text -- cgit v1.2.3 From a42d030fb663ad7eb040498cfc5f0627a27d6cc6 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:11:44 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 4f629e5 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/65 --- src/extract_lambda.py | 8 +++----- tests/test_extract_lambda.py | 34 ++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 19 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 5a5a631..9b17ef2 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -151,9 +151,9 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): table_name = table[0] rows = db.run( f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", - latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, + latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) - print('rows', rows) + print("rows", rows) # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" @@ -183,7 +183,5 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): logger.error(f"Error uploading to S3: {e}") else: load_status["no change"].append(table_name) - logger.info( - f"No new data" - ) + logger.info(f"No new data") return load_status diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 3405743..5a1c5b2 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -12,7 +12,7 @@ from src.extract_lambda import ( DBConnectionException, lambda_handler, process_and_upload_tables, - retrieve_secrets + retrieve_secrets, ) @@ -25,7 +25,9 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: + with patch( + "src.extract_lambda.retrieve_secrets", return_value=env_vars + ) as mock_config: yield mock_config @@ -185,31 +187,35 @@ class TestProcessAndUploadTables: queries = [ "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", "SELECT * FROM Fruits WHERE last_updated > :latest;", - "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits';" + "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits';", ] return_values = [ - [['Fruits']], + [["Fruits"]], [], # No new rows with a more recent last_updated timestamp - [['Food_type'], ['Flavour'], ['Colour'], ['last_updated']] + [["Food_type"], ["Flavour"], ["Colour"], ["last_updated"]], ] vals = dict(zip(queries, return_values)) # Patch the database connection and set return values for queries - with patch('src.extract_lambda.Connection') as mock_db: + with patch("src.extract_lambda.Connection") as mock_db: mock_db().run.side_effect = return_values - s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' + s3_key = "Fruits/2024/08/15/Fruits_16:46:30.csv" existing_files = { - s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red,2022-11-03 14:20:49.962' + s3_key: "Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red,2022-11-03 14:20:49.962" } # Simulate S3 bucket and file setup - s3_client.create_bucket(Bucket='test_extract_bucket', - CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) - s3_client.upload_file('tests/dummy_identical.csv', 'test_extract_bucket', s3_key) - + s3_client.create_bucket( + Bucket="test_extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + s3_client.upload_file( + "tests/dummy_identical.csv", "test_extract_bucket", s3_key + ) + # Run the process_and_upload_tables function process_and_upload_tables(mock_db(), existing_files, client=s3_client) # Assert that the log contains "No new data" - assert 'No new data' in caplog.text + assert "No new data" in caplog.text - # process and upload tables needs more tests \ No newline at end of file + # process and upload tables needs more tests -- cgit v1.2.3 From b499d78dc660017694ec683c90aba3f558c00669 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:14:07 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in f014d1a according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/65 --- tests/test_extract_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 347ef22..3931cfc 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -180,6 +180,7 @@ class TestConnectToDatabase: connect_to_database() assert "Interface error" in caplog.text + class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): caplog.set_level(logging.INFO) @@ -218,4 +219,3 @@ class TestProcessAndUploadTables: process_and_upload_tables(mock_db(), existing_files, client=s3_client) # Assert that the log contains "No new data" assert "No new data" in caplog.text - -- cgit v1.2.3 From e537bdef11d1d518d4df1c057f3624e3fe6da24d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:05:41 +0100 Subject: infra(tf): remove rds.tf --- terraform/rds.tf | 70 -------------------------------------------------------- 1 file changed, 70 deletions(-) delete mode 100644 terraform/rds.tf diff --git a/terraform/rds.tf b/terraform/rds.tf deleted file mode 100644 index a013fb3..0000000 --- a/terraform/rds.tf +++ /dev/null @@ -1,70 +0,0 @@ -# data "aws_availability_zones" "available" {} - -# module "vpc" { -# source = "terraform-aws-modules/vpc/aws" -# version = "5.12.1" - -# name = var.project_name -# cidr = "10.0.0.0/16" -# azs = data.aws_availability_zones.available.names -# public_subnets = ["10.0.4.0/24", "10.0.5.0/24", "10.0.6.0/24"] -# enable_dns_hostnames = true -# enable_dns_support = true -# } - -# resource "aws_db_subnet_group" "Terrific-Totes-sub-gr" { -# name = "tt-db-subnet" -# subnet_ids = module.vpc.public_subnets - -# tags = { -# Name = "${var.project_name}" -# } -# } - -# resource "aws_security_group" "rds" { -# name = "${var.project_name}-rds" -# vpc_id = module.vpc.vpc_id - -# ingress { -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# } - -# egress { -# from_port = 5432 -# to_port = 5432 -# protocol = "tcp" -# cidr_blocks = ["0.0.0.0/0"] -# } - -# tags = { -# Name = "${var.project_name}-rds" -# } -# } - -# resource "aws_db_parameter_group" "Terrific-Totes-param-gr" { -# name = "tt-db-param" -# family = "postgres14" - -# parameter { -# name = "log_connections" -# value = "1" -# } -# } - -# resource "aws_db_instance" "terrific-totes-rds" { -# db_name = var.project_name -# instance_class = "db.t3.micro" -# allocated_storage = 5 -# engine = "postgres" -# engine_version = "14.10" -# username = "" -# password = "" -# db_subnet_group_name = aws_db_subnet_group.Terrific-Totes-sub-gr.name -# vpc_security_group_ids = [aws_security_group.rds.id] -# parameter_group_name = aws_db_parameter_group.Terrific-Totes-param-gr.name -# publicly_accessible = false -# skip_final_snapshot = true -# } -- cgit v1.2.3 From 56b2c376a925132f3bf2c7e6cad4911400955129 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:07:30 +0100 Subject: infra(tf): enforce version constraint on terraform --- terraform/main.tf | 1 + 1 file changed, 1 insertion(+) diff --git a/terraform/main.tf b/terraform/main.tf index 310a251..206fc74 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -1,4 +1,5 @@ terraform { + required_version = ">= 1.8.0" required_providers { aws = { source = "hashicorp/aws" -- cgit v1.2.3 From 35bf4e8668309cb28175ef0224a6bce453abb47f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:19:44 +0100 Subject: chore(tf): replace static tag values in main.tf with variables --- terraform/main.tf | 8 ++++---- terraform/vars.tf | 20 ++++++++++++++++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 206fc74..5ccbec2 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -25,11 +25,11 @@ provider "aws" { region = "eu-west-2" default_tags { tags = { - ProjectName = "Terrific-Totes" - Team = "Team-Bentley" - Environment = "Dev" - GitHubRepo = "de-project-bentley" + ProjectName = var.project_name + Environment = var.environment ManagedBy = "Terraform" + GitHubRepo = var.github_repo + Team = var.team_name } } } diff --git a/terraform/vars.tf b/terraform/vars.tf index 3c88731..1adbcf7 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -33,6 +33,26 @@ variable "project_name" { default = "tt" } +variable "aws_region" { + type = string + default = "eu-west-2" +} + +variable "environment" { + type = string + default = "dev" +} + +variable "github_repo" { + type = string + default = "de-project-bentley" +} + +variable "team_name" { + type = string + default = "Team-Bentley" +} + data "aws_caller_identity" "current" {} data "aws_region" "current" {} -- cgit v1.2.3 From 95c4fe80aea75a9a63b1cfd85abadaab6b96b876 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 19:24:23 +0100 Subject: infra(tf): add state file encryption --- terraform/main.tf | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/terraform/main.tf b/terraform/main.tf index 5ccbec2..33c760c 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -15,9 +15,10 @@ terraform { } } backend "s3" { - bucket = "bentley-project-secrets" - key = "bentley-project/terraform.tfstate" - region = "eu-west-2" + bucket = "bentley-project-secrets" + key = "bentley-project/terraform.tfstate" + region = "eu-west-2" + encrypt = true } } -- cgit v1.2.3 From 22e7de562e62495e547eeff187d86bf9524ae5ca Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:22:16 +0100 Subject: feat: create shell script for terraform destroy/apply --- scripts/deploy.sh | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100755 scripts/deploy.sh diff --git a/scripts/deploy.sh b/scripts/deploy.sh new file mode 100755 index 0000000..0446184 --- /dev/null +++ b/scripts/deploy.sh @@ -0,0 +1,15 @@ +# Deploy Script +# Description: Deploy and destroy Terraform +# WARNING: This will most likely destroy any current infrastructure if protections +# are not in place. Be careful! + +echo "WARNING: This script will destroy any infrastructure for testing." +echo "It should not be used once a proper deployment has been setup." +echo "Would you like to continue?" + +select yn in "Yes" "No"; do + case $yn in + Yes ) cd ../terraform/; terraform destroy -auto-approve; terraform apply -auto-approve; terraform destroy -auto-approve; break;; + No ) exit;; + esac +done -- cgit v1.2.3 From 50302044c64e414ffe0435908146bc718bf6bed9 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:24:35 +0100 Subject: feat(deploy.sh): exit if any command returns non-zero status --- scripts/deploy.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 0446184..16a9e13 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -3,6 +3,9 @@ # WARNING: This will most likely destroy any current infrastructure if protections # are not in place. Be careful! +# Exit if any command has a non-zero status +set -e + echo "WARNING: This script will destroy any infrastructure for testing." echo "It should not be used once a proper deployment has been setup." echo "Would you like to continue?" -- cgit v1.2.3 From 18f7ea0e4254890cd810ff2ee257306d94467faf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:49:28 +0100 Subject: refactor: Improve deploy script user interaction --- scripts/deploy.sh | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index 16a9e13..d7d18ff 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -11,8 +11,30 @@ echo "It should not be used once a proper deployment has been setup." echo "Would you like to continue?" select yn in "Yes" "No"; do - case $yn in - Yes ) cd ../terraform/; terraform destroy -auto-approve; terraform apply -auto-approve; terraform destroy -auto-approve; break;; - No ) exit;; - esac + case $yn in + Yes) + cd ../terraform/ + echo "Would you like to destroy the current infrastructure?" + select destroy_1 in "Yes" "No"; do + case $destroy_1 in + Yes) + terraform destroy + break + ;; + No) + echo "Skipping initial destroy..." + break + ;; + esac + done + + terraform apply -auto-approve + + break + ;; + No) + echo "Operation cancelled..." + exit + ;; + esac done -- cgit v1.2.3 From 68be61c22703d56a10e654702d15407231385b65 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:51:30 +0100 Subject: feat: ask user if they want to destroy new infrastructure --- scripts/deploy.sh | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index d7d18ff..e56088e 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -28,7 +28,21 @@ select yn in "Yes" "No"; do esac done - terraform apply -auto-approve + terraform apply + + echo "Would you like to destroy the newly-created infrastructure?" + select destroy_2 in "Yes" "No"; do + case $destroy_2 in + Yes) + terraform destroy + break + ;; + No) + echo "Skipping final destroy... Infrastructure will remain." + break + ;; + esac + done break ;; -- cgit v1.2.3 From 57e855a797f225cd77401e85a671cde95e07ee70 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:55:19 +0100 Subject: style(tf): improve legibility of lambda.tf sections --- terraform/lambda.tf | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index 72aae04..b7b362b 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,4 +1,7 @@ -# Extract Lambda Function +########################### +# Extract Lambda Function # +########################### + data "archive_file" "extract_lambda_zip" { type = "zip" source_file = "${path.module}/../src/extract_lambda.py" @@ -28,7 +31,10 @@ resource "aws_lambda_function" "extract_lambda" { depends_on = [aws_s3_object.extract_lambda_code] } -# Transform Lambda Function +############################# +# Transform Lambda Function # +############################# + data "archive_file" "transform_lambda_zip" { type = "zip" source_file = "${path.module}/../src/transform_lambda.py" @@ -58,7 +64,10 @@ resource "aws_lambda_function" "transform_lambda" { depends_on = [aws_s3_object.transform_lambda_code] } -# Load Lambda Function +######################## +# Load Lambda Function # +######################## + data "archive_file" "load_lambda_zip" { type = "zip" source_file = "${path.module}/../src/load_lambda.py" @@ -88,7 +97,10 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -# Lambda Layer Specification +###################### +# Lambda Layer Setup # +###################### + locals { layer_dir = "../" layer_zip = "layer.zip" -- cgit v1.2.3 From d0e7b1304efe4ab6de2dc5bef1691b389a5bc449 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 20:58:46 +0100 Subject: refactor(tf): move sections in lambda.tf for better readability --- terraform/lambda.tf | 69 ++++++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 32 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b7b362b..aa730c1 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -1,3 +1,40 @@ +#################### +# Common Variables # +#################### + +locals { + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" +} + +###################### +# Lambda Layer Setup # +###################### + +resource "null_resource" "prepare_layer" { + provisioner "local-exec" { + command = "bash ${local.script_dir}/make_layer_zip.sh" + } +} + +resource "aws_s3_object" "lambda_layer_zip" { + bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id + key = "${local.layer_name}/${local.layer_zip}" + source = "${local.layer_dir}/${local.layer_zip}" + depends_on = [null_resource.prepare_layer] +} + +resource "aws_lambda_layer_version" "lambda_layer" { + layer_name = local.layer_name + compatible_runtimes = ["python3.11"] + s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket + s3_key = aws_s3_object.lambda_layer_zip.key + skip_destroy = true + depends_on = [aws_s3_object.lambda_layer_zip] +} + ########################### # Extract Lambda Function # ########################### @@ -97,35 +134,3 @@ resource "aws_lambda_function" "load_lambda" { depends_on = [aws_s3_object.load_lambda_code] } -###################### -# Lambda Layer Setup # -###################### - -locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" -} - -resource "null_resource" "prepare_layer" { - provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" - } -} - -resource "aws_s3_object" "lambda_layer_zip" { - bucket = aws_s3_bucket.lambda_code_bucket.id #bucket instead of id - key = "${local.layer_name}/${local.layer_zip}" - source = "${local.layer_dir}/${local.layer_zip}" - depends_on = [null_resource.prepare_layer] -} - -resource "aws_lambda_layer_version" "lambda_layer" { - layer_name = local.layer_name - compatible_runtimes = ["python3.11"] - s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket - s3_key = aws_s3_object.lambda_layer_zip.key - skip_destroy = true - depends_on = [aws_s3_object.lambda_layer_zip] -} -- cgit v1.2.3 From b75b7197f08e933cfcd4b69ad5182a01c2886d8e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:05:33 +0100 Subject: refactor: change directory at start of the script to terraform folder --- scripts/deploy.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/deploy.sh b/scripts/deploy.sh index e56088e..f631bbc 100755 --- a/scripts/deploy.sh +++ b/scripts/deploy.sh @@ -6,6 +6,9 @@ # Exit if any command has a non-zero status set -e +# Change current directory to terraform folder at the start +cd ../terraform/ + echo "WARNING: This script will destroy any infrastructure for testing." echo "It should not be used once a proper deployment has been setup." echo "Would you like to continue?" @@ -13,7 +16,6 @@ echo "Would you like to continue?" select yn in "Yes" "No"; do case $yn in Yes) - cd ../terraform/ echo "Would you like to destroy the current infrastructure?" select destroy_1 in "Yes" "No"; do case $destroy_1 in -- cgit v1.2.3 From cfd6b462a874da77ada8facb3b2a3c0e85059fa4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:07:10 +0100 Subject: infra(tf): only create layer zip if doesn't exist --- terraform/lambda.tf | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index aa730c1..b1e0d8e 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -3,10 +3,11 @@ #################### locals { - layer_dir = "../" - layer_zip = "layer.zip" - layer_name = "lambda_layer" - script_dir = "../scripts" + layer_dir = "../" + layer_zip = "layer.zip" + layer_name = "lambda_layer" + script_dir = "../scripts" + layer_zip_path = "${local.layer_dir}/${local.layer_zip}" } ###################### @@ -14,8 +15,13 @@ locals { ###################### resource "null_resource" "prepare_layer" { + + triggers = { + layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" + } + provisioner "local-exec" { - command = "bash ${local.script_dir}/make_layer_zip.sh" + command = "if [ ! -f ${local.layer_zip_path} ]; then bash ${local.script_dir}/make_layer_zip.sh; fi" } } -- cgit v1.2.3 From f035f60c7ece05b70275760238c5513b8f113310 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:09:26 +0100 Subject: docs(tf): add information about layer zip creation --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index b1e0d8e..fc10431 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -16,6 +16,8 @@ locals { resource "null_resource" "prepare_layer" { + # New change: only run the script if the layer zip does not exist + triggers = { layer_zip_exists = fileexists(local.layer_zip_path) ? "exists" : "not_exists" } -- cgit v1.2.3 From 40c2952e628a92e63b3468be4d49f44a234cacce Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:13:03 +0100 Subject: infra(tf): add md5/source_code_hash checks for lambda layer --- terraform/lambda.tf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index fc10431..f8e7515 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -32,6 +32,7 @@ resource "aws_s3_object" "lambda_layer_zip" { key = "${local.layer_name}/${local.layer_zip}" source = "${local.layer_dir}/${local.layer_zip}" depends_on = [null_resource.prepare_layer] + etag = fileexists(local.layer_zip_path) ? filemd5(local.layer_zip_path) : null } resource "aws_lambda_layer_version" "lambda_layer" { @@ -39,6 +40,7 @@ resource "aws_lambda_layer_version" "lambda_layer" { compatible_runtimes = ["python3.11"] s3_bucket = aws_s3_bucket.lambda_code_bucket.bucket s3_key = aws_s3_object.lambda_layer_zip.key + source_code_hash = fileexists(local.layer_zip_path) ? filebase64sha256(local.layer_zip_path) : null skip_destroy = true depends_on = [aws_s3_object.lambda_layer_zip] } -- cgit v1.2.3 From e5715bc33d4470ceccb17c6853c3e52d4b1035d3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:20:09 +0100 Subject: chore(tf): add tags to s3 buckets --- terraform/s3.tf | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/terraform/s3.tf b/terraform/s3.tf index d5cdee3..97910c8 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,14 +1,24 @@ ### EXTRACT BUCKET SET-UP resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" + + tags = { + Name = "Ingestion Bucket" + } } ### TRANSFORM BUCKET SET-UP resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" + tags = { + Name = "Transform Bucket" + } } ### LAMBDA BUCKET resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" + tags = { + Name = "Load Bucket" + } } -- cgit v1.2.3 From 507b3071633fccc9aa1411880dd984ca346a141b Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:22:29 +0100 Subject: docs(tf): improve legibility of s3.tf sections --- terraform/s3.tf | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/terraform/s3.tf b/terraform/s3.tf index 97910c8..6ff58fd 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -1,4 +1,7 @@ -### EXTRACT BUCKET SET-UP +######################## +# EXTRACT BUCKET SETUP # +######################## + resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" @@ -7,7 +10,10 @@ resource "aws_s3_bucket" "extract_bucket" { } } -### TRANSFORM BUCKET SET-UP +########################## +# TRANSFORM BUCKET SETUP # +########################## + resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" tags = { @@ -15,7 +21,10 @@ resource "aws_s3_bucket" "transform_bucket" { } } -### LAMBDA BUCKET +####################### +# LAMBDA BUCKET SETUP # +####################### + resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" tags = { -- cgit v1.2.3 From 1cb84bd663261c416a516b0dc59dbf8d62c4c1a7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:23:07 +0100 Subject: docs(tf): correct lambda bucket name tag --- terraform/s3.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/s3.tf b/terraform/s3.tf index 6ff58fd..3e0f5b7 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -28,6 +28,6 @@ resource "aws_s3_bucket" "transform_bucket" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" tags = { - Name = "Load Bucket" + Name = "Lambda Bucket" } } -- cgit v1.2.3 From 795c7c2917c2780e8ffdf0716cbedf3426dcbd5e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:25:21 +0100 Subject: infra(tf): experimental - add versioning to protect against accidental deletes/overwrites" --- terraform/s3.tf | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/terraform/s3.tf b/terraform/s3.tf index 3e0f5b7..edfe0a0 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -10,6 +10,13 @@ resource "aws_s3_bucket" "extract_bucket" { } } +resource "aws_s3_bucket_versioning" "extract_bucket_versioning" { + bucket = aws_s3_bucket.extract_bucket.id + versioning_configuration { + status = "Enabled" + } +} + ########################## # TRANSFORM BUCKET SETUP # ########################## @@ -21,6 +28,14 @@ resource "aws_s3_bucket" "transform_bucket" { } } + +resource "aws_s3_bucket_versioning" "transform_bucket_versioning" { + bucket = aws_s3_bucket.transform_bucket.id + versioning_configuration { + status = "Enabled" + } +} + ####################### # LAMBDA BUCKET SETUP # ####################### -- cgit v1.2.3 From 1bbc12702a8fa6d5139440c9d04e5bfabd96581d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:32:32 +0100 Subject: infra(tf): add versioning to lambda_code_bucket --- terraform/s3.tf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/terraform/s3.tf b/terraform/s3.tf index edfe0a0..d17a4fe 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -46,3 +46,10 @@ resource "aws_s3_bucket" "lambda_code_bucket" { Name = "Lambda Bucket" } } + +resource "aws_s3_bucket_versioning" "lambda_bucket_versioning" { + bucket = aws_s3_bucket.lambda_code_bucket.id + versioning_configuration { + status = "Enabled" + } +} -- cgit v1.2.3 From b9a3d9dbaa1eedc25d5f8d12bd2be1a8a3841b42 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:39:05 +0100 Subject: docs(tf): improve legibility of events.tf sections --- terraform/events.tf | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/terraform/events.tf b/terraform/events.tf index 263141f..c2efbbc 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -1,3 +1,7 @@ +################# +# Random String # +################# + resource "random_string" "eventbridge_suffix" { length = 8 special = false @@ -16,6 +20,10 @@ resource "random_string" "s3_transform_suffix" { upper = false } +############################# +# EventBridge Configuration # +############################# + resource "aws_cloudwatch_event_rule" "lambda_trigger" { name = "lambda-scheduled-trigger" description = "Schedule to trigger the Lambda function" @@ -41,7 +49,10 @@ resource "aws_lambda_permission" "allow_eventbridge" { } } -# below is step function 1 +######################################## +# S3 Extract Bucket Notification Setup # +######################################## + resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" @@ -66,6 +77,10 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { depends_on = [aws_lambda_permission.allow_s3_ingestion] } +########################################## +# S3 Transform Bucket Notification Setup # +########################################## + resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" -- cgit v1.2.3 From 09b8010a453c99164540981060177fdd2280df7e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:41:31 +0100 Subject: infra(tf): remove repetitive suffix resources in events.tf --- terraform/events.tf | 26 +++++++------------------- 1 file changed, 7 insertions(+), 19 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index c2efbbc..832a965 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -2,19 +2,7 @@ # Random String # ################# -resource "random_string" "eventbridge_suffix" { - length = 8 - special = false - upper = false -} - -resource "random_string" "s3_ingestion_suffix" { - length = 8 - special = false - upper = false -} - -resource "random_string" "s3_transform_suffix" { +resource "random_string" "suffix" { length = 8 special = false upper = false @@ -38,14 +26,14 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" + statement_id = "AllowExecutionFromEventBridge${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn lifecycle { - replace_triggered_by = [random_string.eventbridge_suffix] + replace_triggered_by = [random_string.suffix] } } @@ -54,14 +42,14 @@ resource "aws_lambda_permission" "allow_eventbridge" { ######################################## resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder lifecycle { - replace_triggered_by = [random_string.s3_ingestion_suffix] + replace_triggered_by = [random_string.suffix] } } @@ -82,14 +70,14 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ########################################## resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder principal = "s3.amazonaws.com" source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder lifecycle { - replace_triggered_by = [random_string.s3_transform_suffix] + replace_triggered_by = [random_string.suffix] } } -- cgit v1.2.3 From 367100c2d118847a775f4eba87a8c9033c872cb9 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:44:13 +0100 Subject: docs(tf): remove redundant comments --- terraform/events.tf | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index 832a965..0113f5f 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -21,7 +21,7 @@ resource "aws_cloudwatch_event_rule" "lambda_trigger" { resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { rule = aws_cloudwatch_event_rule.lambda_trigger.name target_id = "TargetFunctionV1" - arn = aws_lambda_function.extract_lambda.arn #replaced lambda name placeholder + arn = aws_lambda_function.extract_lambda.arn depends_on = [aws_lambda_permission.allow_eventbridge] } @@ -44,9 +44,9 @@ resource "aws_lambda_permission" "allow_eventbridge" { resource "aws_lambda_permission" "allow_s3_ingestion" { statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.extract_bucket.arn #replaced bucket name placeholder + source_arn = aws_s3_bucket.extract_bucket.arn lifecycle { replace_triggered_by = [random_string.suffix] @@ -55,11 +55,11 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { resource "aws_s3_bucket_notification" "extract_bucket_notification" { - bucket = aws_s3_bucket.extract_bucket.id #replaced bucket name placeholder + bucket = aws_s3_bucket.extract_bucket.id lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + lambda_function_arn = aws_lambda_function.transform_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_ingestion] @@ -72,9 +72,9 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { resource "aws_lambda_permission" "allow_s3_transform_bucket" { statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" action = "lambda:InvokeFunction" - function_name = aws_lambda_function.transform_lambda.function_name #replaced lambda name placeholder + function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" - source_arn = aws_s3_bucket.transform_bucket.arn #replaced bucket name placeholder + source_arn = aws_s3_bucket.transform_bucket.arn lifecycle { replace_triggered_by = [random_string.suffix] @@ -83,11 +83,11 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { resource "aws_s3_bucket_notification" "transform_bucket_notification" { - bucket = aws_s3_bucket.transform_bucket.id #replaced bucket name placeholder + bucket = aws_s3_bucket.transform_bucket.id lambda_function { events = ["s3:ObjectCreated:*"] - lambda_function_arn = aws_lambda_function.transform_lambda.arn #replaced lambda name placeholder + lambda_function_arn = aws_lambda_function.transform_lambda.arn } depends_on = [aws_lambda_permission.allow_s3_transform_bucket] -- cgit v1.2.3 From a9fb82f5c96e0ba98d6d3453ce900f2ca22157ec Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:48:00 +0100 Subject: infra(tf): remove unused declaration in vars.tf --- terraform/vars.tf | 5 ----- 1 file changed, 5 deletions(-) diff --git a/terraform/vars.tf b/terraform/vars.tf index 1adbcf7..b3e3e47 100644 --- a/terraform/vars.tf +++ b/terraform/vars.tf @@ -33,11 +33,6 @@ variable "project_name" { default = "tt" } -variable "aws_region" { - type = string - default = "eu-west-2" -} - variable "environment" { type = string default = "dev" -- cgit v1.2.3 From c091506dc8e01741f54f9a8d289515c8d5ffbecf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:49:00 +0100 Subject: infra(tf): add version constraint for random in main.tf --- terraform/main.tf | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/terraform/main.tf b/terraform/main.tf index 33c760c..ad7b335 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,6 +13,10 @@ terraform { source = "hashicorp/archive" version = "~>2.5.0" } + random_string = { + source = "hashicorp/random" + version = "~>3.6.2" + } } backend "s3" { bucket = "bentley-project-secrets" -- cgit v1.2.3 From ce2761b311523a118cdead885ba7fcf1f7a4cd68 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 21:52:15 +0100 Subject: fix(tf): correct random_string to random in main.tf --- terraform/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/main.tf b/terraform/main.tf index ad7b335..6577b70 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -13,7 +13,7 @@ terraform { source = "hashicorp/archive" version = "~>2.5.0" } - random_string = { + random = { source = "hashicorp/random" version = "~>3.6.2" } -- cgit v1.2.3 From 88e71818aaf1bf67e4d2807d22d8122b7bf184f1 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 22:20:21 +0100 Subject: refactor(log): implement logging ancestry - avoid using root logger --- src/extract_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 15fe785..6f841b4 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -8,7 +8,7 @@ from datetime import datetime import re -logger = logging.getLogger() +logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) # DB Exception class -- cgit v1.2.3 From 84b3dea3833ae65d53a1007567ee19c31bf34ee3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 22:28:31 +0100 Subject: refactor(retrieve_secrets): use aws recommended method for retrieving secrets --- src/extract_lambda.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 6f841b4..1df4c34 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -55,18 +55,21 @@ def lambda_handler(event, context): db.close() -def retrieve_secrets( - sm_client=boto3.client("secretsmanager"), secret_name="bentley-secrets" -): +def retrieve_secrets(): + secret_name = "bentley-secrets" + region_name = "eu-west-2" + + # Create a Secrets Manager client + session = boto3.session.Session() + client = session.client(service_name="secretsmanager", region_name=region_name) + try: - response = sm_client.get_secret_value(SecretId=secret_name) - if "SecretString" in response: - secret = json.loads(response["SecretString"]) - return secret + get_secret_value_response = client.get_secret_value(SecretId=secret_name) except ClientError as e: - logger.error(f"Could not retrieve secrets: {e}") raise e + return get_secret_value_response["SecretString"] + def connect_to_database() -> Connection: try: -- cgit v1.2.3 From 3d4d74aa69db85e3c840b3b73c028f4e9f83d1f7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 22:29:41 +0100 Subject: refactor(lambda_handler): remove unnecessary else statement --- src/extract_lambda.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 1df4c34..99117a4 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -39,14 +39,13 @@ def lambda_handler(event, context): "statusCode": 200, "body": json.dumps("No changes detected, no CSV files were uploaded."), } - else: - return { - "statusCode": 200, - "body": json.dumps( - f"""CSV files processed for {', '.join(any_changes['updated'])} and uploaded successfully.{ - 'The following tables were not updated: '+', '.join(any_changes['no change']) if any_changes['no change'] else ''}""" - ), - } + return { + "statusCode": 200, + "body": json.dumps( + f"""CSV files processed for {', '.join(any_changes['updated'])} and uploaded successfully.{ + 'The following tables were not updated: '+', '.join(any_changes['no change']) if any_changes['no change'] else ''}""" + ), + } except Exception as e: logger.error(f"Error: {e}") return {"statusCode": 500, "body": json.dumps("Internal server error.")} -- cgit v1.2.3 From 4699b3506307cb8556a7cc5f12fbe4df7a5c9a6b Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 22:31:58 +0100 Subject: refactor(retrieve_secrets): improve error handling when retrieving secrets --- src/extract_lambda.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 99117a4..63a80ce 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -66,6 +66,9 @@ def retrieve_secrets(): get_secret_value_response = client.get_secret_value(SecretId=secret_name) except ClientError as e: raise e + except KeyError: + logger.error(f"Secret {secret_name} does not contain a SecretString") + raise ValueError(f"Secret {secret_name} does not contain a SecretString") return get_secret_value_response["SecretString"] -- cgit v1.2.3 From 8353621c862e75d1573ff8338852aa7d54d5d2e8 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 22:36:37 +0100 Subject: refactor(retrieve_secrets): add logging for ClientError --- src/extract_lambda.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 63a80ce..485c021 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -65,6 +65,7 @@ def retrieve_secrets(): try: get_secret_value_response = client.get_secret_value(SecretId=secret_name) except ClientError as e: + logger.error(f"Failed to retrieve secret {secret_name}: {str(e)}") raise e except KeyError: logger.error(f"Secret {secret_name} does not contain a SecretString") -- cgit v1.2.3 From bcbadd508dbc1a53864e64cb1e2eccce53daa187 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 22:37:43 +0100 Subject: chore: reorganise imports in extract_lambda --- src/extract_lambda.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 485c021..8353481 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -1,12 +1,12 @@ -from pg8000.native import Connection, InterfaceError, identifier -import boto3 import csv -from botocore.exceptions import ClientError -import logging import json -from datetime import datetime +import logging import re +from datetime import datetime +import boto3 +from botocore.exceptions import ClientError +from pg8000.native import Connection, InterfaceError, identifier logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) -- cgit v1.2.3 From a8ce060732ed3064696f2d6c5459ffa176fd02f7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:02:56 +0100 Subject: fix(tf): lambda permissions should be created before destroyed --- terraform/events.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index 0113f5f..9885a86 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -33,7 +33,8 @@ resource "aws_lambda_permission" "allow_eventbridge" { source_arn = aws_cloudwatch_event_rule.lambda_trigger.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } @@ -49,7 +50,8 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { source_arn = aws_s3_bucket.extract_bucket.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } @@ -77,7 +79,8 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { source_arn = aws_s3_bucket.transform_bucket.arn lifecycle { - replace_triggered_by = [random_string.suffix] + create_before_destroy = true + replace_triggered_by = [random_string.suffix] } } -- cgit v1.2.3 From b8574d4c4bf262a8034d21b770fd4287022c2648 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:07:28 +0100 Subject: fix(tf): re-add separate random_string suffixes in events.tf --- terraform/events.tf | 26 +++++++++++++++++++------- 1 file changed, 19 insertions(+), 7 deletions(-) diff --git a/terraform/events.tf b/terraform/events.tf index 9885a86..53ae10a 100644 --- a/terraform/events.tf +++ b/terraform/events.tf @@ -2,7 +2,19 @@ # Random String # ################# -resource "random_string" "suffix" { +resource "random_string" "eventbridge_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_ingestion_suffix" { + length = 8 + special = false + upper = false +} + +resource "random_string" "s3_transform_suffix" { length = 8 special = false upper = false @@ -26,7 +38,7 @@ resource "aws_cloudwatch_event_target" "extract_lambda_cw_event" { } resource "aws_lambda_permission" "allow_eventbridge" { - statement_id = "AllowExecutionFromEventBridge${random_string.suffix.result}" + statement_id = "AllowExecutionFromEventBridge${random_string.eventbridge_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.extract_lambda.function_name principal = "events.amazonaws.com" @@ -34,7 +46,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.eventbridge_suffix] } } @@ -43,7 +55,7 @@ resource "aws_lambda_permission" "allow_eventbridge" { ######################################## resource "aws_lambda_permission" "allow_s3_ingestion" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_ingestion_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" @@ -51,7 +63,7 @@ resource "aws_lambda_permission" "allow_s3_ingestion" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.s3_ingestion_suffix] } } @@ -72,7 +84,7 @@ resource "aws_s3_bucket_notification" "extract_bucket_notification" { ########################################## resource "aws_lambda_permission" "allow_s3_transform_bucket" { - statement_id = "AllowS3InvokeLambdaTransform${random_string.suffix.result}" + statement_id = "AllowS3InvokeLambdaTransform${random_string.s3_transform_suffix.result}" action = "lambda:InvokeFunction" function_name = aws_lambda_function.transform_lambda.function_name principal = "s3.amazonaws.com" @@ -80,7 +92,7 @@ resource "aws_lambda_permission" "allow_s3_transform_bucket" { lifecycle { create_before_destroy = true - replace_triggered_by = [random_string.suffix] + replace_triggered_by = [random_string.s3_transform_suffix] } } -- cgit v1.2.3 From caed81dc699b9b4105da2b8924310f1a370217c7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:13:39 +0100 Subject: refactor: add timestamp function in extract_lambda.py --- src/extract_lambda.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 8353481..ad3c970 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -129,6 +129,16 @@ def list_existing_s3_files(bucket_name=extract_bucket(), client=boto3.client("s3 return existing_files +def get_latest_timestamp(existing_files): + all_datetimes = [] + for file_name in existing_files.keys(): + match = re.search(r"\/(.+/).+_(.+)\.csv", file_name) + if match: + datetime_str = "".join(match.group(1, 2)) + all_datetimes.append(datetime.strptime(datetime_str, "%Y/%m/%d/%H:%M:%S")) + return max(all_datetimes) if all_datetimes else datetime.min + + def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): """Creates a list of the tables from a database query and then selects everything from each table in individual queries @@ -137,22 +147,17 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): to files, or new tables/files it uploads them to the s3 bucket """ load_status = {"updated": [], "no change": []} - # Retrieving the latest file timestamp from S3 extract bucket - all_datetimes = [] - for file_names in existing_files.keys(): - datetime_str_on_s3 = "".join( - re.search(r"\/(.+/).+_(.+)\.csv", file_names).group(1, 2) - ) - all_datetimes.append(datetime.strptime(datetime_str_on_s3, "%Y/%m/%d/%H:%M:%S")) - latest_timestamp = max(all_datetimes) + latest_timestamp = get_latest_timestamp(existing_files) - # Iterating through tables on the database and retrieving only latest changes vs previous file load tables = db.run( """ - SELECT table_name - FROM information_schema.tables - WHERE table_schema='public' AND table_type='BASE TABLE';""" + SELECT table_name + FROM information_schema.tables + WHERE table_schema='public' + AND table_type='BASE TABLE'; + """ ) + for table in tables: print(tables) table_name = table[0] -- cgit v1.2.3 From 610d23e7ed0f39e5ecb0dd25c3a1e3cba20d662e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:26:58 +0100 Subject: refactor: remove print statements in process_and_upload_tables --- src/extract_lambda.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index ad3c970..7c6c3d1 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -159,13 +159,11 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): ) for table in tables: - print(tables) table_name = table[0] rows = db.run( f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) - print("rows", rows) # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" -- cgit v1.2.3 From 5be3b130170c82360ff9715f5c09b9e815fc16f4 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:32:25 +0100 Subject: feat: use buffers for s3 upload instead of csv files --- src/extract_lambda.py | 50 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 7c6c3d1..f38e24a 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -3,6 +3,7 @@ import json import logging import re from datetime import datetime +from io import StringIO import boto3 from botocore.exceptions import ClientError @@ -139,6 +140,26 @@ def get_latest_timestamp(existing_files): return max(all_datetimes) if all_datetimes else datetime.min +def stream_to_s3(table_name, rows, column_names, s3_client, bucket_name, s3_key): + csv_buffer = StringIO() + csv_writer = csv.writer(csv_buffer) + + csv_writer.writerow(column_names) + + for row in rows: + csv_writer.writerow(row) + + if csv_buffer.tell() > 5 * 1024 * 1024: + csv_buffer.seek(0) + s3_client.upload_fileobj(csv_buffer, bucket_name, s3_key) + csv_buffer.truncate(0) + csv_buffer.seek(0) + + if csv_buffer.tell() > 0: + csv_buffer.seek(0) + s3_client.upload_fileobj(csv_buffer, bucket_name, s3_key) + + def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): """Creates a list of the tables from a database query and then selects everything from each table in individual queries @@ -164,29 +185,24 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) - # Creating a temporary file path and writing the column name to it followed by each row of data if rows: - csv_file_path = f"/tmp/{table_name}.csv" - with open(csv_file_path, "w", newline="") as file: - writer = csv.writer(file) - # column_names = [desc["name"] for desc in db.columns(f"SELECT * FROM {table_name};")] - column_names = [ - col_name[0] - for col_name in db.run( - """SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = :table ;""", - table=table_name, - ) - ] - writer.writerow(column_names) - writer.writerows(rows) + column_names = [ + col_name[0] + for col_name in db.run( + """SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = :table ;""", + table=table_name, + ) + ] + s3_key = datetime.strftime( datetime.today(), f"{table_name}/%Y/%m/%d/{table_name}_%H:%M:%S.csv" ) - # Writing the new file to S3 extract bucket: try: - client.upload_file(csv_file_path, extract_bucket(), s3_key) + stream_to_s3( + table_name, rows, column_names, client, extract_bucket(), s3_key + ) load_status["updated"].append(table_name) logger.info(f"Uploaded {s3_key} to S3.") except ClientError as e: -- cgit v1.2.3 From 3e80acb28eeeb0eaff97c2363124a8c6e95bcb13 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Mon, 19 Aug 2024 23:44:52 +0100 Subject: refactor: optimise s3 streaming & file naming --- src/extract_lambda.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index f38e24a..8575b08 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -149,15 +149,9 @@ def stream_to_s3(table_name, rows, column_names, s3_client, bucket_name, s3_key) for row in rows: csv_writer.writerow(row) - if csv_buffer.tell() > 5 * 1024 * 1024: - csv_buffer.seek(0) - s3_client.upload_fileobj(csv_buffer, bucket_name, s3_key) - csv_buffer.truncate(0) - csv_buffer.seek(0) + csv_buffer.seek(0) - if csv_buffer.tell() > 0: - csv_buffer.seek(0) - s3_client.upload_fileobj(csv_buffer, bucket_name, s3_key) + s3_client.upload_fileobj(csv_buffer, bucket_name, s3_key) def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): @@ -190,13 +184,14 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): col_name[0] for col_name in db.run( """SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = :table ;""", + WHERE table_name = :table ;""", table=table_name, ) ] - s3_key = datetime.strftime( - datetime.today(), f"{table_name}/%Y/%m/%d/{table_name}_%H:%M:%S.csv" + s3_key = ( + f"{table_name}/{datetime.now().strftime('%Y/%m/%d')}/" + f"{table_name}_{datetime.now().strftime('%H:%M:%S')}.csv" ) try: -- cgit v1.2.3 From 7c77382fdaf236247a35f35810d66a86923156dd Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:10:40 +0100 Subject: fix(): update expected response message --- tests/test_extract_lambda.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 3931cfc..9362a6c 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -72,7 +72,11 @@ class TestLambdaHandler: ] with patch("src.extract_lambda.connect_to_database", return_value=mock_db): mock_process_and_upload_tables = mocker.patch( - "src.extract_lambda.process_and_upload_tables", return_value=mock_db + "src.extract_lambda.process_and_upload_tables", + return_value={ + "updated": ["Fruits"], + "no change": ["Vegetable", "Berry"], + }, ) mock_list_existing_s3_files = mocker.patch( "src.extract_lambda.list_existing_s3_files", return_value={} @@ -81,9 +85,9 @@ class TestLambdaHandler: context = {} response = lambda_handler(event, context) assert response["statusCode"] == 200 - assert ( - json.loads(response["body"]) - == "CSV files processed and uploaded successfully." + assert json.loads(response["body"]) == ( + "CSV files processed for Fruits and uploaded successfully." + "The following tables were not updated: Vegetable, Berry" ) mock_list_existing_s3_files.assert_called_once() mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) -- cgit v1.2.3 From bf7aab5cdbf2007824f0fb2bff2de5a4fa8196ba Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:11:37 +0100 Subject: chore(tests): rename lambda_handler class test functions --- tests/test_extract_lambda.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 9362a6c..b9e3a4b 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -58,7 +58,7 @@ def s3_mock_bucket(s3_client): class TestLambdaHandler: - def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): + def test_files_processed_and_uploaded_successfully(self, mocker): mock_db = MagicMock() mock_db.run.side_effect = [ [["Fruits"]], @@ -93,7 +93,7 @@ class TestLambdaHandler: mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) mock_db.close.assert_called_once() - def test_lambda_handler_no_changes_detected_no_files_uploaded(self, mocker): + def test_no_changes_detected_no_files_uploaded(self, mocker): mock_db = MagicMock() mock_db.run.side_effect = [ [["Fruits"]], @@ -125,7 +125,7 @@ class TestLambdaHandler: mock_process_and_upload_tables.assert_called_once_with(mock_db, {}) mock_db.close.assert_called_once() - def test_lambda_handler_exception_error(self, mocker): + def test_exception_error(self, mocker): with patch( "src.extract_lambda.connect_to_database", side_effect=Exception("Database connection error"), -- cgit v1.2.3 From 32175a3b4387a8218b4e21561173445fd5b5df1d Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:13:45 +0100 Subject: fix(): update expected response message for second test --- tests/test_extract_lambda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index b9e3a4b..3d15927 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -108,7 +108,8 @@ class TestLambdaHandler: with patch("src.extract_lambda.connect_to_database", return_value=mock_db): mock_process_and_upload_tables = mocker.patch( - "src.extract_lambda.process_and_upload_tables", return_value=False + "src.extract_lambda.process_and_upload_tables", + return_value={"updated": [], "no change": ["Fruits"]}, ) mock_list_existing_s3_files = mocker.patch( "src.extract_lambda.list_existing_s3_files", return_value={} -- cgit v1.2.3 From 640b0685cd795c03b571b3ca26fc9030b86c4f99 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:18:16 +0100 Subject: fix(extract_lambda): fix UnboundLocalError when db is called before it is assigned a value --- src/extract_lambda.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 8575b08..7efaac0 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -29,6 +29,7 @@ def lambda_handler(event, context): and converts all tables to CSV and if any of those tables do not exist in, or are different to the ones in s3, it uploads them it uses 3 helper functions to achieve these 3 functionalities """ + db = None try: db = connect_to_database() existing_files = list_existing_s3_files() -- cgit v1.2.3 From 746f4206b2f30126c3c09ac11a2d49be3259fe6f Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:42:54 +0100 Subject: infra(tf): add secrets manager permissions I feel like what I've done is bad but we'll find out soon. --- terraform/iam.tf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/terraform/iam.tf b/terraform/iam.tf index 7585ff8..a36cfdf 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -169,3 +169,30 @@ resource "aws_iam_role_policy_attachment" "cloudwatch_events_attachment" { role = aws_iam_role.multi_service_role.name policy_arn = aws_iam_policy.cloudwatch_events_policy.arn } + +######################### +# SECRETS MANAGER SETUP # +######################### + +# Policy Doc +data "aws_iam_policy_document" "secrets_manager_policy_doc" { + statement { + effect = "Allow" + actions = [ + "secretsmanager:GetSecretValue" + ] + resources = [] + } +} + +# SM Policy Resource +resource "aws_iam_policy" "secrets_manager_policy" { + name = "secrets_manager_policy" + policy = data.aws_iam_policy_document.secrets_manager_policy_doc.json +} + +# Attach SM Policy to Role +resource "aws_iam_role_policy_attachment" "secrets_manager_attachment" { + role = aws_iam_role.multi_service_role.name + policy_arn = aws_iam_policy.secrets_manager_policy.arn +} -- cgit v1.2.3 From 2045888e1ae497444c58347096547f0475bba7a1 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 00:51:11 +0100 Subject: infra(tf): add resource access for secrets-manager policy doc --- terraform/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index a36cfdf..a8054ca 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -181,7 +181,7 @@ data "aws_iam_policy_document" "secrets_manager_policy_doc" { actions = [ "secretsmanager:GetSecretValue" ] - resources = [] + resources = ["arn:aws:secretsmanager:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:secret:bentley-secrets-Na0yc8"] } } -- cgit v1.2.3 From d34ad9649648c178ac24b58832982f5c37aca48e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 01:03:34 +0100 Subject: fix(extract_lambda): parse secrets string as json dict to access secret values --- src/extract_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 7efaac0..9de6214 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -78,7 +78,7 @@ def retrieve_secrets(): def connect_to_database() -> Connection: try: - secrets = retrieve_secrets() + secrets = json.loads(retrieve_secrets()) host = secrets["host"] port = secrets["port"] user = secrets["user"] -- cgit v1.2.3 From ae57535d9f201d6fd749d4286551884d3c86fd60 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 10:26:48 +0100 Subject: infra(tf): add missing ListObjectsV2 permission --- terraform/iam.tf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index a8054ca..3ac8c45 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -40,7 +40,8 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectRetention", "s3:PutObjectTagging", "s3:PutObjectAcl", - "s3:ListObjects" + "s3:ListObjects", + "s3:ListObjectsV2" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", -- cgit v1.2.3 From e25bee6c1c9db8edaf3197f0dc48fa3c63e61744 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 11:01:55 +0100 Subject: feat: revert s3 streaming to previous implementation for uploading --- src/extract_lambda.py | 56 +++++++++++++++++++++++---------------------------- 1 file changed, 25 insertions(+), 31 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 7efaac0..4921034 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -49,7 +49,7 @@ def lambda_handler(event, context): ), } except Exception as e: - logger.error(f"Error: {e}") + logger.error(f"Error: {e}", exc_info=True) return {"statusCode": 500, "body": json.dumps("Internal server error.")} finally: if db: @@ -78,7 +78,7 @@ def retrieve_secrets(): def connect_to_database() -> Connection: try: - secrets = retrieve_secrets() + secrets = json.loads(retrieve_secrets()) host = secrets["host"] port = secrets["port"] user = secrets["user"] @@ -141,20 +141,6 @@ def get_latest_timestamp(existing_files): return max(all_datetimes) if all_datetimes else datetime.min -def stream_to_s3(table_name, rows, column_names, s3_client, bucket_name, s3_key): - csv_buffer = StringIO() - csv_writer = csv.writer(csv_buffer) - - csv_writer.writerow(column_names) - - for row in rows: - csv_writer.writerow(row) - - csv_buffer.seek(0) - - s3_client.upload_fileobj(csv_buffer, bucket_name, s3_key) - - def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): """Creates a list of the tables from a database query and then selects everything from each table in individual queries @@ -180,25 +166,29 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) + # Creating a temporary file path and writing the column name to it followed by each row of data if rows: - column_names = [ - col_name[0] - for col_name in db.run( - """SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS - WHERE table_name = :table ;""", - table=table_name, - ) - ] - - s3_key = ( - f"{table_name}/{datetime.now().strftime('%Y/%m/%d')}/" - f"{table_name}_{datetime.now().strftime('%H:%M:%S')}.csv" + csv_file_path = f"/tmp/{table_name}.csv" + with open(csv_file_path, "w", newline="") as file: + writer = csv.writer(file) + # column_names = [desc["name"] for desc in db.columns(f"SELECT * FROM {table_name};")] + column_names = [ + col_name[0] + for col_name in db.run( + """SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS + WHERE table_name = :table ;""", + table=table_name, + ) + ] + writer.writerow(column_names) + writer.writerows(rows) + s3_key = datetime.strftime( + datetime.today(), f"{table_name}/%Y/%m/%d/{table_name}_%H:%M:%S.csv" ) + # Writing the new file to S3 extract bucket: try: - stream_to_s3( - table_name, rows, column_names, client, extract_bucket(), s3_key - ) + client.upload_file(csv_file_path, extract_bucket(), s3_key) load_status["updated"].append(table_name) logger.info(f"Uploaded {s3_key} to S3.") except ClientError as e: @@ -207,3 +197,7 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): load_status["no change"].append(table_name) logger.info(f"No new data") return load_status + + +if __name__ == "__main__": + lambda_handler(None, None) -- cgit v1.2.3 From 5211751b69a894874945e3a916c33781a327ab10 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 11:26:26 +0100 Subject: feat: conditional logic for if bucket is empty --- src/extract_lambda.py | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 4921034..6216446 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -124,6 +124,7 @@ def list_existing_s3_files(bucket_name=extract_bucket(), client=boto3.client("s3 logger.error(f"Error retrieving S3 object {s3_key}: {e}") else: logger.error("The bucket is empty") + return None except ClientError as e: logger.error(f"Error listing S3 objects: {e}") @@ -132,13 +133,18 @@ def list_existing_s3_files(bucket_name=extract_bucket(), client=boto3.client("s3 def get_latest_timestamp(existing_files): - all_datetimes = [] - for file_name in existing_files.keys(): - match = re.search(r"\/(.+/).+_(.+)\.csv", file_name) - if match: - datetime_str = "".join(match.group(1, 2)) - all_datetimes.append(datetime.strptime(datetime_str, "%Y/%m/%d/%H:%M:%S")) - return max(all_datetimes) if all_datetimes else datetime.min + if existing_files: + all_datetimes = [] + for file_name in existing_files.keys(): + match = re.search(r"\/(.+/).+_(.+)\.csv", file_name) + if match: + datetime_str = "".join(match.group(1, 2)) + all_datetimes.append( + datetime.strptime(datetime_str, "%Y/%m/%d/%H:%M:%S") + ) + return max(all_datetimes) if all_datetimes else datetime.min + + return existing_files def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): @@ -163,8 +169,16 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): for table in tables: table_name = table[0] rows = db.run( - f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", - latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, + f""" + SELECT * FROM {identifier(table_name)} + WHERE last_updated >= :latest; + """, + latest={ + datetime.strftime( + latest_timestamp if latest_timestamp else datetime(1990, 1, 1), + "%Y-%m-%d %H:%M:%S", + ) + }, ) # Creating a temporary file path and writing the column name to it followed by each row of data if rows: -- cgit v1.2.3 From dc3a7e74ddf549dad05745c64201aaf0d3402213 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 11:31:25 +0100 Subject: feat: add advanced logging --- src/extract_lambda.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 6216446..9daf662 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -10,8 +10,12 @@ from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError, identifier logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - +logging.basicConfig( + format="{asctime} - {levelname} - {message}", + style="{", + datefmt="%Y-%m-%d %H:%M", + level=logging.INFO, +) # DB Exception class @@ -168,11 +172,13 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): for table in tables: table_name = table[0] - rows = db.run( - f""" + base_query = f""" SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest; - """, + """ + logger.info(f"Processing table: {table_name}") + rows = db.run( + base_query, latest={ datetime.strftime( latest_timestamp if latest_timestamp else datetime(1990, 1, 1), -- cgit v1.2.3 From 35397e8bad42a8c507d1fb13007c6da2f947e851 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 11:44:30 +0100 Subject: feat: add additional logging and exclude unnecessary table --- src/extract_lambda.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 9daf662..fe22192 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -165,7 +165,7 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): """ SELECT table_name FROM information_schema.tables - WHERE table_schema='public' + WHERE table_schema='public' AND table_name != '_prisma_migrations' AND table_type='BASE TABLE'; """ ) @@ -176,16 +176,18 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest; """ - logger.info(f"Processing table: {table_name}") - rows = db.run( - base_query, - latest={ + latest = ( + { datetime.strftime( latest_timestamp if latest_timestamp else datetime(1990, 1, 1), "%Y-%m-%d %H:%M:%S", ) }, ) + logger.info(f"Processing table: {table_name}") + logger.info(f"Latest timestamp: {latest[0]}") + rows = db.run(base_query, latest=latest) + logger.info(f"Rows: {rows}") # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" -- cgit v1.2.3 From be911e22a964bdf7d5a4421cde7d7c6df447ed5c Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 11:49:59 +0100 Subject: refactor: change rows output to debug logger output --- src/extract_lambda.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index fe22192..e9f438b 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -16,7 +16,6 @@ logging.basicConfig( datefmt="%Y-%m-%d %H:%M", level=logging.INFO, ) -# DB Exception class class DBConnectionException(Exception): @@ -187,7 +186,7 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): logger.info(f"Processing table: {table_name}") logger.info(f"Latest timestamp: {latest[0]}") rows = db.run(base_query, latest=latest) - logger.info(f"Rows: {rows}") + logger.debug(f"Rows: {rows}") # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" -- cgit v1.2.3 From e788a90307831d968fcac51dc5d70d356a5a5f63 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Tue, 20 Aug 2024 12:05:56 +0100 Subject: Complete: completed testing for extract bucket - all passing --- tests/test_extract_lambda.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 3d15927..3cd2405 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -15,6 +15,7 @@ from src.extract_lambda import ( lambda_handler, process_and_upload_tables, retrieve_secrets, + extract_bucket ) @@ -146,6 +147,31 @@ class TestLambdaHandler: mock_process_and_upload_tables.assert_not_called() +class TestExtractBucket: + def test_extract_bucket_returns_bucket_name(self, s3_client, s3_mock_bucket): + result = extract_bucket(s3_client) + assert result == "extract_bucket" + + def test_bucket_returns_first_bucket(self, s3_client): + bucket1 = s3_client.create_bucket( + Bucket='bucket1', + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + result = extract_bucket(s3_client) + assert result == "extract_bucket" + + def test_returns_index_error_if_no_buckets(self, s3_client): + s3_client.delete_bucket( + Bucket="extract_bucket" + ) + s3_client.delete_bucket( + Bucket="bucket1" + ) + + with pytest.raises(IndexError, match="list index out of range"): + extract_bucket(s3_client) + + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): logger = logging.getLogger() @@ -165,7 +191,6 @@ class TestListExistingS3Files: class TestConnectToDatabase: - # had mock_config in param def test_connect_to_database(mock_conn, mock_config): with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() @@ -187,7 +212,7 @@ class TestConnectToDatabase: class TestProcessAndUploadTables: - def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): + def test_error_process_and_upload_tables(self, mock_conn, s3_client, caplog): caplog.set_level(logging.INFO) # Mock return values for database queries -- cgit v1.2.3 From 346aadfbf2208a0660ffc09959a91fc2f7b48c79 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 12:07:17 +0100 Subject: infra(tf): force-destroy buckets --- terraform/s3.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/s3.tf b/terraform/s3.tf index d17a4fe..14e8835 100644 --- a/terraform/s3.tf +++ b/terraform/s3.tf @@ -4,7 +4,7 @@ resource "aws_s3_bucket" "extract_bucket" { bucket_prefix = "${var.s3_extract_bucket_name}-" - + force_destroy = true tags = { Name = "Ingestion Bucket" } @@ -23,6 +23,7 @@ resource "aws_s3_bucket_versioning" "extract_bucket_versioning" { resource "aws_s3_bucket" "transform_bucket" { bucket_prefix = "${var.s3_transform_bucket_name}-" + force_destroy = true tags = { Name = "Transform Bucket" } @@ -42,6 +43,7 @@ resource "aws_s3_bucket_versioning" "transform_bucket_versioning" { resource "aws_s3_bucket" "lambda_code_bucket" { bucket_prefix = "${var.s3_code_bucket_name}-" + force_destroy = true tags = { Name = "Lambda Bucket" } -- cgit v1.2.3 From 0870dc49ddbd6024dddb289909487a15c26a3383 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Tue, 20 Aug 2024 11:08:24 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in e788a90 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/72 --- tests/test_extract_lambda.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 3cd2405..548ce67 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -15,7 +15,7 @@ from src.extract_lambda import ( lambda_handler, process_and_upload_tables, retrieve_secrets, - extract_bucket + extract_bucket, ) @@ -154,24 +154,20 @@ class TestExtractBucket: def test_bucket_returns_first_bucket(self, s3_client): bucket1 = s3_client.create_bucket( - Bucket='bucket1', + Bucket="bucket1", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, ) result = extract_bucket(s3_client) assert result == "extract_bucket" def test_returns_index_error_if_no_buckets(self, s3_client): - s3_client.delete_bucket( - Bucket="extract_bucket" - ) - s3_client.delete_bucket( - Bucket="bucket1" - ) + s3_client.delete_bucket(Bucket="extract_bucket") + s3_client.delete_bucket(Bucket="bucket1") with pytest.raises(IndexError, match="list index out of range"): extract_bucket(s3_client) - + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): logger = logging.getLogger() -- cgit v1.2.3 From 2a914add8391f345ee1096b9deb729c05d3e06c3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:15:02 +0100 Subject: feat: add more logging for debugging --- src/extract_lambda.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index e9f438b..24f0981 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -10,13 +10,16 @@ from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError, identifier logger = logging.getLogger(__name__) + logging.basicConfig( format="{asctime} - {levelname} - {message}", style="{", datefmt="%Y-%m-%d %H:%M", - level=logging.INFO, + level=logging.DEBUG, ) +logging.getLogger("botocore").setLevel(logging.WARNING) + class DBConnectionException(Exception): """Wraps pg8000.native Error or DatabaseError.""" @@ -110,7 +113,7 @@ def list_existing_s3_files(bucket_name=extract_bucket(), client=boto3.client("s3 results of listing the contents of the s3 bucket, then returns the populated dictionary """ - + logging.info("Listing existing S3 files") existing_files = {} try: -- cgit v1.2.3 From 5493cdc71da4730c4e388d9718f278bc2f14badf Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:15:28 +0100 Subject: infra(tf): add ListBucket and GetObject permissions --- terraform/iam.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/terraform/iam.tf b/terraform/iam.tf index 3ac8c45..3d62b69 100644 --- a/terraform/iam.tf +++ b/terraform/iam.tf @@ -41,7 +41,8 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { "s3:PutObjectTagging", "s3:PutObjectAcl", "s3:ListObjects", - "s3:ListObjectsV2" + "s3:ListObjectsV2", + "s3:GetObject" ] resources = [ "${aws_s3_bucket.extract_bucket.arn}/*", @@ -53,8 +54,10 @@ data "aws_iam_policy_document" "s3_data_policy_doc" { statement { effect = "Allow" actions = [ - "s3:ListBuckets", - "s3:ListAllMyBuckets" + "s3:ListBucket", + "s3:ListAllMyBuckets", + "s3:ListObjectsV2", + "s3:ListObjects" ] resources = [ "arn:aws:s3:::*", -- cgit v1.2.3 From 53686e2e466bc38f65da15ec617b43e43a1af9f7 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:25:13 +0100 Subject: chore: tidy-up repository & remove unused files --- Makefile | 80 -------------------------------------------------- src/secrets_manager.py | 49 ------------------------------- test.py | 0 3 files changed, 129 deletions(-) delete mode 100644 Makefile delete mode 100644 src/secrets_manager.py delete mode 100644 test.py diff --git a/Makefile b/Makefile deleted file mode 100644 index 077cd98..0000000 --- a/Makefile +++ /dev/null @@ -1,80 +0,0 @@ -############################################## -# # -# MAKEFILE TO BUILD THE PROJECT # -# # -############################################## - -PROJECT_NAME = de-project-bentley -REGION = eu-west-2 -PYTHON_INTERPRETER = python -WD=$(shell pwd) -PYTHONPATH=${WD} -SHELL := /bin/bash -PROFILE = default -PIP:=pip - -## PYTHON INTERPRETER ENVIRONMENT -create-environment: - @echo ">>> About to create environment: $(PROJECT_NAME)..." - @echo ">>> check python3 version" - ( \ - $(PYTHON_INTERPRETER) --version; \ - ) - @echo ">>> Setting up VirtualEnv." - ( \ - $(PIP) install -q virtualenv virtualenvwrapper; \ - virtualenv venv --python=$(PYTHON_INTERPRETER); \ - ) - -ACTIVATE_ENV := source venv/bin/activate - -# Execute python related functionalities from within the project's environment -define execute_in_env - $(ACTIVATE_ENV) && $1 -endef - -## Build the environment requirements -requirements: create-environment - $(call execute_in_env, $(PIP) install -r ./requirements.txt) - -# Set Up -## Install bandit -bandit: - $(call execute_in_env, $(PIP) install bandit) - -## Install safety -safety: - $(call execute_in_env, $(PIP) install safety) - -## Install black -black: - $(call execute_in_env, $(PIP) install black) - -## Install coverage -coverage: - $(call execute_in_env, $(PIP) install coverage) - -## Set up dev requirements (bandit, safety, black) -dev-setup: bandit safety black coverage - -# Build / Run - -## Run the security test (bandit + safety) -security-test: - $(call execute_in_env, safety check -r ./requirements.txt) - $(call execute_in_env, bandit -lll */*.py *c/*/*.py) - -## Run the black code check -run-black: - $(call execute_in_env, black ./src/*/*.py ./test/*/*.py) - -## Run the unit tests -unit-test: - $(call execute_in_env, PYTHONPATH=${PYTHONPATH} pytest -v) - -## Run the coverage check -check-coverage: - $(call execute_in_env, PYTHONPATH=${PYTHONPATH} pytest --cov=src test/) - -## Run all checks -run-checks: security-test run-black unit-test check-coverage diff --git a/src/secrets_manager.py b/src/secrets_manager.py deleted file mode 100644 index 3484688..0000000 --- a/src/secrets_manager.py +++ /dev/null @@ -1,49 +0,0 @@ -import boto3 -from botocore.exceptions import ClientError -import json - - -def sm_client(): - sm_client = boto3.client("secretsmanager") - yield sm_client - - -def create_secret( - sm_client, secret_name, cohort_id, user, password, host, database, port -): - secret = { - "cohort_id": cohort_id, - "user": user, - "password": password, - "host": host, - "database": database, - "port": port, - } - - response = sm_client.create_secret( - Name=secret_name, SecretString=json.dumps(secret) - ) - - print(response) - return response - - -def list_secret(sm_client): - response = sm_client.list_secrets() - secret_dict = response["SecretList"] - secret_names = [] - for items in secret_dict: - secret_names.append(items["Name"]) - print(f"{len(secret_names)} secret(s) available") - for name in secret_names: - print(name) - return secret_names - - -def retrieve_secrets(sm_client): - response = sm_client.get_secrets() - - -# retrieve secret -# so lambda can access totesy db -# so lambda connect to the db and then retrieve the data diff --git a/test.py b/test.py deleted file mode 100644 index e69de29..0000000 -- cgit v1.2.3 From d1f3fd505c4deea8cc811a851cda15d00419ade3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 15:50:23 +0100 Subject: hotfix: increase lambda timeout --- terraform/lambda.tf | 3 +++ 1 file changed, 3 insertions(+) diff --git a/terraform/lambda.tf b/terraform/lambda.tf index f8e7515..d33a6c9 100644 --- a/terraform/lambda.tf +++ b/terraform/lambda.tf @@ -70,6 +70,7 @@ resource "aws_lambda_function" "extract_lambda" { handler = "extract_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.extract_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true @@ -103,6 +104,7 @@ resource "aws_lambda_function" "transform_lambda" { handler = "transform_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.transform_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true @@ -136,6 +138,7 @@ resource "aws_lambda_function" "load_lambda" { handler = "load_lambda.lambda_handler" runtime = "python3.11" source_code_hash = data.archive_file.load_lambda_zip.output_base64sha256 + timeout = 180 lifecycle { create_before_destroy = true -- cgit v1.2.3 From e8995e003c2cf616daf7a23fd525d9e0cc886bb0 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 20 Aug 2024 22:47:39 +0100 Subject: ci: update deploy.yml --- .github/workflows/deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 5672048..09b8490 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -11,6 +11,7 @@ on: jobs: deploy-terraform: + if: github.ref == 'refs/heads/main' name: Deploy Terraform runs-on: ubuntu-latest #needs: run-checks (must ref on-commit.yml file) -- cgit v1.2.3 From ad19a8bac6ad0411e3c2c2530b0ca6ee1541d072 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 20 Aug 2024 22:51:05 +0100 Subject: chore: rm workflow file from development --- .github/workflows/deploy.yml | 43 ------------------------------------------- 1 file changed, 43 deletions(-) delete mode 100644 .github/workflows/deploy.yml diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml deleted file mode 100644 index 09b8490..0000000 --- a/.github/workflows/deploy.yml +++ /dev/null @@ -1,43 +0,0 @@ -name: deploy-terraform - -on: - pull_request: - branches: - - main - push: - branches: - - main - - -jobs: - deploy-terraform: - if: github.ref == 'refs/heads/main' - name: Deploy Terraform - runs-on: ubuntu-latest - #needs: run-checks (must ref on-commit.yml file) - environment: production - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Install Terraform - uses: hashicorp/setup-terraform@v3 - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ secrets.AWS_REGION }} - - - name: Terraform Init - working-directory: terraform - run: terraform init - - - name: Terraform Plan - working-directory: terraform - run: terraform plan - - - name: Terraform Apply - working-directory: terraform - run: terraform apply --auto-approve -- cgit v1.2.3 From f259504a87e24b0dae6f2e06acafdf881d4ec96e Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 20 Aug 2024 23:01:39 +0100 Subject: test: test trigger for ci/cd --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cbb446c..7d7e499 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ The solution showcases our skills in: - Amazon Web Services (AWS) - Agile methodologies -# Main Objective +# Main Objectives Our goal is to create a reliable ETL (Extract, Transform, Load) pipeline that can: @@ -48,4 +48,4 @@ others. TBA # Contributors -TBA \ No newline at end of file +TBA -- cgit v1.2.3 From 9511ac7958efcadad6cd1323027674988042bee9 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 20 Aug 2024 23:09:46 +0100 Subject: ci: create dev-tests.yml --- .github/workflows/dev-tests.yml | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 .github/workflows/dev-tests.yml diff --git a/.github/workflows/dev-tests.yml b/.github/workflows/dev-tests.yml new file mode 100644 index 0000000..9f71515 --- /dev/null +++ b/.github/workflows/dev-tests.yml @@ -0,0 +1,49 @@ +name: dev-tests + +on: + pull_request: + branches: + - development + push: + branches: + - development + +jobs: + validate-and-test: + name: Validate Terraform and Run Tests + runs-on: ubuntu-latest + environment: testing + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Install Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Terraform Init + working-directory: terraform + run: terraform init -backend=false + + - name: Terraform Validate + working-directory: terraform + run: terraform validate + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.11' + + - name: Install Python dependencies + run: | + python -m pip install --upgrade pip + pip install pytest pytest-testdox + pip install -r requirements.txt + + - name: Run pytest + run: pytest tests/ -vvrP --testdox + continue-on-error: true + id: pytest + + - name: Check on failures + if: steps.pytest.outcome == 'failure' + run: exit 1 -- cgit v1.2.3 From 0cf8f2c238c2f86ee6c97ed7b95e78c67d1782b5 Mon Sep 17 00:00:00 2001 From: Alex Date: Tue, 20 Aug 2024 23:13:34 +0100 Subject: ci: remove environment for dev-tests.yml --- .github/workflows/dev-tests.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/dev-tests.yml b/.github/workflows/dev-tests.yml index 9f71515..d66f1c6 100644 --- a/.github/workflows/dev-tests.yml +++ b/.github/workflows/dev-tests.yml @@ -12,7 +12,6 @@ jobs: validate-and-test: name: Validate Terraform and Run Tests runs-on: ubuntu-latest - environment: testing steps: - name: Checkout Repo uses: actions/checkout@v4 -- cgit v1.2.3