From dd536c3209fc37423af4219a941c006bdb6b3c4f Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Mon, 19 Aug 2024 10:32:57 +0100 Subject: deleted the test folder --- test/test_secrets_manager.py | 34 ---------------------------------- 1 file changed, 34 deletions(-) delete mode 100644 test/test_secrets_manager.py diff --git a/test/test_secrets_manager.py b/test/test_secrets_manager.py deleted file mode 100644 index 86533bc..0000000 --- a/test/test_secrets_manager.py +++ /dev/null @@ -1,34 +0,0 @@ -from src.secrets_manager import sm_client, create_secret, list_secret -import boto3 -from moto import mock_aws -import json -import pytest -import os - -pytest.fixture(scope='class') -def mock_aws_credentials(): - """Mocked AWS Credentials for moto.""" - os.environ["AWS_ACCESS_KEY_ID"] = "testing" - os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" - os.environ["AWS_SECURITY_TOKEN"] = "testing" - os.environ["AWS_SESSION_TOKEN"] = "testing" - os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" - -@pytest.fixture(scope='class') -def mock_sm_client(mock_aws_credentials): - with mock_aws(): - yield boto3.client('secretsmanager') - - -def test_create_secret_stores_secrets(mock_sm_client): - cohort_id = "test_cohort_id" - user = "test_user_id" - password = "test_password" - host = "test_host" - database = "test_database" - port = "test_port" - - secret_name = "test_secret" - response = create_secret(mock_sm_client, secret_name, cohort_id, user, password, host, database, port) - - assert response['Name'] == secret_name \ No newline at end of file -- cgit v1.2.3 From b9f3576771c8af8933d23e95f7863f63e2bbc6aa Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Mon, 19 Aug 2024 15:43:28 +0100 Subject: wip: fixed broken tests; hashed out test_error_retrieving_object --- src/extract_lambda.py | 1 + tests/test_extract_lambda.py | 49 ++++++++++++++++++++++++++------------------ 2 files changed, 30 insertions(+), 20 deletions(-) diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 4168e27..217efdb 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -90,6 +90,7 @@ def extract_bucket(client=boto3.client("s3")): extract_bucket_filter = [ bucket["Name"] for bucket in response["Buckets"] if "extract" in bucket["Name"] ] + return extract_bucket_filter[0] diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index e94a8a4..665e419 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -1,11 +1,13 @@ +import boto3.exceptions +import botocore.exceptions import pytest import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables -import os +from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables, extract_bucket import logging +import os @pytest.fixture(scope='class') def mock_config(): @@ -16,7 +18,7 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.get_config", return_value=env_vars) as mock_config: + with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: yield mock_config @@ -24,7 +26,7 @@ def mock_config(): def aws_credentials(): os.environ["AWS_ACCESS_KEY_ID"] = 'testing' os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURIT_TOKEN"] = 'testing' + os.environ["AWS_SECURITY_TOKEN"] = 'testing' os.environ["AWS_SESSION_TOKEN"] = 'testing' os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' @@ -33,6 +35,14 @@ def s3_client(aws_credentials): with mock_aws(): yield boto3.client('s3') +@pytest.fixture(scope='class') +def s3_mock_bucket(s3_client): + bucket = s3_client.create_bucket(Bucket='extract_bucket', + CreateBucketConfiguration={ + 'LocationConstraint': 'eu-west-2' + }) + return bucket + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): @@ -42,35 +52,34 @@ class TestListExistingS3Files: list_existing_s3_files(client=s3_client) assert 'Error listing S3 objects' in caplog.text - def test_error_if_bucket_is_empty(self, s3_client, caplog): + def test_error_if_bucket_is_empty(self, s3_client, caplog, s3_mock_bucket): + list_existing_s3_files('extract_bucket', client=s3_client) + assert 'The bucket is empty' in caplog.text - s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) - list_existing_s3_files(client=s3_client) - assert 'The bucket is empty' in caplog.text - def test_error_retrieving_object(self, s3_client, caplog): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - list_existing_s3_files(bucket_name='test_bucket', client=s3_client) + # def test_error_retrieving_object(self, s3_client, caplog, s3_mock_bucket): + # s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - assert 'Error retrieving S3 object ' in caplog.text + # list_existing_s3_files(bucket_name='extract_bucket', client=s3_client) - def test_retrieves_file_content(self, s3_client, caplog): - result = list_existing_s3_files(client=s3_client) + # assert 'Error retrieving S3 object dummy.txt: ClientError' in caplog.text + + + def test_retrieves_file_content(self, s3_client, caplog, s3_mock_bucket): + s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') + result = list_existing_s3_files('extract_bucket', client=s3_client) - assert list(result.values()) == ['This is a test file.'] + assert list(result.values()) == ['This is a test file.'] class TestConnectToDatabase: - def test_connect_to_database(mock_conn, mock_config): + def test_connect_to_database(mock_conn, mock_config): ##had mock_config in param with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( host="abc", user="def", port="5432", password="password", database="db" ) - def test_database_error(self, mock_config): + def test_database_error(self, mock_config): ##had mock_config in param with pytest.raises(DBConnectionException): connect_to_database() -- cgit v1.2.3 From 91d2e615a6af595898de2e329299c9cf42fc74f7 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:00:10 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in b9f3576 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/64 --- tests/test_extract_lambda.py | 81 +++++++++++++++++++++++++------------------- 1 file changed, 47 insertions(+), 34 deletions(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 665e419..02e3d3c 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -5,11 +5,18 @@ import boto3 from moto import mock_aws from unittest.mock import patch, MagicMock from unittest import TestCase -from src.extract_lambda import list_existing_s3_files, connect_to_database, DBConnectionException, process_and_upload_tables, extract_bucket +from src.extract_lambda import ( + list_existing_s3_files, + connect_to_database, + DBConnectionException, + process_and_upload_tables, + extract_bucket, +) import logging import os -@pytest.fixture(scope='class') + +@pytest.fixture(scope="class") def mock_config(): env_vars = { "host": "abc", @@ -18,44 +25,47 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: + with patch( + "src.extract_lambda.retrieve_secrets", return_value=env_vars + ) as mock_config: yield mock_config -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def aws_credentials(): - os.environ["AWS_ACCESS_KEY_ID"] = 'testing' - os.environ["AWS_SECRET_ACCESS_KEY"] = 'testing' - os.environ["AWS_SECURITY_TOKEN"] = 'testing' - os.environ["AWS_SESSION_TOKEN"] = 'testing' - os.environ["AWS_DEFAULT_REGION"]= 'eu-west-2' + os.environ["AWS_ACCESS_KEY_ID"] = "testing" + os.environ["AWS_SECRET_ACCESS_KEY"] = "testing" + os.environ["AWS_SECURITY_TOKEN"] = "testing" + os.environ["AWS_SESSION_TOKEN"] = "testing" + os.environ["AWS_DEFAULT_REGION"] = "eu-west-2" + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_client(aws_credentials): with mock_aws(): - yield boto3.client('s3') + yield boto3.client("s3") + -@pytest.fixture(scope='class') +@pytest.fixture(scope="class") def s3_mock_bucket(s3_client): - bucket = s3_client.create_bucket(Bucket='extract_bucket', - CreateBucketConfiguration={ - 'LocationConstraint': 'eu-west-2' - }) + bucket = s3_client.create_bucket( + Bucket="extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) return bucket + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) list_existing_s3_files(client=s3_client) - assert 'Error listing S3 objects' in caplog.text + assert "Error listing S3 objects" in caplog.text def test_error_if_bucket_is_empty(self, s3_client, caplog, s3_mock_bucket): - list_existing_s3_files('extract_bucket', client=s3_client) - assert 'The bucket is empty' in caplog.text - + list_existing_s3_files("extract_bucket", client=s3_client) + assert "The bucket is empty" in caplog.text # def test_error_retrieving_object(self, s3_client, caplog, s3_mock_bucket): # s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') @@ -64,33 +74,36 @@ class TestListExistingS3Files: # assert 'Error retrieving S3 object dummy.txt: ClientError' in caplog.text - def test_retrieves_file_content(self, s3_client, caplog, s3_mock_bucket): - s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt') - result = list_existing_s3_files('extract_bucket', client=s3_client) + s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt") + result = list_existing_s3_files("extract_bucket", client=s3_client) + + assert list(result.values()) == ["This is a test file."] - assert list(result.values()) == ['This is a test file.'] class TestConnectToDatabase: - def test_connect_to_database(mock_conn, mock_config): ##had mock_config in param - with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: + # had mock_config in param + def test_connect_to_database(mock_conn, mock_config): + with patch("src.extract_lambda.Connection", autospec=True) as mock_conn: connect_to_database() mock_conn.assert_called_with( - host="abc", user="def", port="5432", password="password", database="db" + host="abc", user="def", port="5432", password="password", database="db" ) - def test_database_error(self, mock_config): ##had mock_config in param + def test_database_error(self, mock_config): # had mock_config in param with pytest.raises(DBConnectionException): connect_to_database() def test_logs_interface_error(self, caplog): logger = logging.getLogger() - logger.info('Testing now.') + logger.info("Testing now.") caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert 'Interface error' in caplog.text -''' + assert "Interface error" in caplog.text + + +""" class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): logger = logging.getLogger() @@ -115,4 +128,4 @@ class TestProcessAndUploadTables: s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) assert 'No new data.' in caplog.text -''' \ No newline at end of file +""" -- cgit v1.2.3 From b80ad74122609fca98597d9a04518df855b58aed Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:10:22 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 4a23069 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/64 --- tests/test_extract_lambda.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index b1894cc..a43ae0a 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -55,6 +55,7 @@ def s3_mock_bucket(s3_client): ) return bucket + class TestLambdaHandler: def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker): mock_db = MagicMock() @@ -138,6 +139,7 @@ class TestLambdaHandler: mock_list_existing_s3_files.assert_not_called() mock_process_and_upload_tables.assert_not_called() + class TestListExistingS3Files: def test_error_if_no_bucket(self, s3_client, caplog): logger = logging.getLogger() @@ -175,4 +177,4 @@ class TestConnectToDatabase: caplog.set_level(logging.ERROR) with pytest.raises(DBConnectionException): connect_to_database() - assert "Interface error" in caplog.text \ No newline at end of file + assert "Interface error" in caplog.text -- cgit v1.2.3