Merge branch 'development' into extract-lambda-code-update

2 files changed, 111 insertions, 53 deletions

diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py
index 02e3d3c..b1894cc 100644
--- a/tests/test_extract_lambda.py
+++ b/tests/test_extract_lambda.py
@@ -5,15 +5,16 @@ import boto3
 from moto import mock_aws
 from unittest.mock import patch, MagicMock
 from unittest import TestCase
+import os
+import logging
+import json
 from src.extract_lambda import (
     list_existing_s3_files,
     connect_to_database,
     DBConnectionException,
+    lambda_handler,
     process_and_upload_tables,
-    extract_bucket,
 )
-import logging
-import os
 
 
 @pytest.fixture(scope="class")
@@ -54,6 +55,88 @@ def s3_mock_bucket(s3_client):
     )
     return bucket
 
+class TestLambdaHandler:
+    def test_lambda_handler_files_processed_and_uploaded_successfully(self, mocker):
+        mock_db = MagicMock()
+        mock_db.run.side_effect = [
+            [["Fruits"]],
+            [["Vegetable", "Sour", "Green"], ["Berry", "Sweet", "Red"]],
+            [["Food_type"], ["Flavour"], ["Colour"]],
+        ]
+        mock_db.columns.return_value = [
+            {"name": "Food_type"},
+            {"name": "Flavour"},
+            {"name": "Colour"},
+        ]
+        with patch("src.extract_lambda.connect_to_database", return_value=mock_db):
+            mock_process_and_upload_tables = mocker.patch(
+                "src.extract_lambda.process_and_upload_tables", return_value=mock_db
+            )
+            mock_list_existing_s3_files = mocker.patch(
+                "src.extract_lambda.list_existing_s3_files", return_value={}
+            )
+            event = {}
+            context = {}
+            response = lambda_handler(event, context)
+            assert response["statusCode"] == 200
+            assert (
+                json.loads(response["body"])
+                == "CSV files processed and uploaded successfully."
+            )
+            mock_list_existing_s3_files.assert_called_once()
+            mock_process_and_upload_tables.assert_called_once_with(mock_db, {})
+            mock_db.close.assert_called_once()
+
+    def test_lambda_handler_no_changes_detected_no_files_uploaded(self, mocker):
+        mock_db = MagicMock()
+        mock_db.run.side_effect = [
+            [["Fruits"]],
+            [["Vegetable", "Sour", "Green"], ["Berry", "Sweet", "Red"]],
+            [["Food_type"], ["Flavour"], ["Colour"]],
+        ]
+        mock_db.columns.return_value = [
+            {"name": "Food_type"},
+            {"name": "Flavour"},
+            {"name": "Colour"},
+        ]
+
+        with patch("src.extract_lambda.connect_to_database", return_value=mock_db):
+            mock_process_and_upload_tables = mocker.patch(
+                "src.extract_lambda.process_and_upload_tables", return_value=False
+            )
+            mock_list_existing_s3_files = mocker.patch(
+                "src.extract_lambda.list_existing_s3_files", return_value={}
+            )
+            event = {}
+            context = {}
+            response = lambda_handler(event, context)
+            assert response["statusCode"] == 200
+            assert (
+                json.loads(response["body"])
+                == "No changes detected, no CSV files were uploaded."
+            )
+            mock_list_existing_s3_files.assert_called_once()
+            mock_process_and_upload_tables.assert_called_once_with(mock_db, {})
+            mock_db.close.assert_called_once()
+
+    def test_lambda_handler_exception_error(self, mocker):
+        with patch(
+            "src.extract_lambda.connect_to_database",
+            side_effect=Exception("Database connection error"),
+        ):
+            mock_process_and_upload_tables = mocker.patch(
+                "src.extract_lambda.process_and_upload_tables"
+            )
+            mock_list_existing_s3_files = mocker.patch(
+                "src.extract_lambda.list_existing_s3_files"
+            )
+            event = {}
+            context = {}
+            response = lambda_handler(event, context)
+            assert response["statusCode"] == 500
+            assert json.loads(response["body"]) == "Internal server error."
+            mock_list_existing_s3_files.assert_not_called()
+            mock_process_and_upload_tables.assert_not_called()
 
 class TestListExistingS3Files:
     def test_error_if_no_bucket(self, s3_client, caplog):
@@ -67,17 +150,9 @@ class TestListExistingS3Files:
         list_existing_s3_files("extract_bucket", client=s3_client)
         assert "The bucket is empty" in caplog.text
 
-    # def test_error_retrieving_object(self, s3_client, caplog, s3_mock_bucket):
-    #     s3_client.upload_file('tests/dummy.txt', 'extract_bucket', 'dummy.txt')
-
-    #     list_existing_s3_files(bucket_name='extract_bucket', client=s3_client)
-
-    #     assert 'Error retrieving S3 object dummy.txt: ClientError' in caplog.text
-
     def test_retrieves_file_content(self, s3_client, caplog, s3_mock_bucket):
         s3_client.upload_file("tests/dummy.txt", "extract_bucket", "dummy.txt")
         result = list_existing_s3_files("extract_bucket", client=s3_client)
-
         assert list(result.values()) == ["This is a test file."]
 
 
@@ -100,32 +175,4 @@ class TestConnectToDatabase:
         caplog.set_level(logging.ERROR)
         with pytest.raises(DBConnectionException):
             connect_to_database()
-        assert "Interface error" in caplog.text
-
-
-"""
-class TestProcessAndUploadTables:
-    def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog):
-        logger = logging.getLogger()
-        logger.info('Testing now.')
-        caplog.set_level(logging.ERROR)
-        ####
-        queries = ["SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';",
-                    "SELECT * FROM Fruits;",
-                    "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'"]
-        return_values = [[['Fruits']],
-                        [['Vegetable','Sour','Green'],['Berry','Sweet','Red']],
-                        [['Food_type'],['Flavour'],['Colour']]]
-        vals = dict(zip(queries,return_values))
-
-        ####
-        with patch('src.extract_lambda.connect_to_database') as mock_db:
-            mock_db().run.side_effects = return_values
-            s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv'
-            existing_files = {s3_key: 'Food_type,Flavour,Colour\nFruit,Sour,Green\nBerry,Sweet,Red'}
-            s3_client.create_bucket(Bucket='extract_bucket', 
-                        CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'})
-            s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key)
-            process_and_upload_tables(mock_db(), existing_files, client=s3_client)
-            assert 'No new data.' in caplog.text
-"""
+        assert "Interface error" in caplog.text
\ No newline at end of file
diff --git a/tests/test_secrets_manager.py b/tests/test_secrets_manager.py
index a30be86..609c572 100644
--- a/tests/test_secrets_manager.py
+++ b/tests/test_secrets_manager.py
@@ -3,10 +3,11 @@ import boto3
 import botocore.exceptions
 from moto import mock_aws
 import json
-import pytest 
+import pytest
 import os
 
-@pytest.fixture(scope='function')
+
+@pytest.fixture(scope="function")
 def aws_credentials():
     """Mocked AWS Credentials for moto."""
     os.environ["AWS_ACCESS_KEY_ID"] = "testing"
@@ -15,12 +16,14 @@ def aws_credentials():
     os.environ["AWS_SESSION_TOKEN"] = "testing"
     os.environ["AWS_DEFAULT_REGION"] = "eu-west-2"
 
-@pytest.fixture(scope='function')
+
+@pytest.fixture(scope="function")
 def mock_sm_client(aws_credentials):
     with mock_aws():
         yield boto3.client("secretsmanager")
 
-@pytest.fixture(scope='function')
+
+@pytest.fixture(scope="function")
 def mock_store_secret(mock_sm_client):
     secret = {
         "cohort_id": "test_cohort_id",
@@ -28,15 +31,18 @@ def mock_store_secret(mock_sm_client):
         "password": "test_password",
         "host": "test_host",
         "database": "test_database",
-        "port": "test_port"
+        "port": "test_port",
     }
 
     secret_name = "test_secret"
 
-    response = mock_sm_client.create_secret(Name=secret_name, SecretString=json.dumps(secret))
+    response = mock_sm_client.create_secret(
+        Name=secret_name, SecretString=json.dumps(secret)
+    )
 
     return response
 
+
 def test_retrieves_secrets_returns_dictionary(mock_sm_client, mock_store_secret):
     secret_name = "test_secret"
 
@@ -44,8 +50,10 @@ def test_retrieves_secrets_returns_dictionary(mock_sm_client, mock_store_secret)
 
     assert isinstance(result, dict)
 
-def test_retrieves_secrets_returns_correct_keys_and_values(mock_sm_client, mock_store_secret):
 
+def test_retrieves_secrets_returns_correct_keys_and_values(
+    mock_sm_client, mock_store_secret
+):
     secret_name = "test_secret"
 
     result = retrieve_secrets(mock_sm_client, secret_name)
@@ -57,17 +65,20 @@ def test_retrieves_secrets_returns_correct_keys_and_values(mock_sm_client, mock_
     assert result["database"] == "test_database"
     assert result["port"] == "test_port"
 
-def test_retrieves_secrets_raises_error_if_secret_name_incorrect_data_type(mock_sm_client):
-    secret_name = [1, 2, 3]
 
+def test_retrieves_secrets_raises_error_if_secret_name_incorrect_data_type(
+    mock_sm_client,
+):
+    secret_name = [1, 2, 3]
 
     with pytest.raises(botocore.exceptions.ParamValidationError) as error:
         retrieve_secrets(mock_sm_client, secret_name)
 
 
-def test_retrieves_secrets_raises_error_if_secret_name_does_not_exist(mock_sm_client, mock_store_secret):
-    secret_name = 'test_secret_2'
-
+def test_retrieves_secrets_raises_error_if_secret_name_does_not_exist(
+    mock_sm_client, mock_store_secret
+):
+    secret_name = "test_secret_2"
 
     with pytest.raises(botocore.exceptions.ClientError) as error:
-        retrieve_secrets(mock_sm_client, secret_name)
\ No newline at end of file
+        retrieve_secrets(mock_sm_client, secret_name)