From 17275cb511b0b134e34247f3cbf008a4da50768b Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 15:03:50 +0100 Subject: wip: amend extract_lambda test --- tests/test_extract_lambda.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'tests/test_extract_lambda.py') diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 7707cbf..4a5157b 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -12,6 +12,7 @@ from src.extract_lambda import ( DBConnectionException, lambda_handler, process_and_upload_tables, + retrieve_secrets ) @@ -24,7 +25,7 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.get_config", return_value=env_vars) as mock_config: + with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: yield mock_config @@ -140,7 +141,7 @@ class TestListExistingS3Files: Bucket="extract_bucket", CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, ) - list_existing_s3_files(client=s3_client) + list_existing_s3_files("extract_bucket", client=s3_client) assert "The bucket is empty" in caplog.text def test_error_retrieving_object(self, s3_client, caplog): @@ -176,9 +177,8 @@ class TestConnectToDatabase: assert "Interface error" in caplog.text -""" class TestProcessAndUploadTables: - def test_error_process_and_upload_tables(mock_conn, mock_config, s3_client, caplog): + def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): logger = logging.getLogger() logger.info('Testing now.') caplog.set_level(logging.ERROR) @@ -188,17 +188,17 @@ class TestProcessAndUploadTables: "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'"] return_values = [[['Fruits']], [['Vegetable','Sour','Green'],['Berry','Sweet','Red']], - [['Food_type'],['Flavour'],['Colour']]] + [['Food_type'],['Flavour'],['Colour']]] # why are individual column names in lists vals = dict(zip(queries,return_values)) + # {"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';": [['Fruits']], 'SELECT * FROM Fruits;': [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'": [['Food_type'], ['Flavour'], ['Colour']]} - #### - with patch('src.extract_lambda.connect_to_database') as mock_db: - mock_db().run.side_effects = return_values + with patch('src.extract_lambda.Connection') as mock_db: + mock_db().run.side_effect = return_values s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' existing_files = {s3_key: 'Food_type,Flavour,Colour\nFruit,Sour,Green\nBerry,Sweet,Red'} - s3_client.create_bucket(Bucket='extract_bucket', + s3_client.create_bucket(Bucket='test_extract_bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) + print(s3_client.list_buckets) s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) - assert 'No new data.' in caplog.text -""" + assert 'No new data.' in caplog.text \ No newline at end of file -- cgit v1.2.3 From e4b66476a174edb68992b00b37bef2d0e0be3969 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 15:57:14 +0100 Subject: wip: fixing last test --- tests/test_extract_lambda.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) (limited to 'tests/test_extract_lambda.py') diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 4a5157b..01d7add 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -181,24 +181,27 @@ class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): logger = logging.getLogger() logger.info('Testing now.') - caplog.set_level(logging.ERROR) + caplog.set_level(logging.INFO) #### - queries = ["SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", - "SELECT * FROM Fruits;", - "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'"] + queries = [ + "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", + "SELECT * FROM Fruits;", + "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'" + ] return_values = [[['Fruits']], - [['Vegetable','Sour','Green'],['Berry','Sweet','Red']], - [['Food_type'],['Flavour'],['Colour']]] # why are individual column names in lists + [['Vegetable','Sour','Green','2022-11-03 14:20:49.962'],['Berry','Sweet','Red','2022-11-03 14:20:49.962']], + [['Food_type'],['Flavour'],['Colour'],['last_updated']]] # why are individual column names in lists vals = dict(zip(queries,return_values)) # {"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';": [['Fruits']], 'SELECT * FROM Fruits;': [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'": [['Food_type'], ['Flavour'], ['Colour']]} with patch('src.extract_lambda.Connection') as mock_db: mock_db().run.side_effect = return_values s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' - existing_files = {s3_key: 'Food_type,Flavour,Colour\nFruit,Sour,Green\nBerry,Sweet,Red'} + existing_files = {s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red, 2022-11-03 14:20:49.962'} s3_client.create_bucket(Bucket='test_extract_bucket', CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) - print(s3_client.list_buckets) - s3_client.upload_file('tests/dummy_identical.csv', 'extract_bucket', s3_key) + s3_client.upload_file('tests/dummy_identical.csv', 'test_extract_bucket', s3_key) process_and_upload_tables(mock_db(), existing_files, client=s3_client) - assert 'No new data.' in caplog.text \ No newline at end of file + print('logger', logger.info('hello')) + print('our test', caplog.text) + assert 'No new data' in caplog.text \ No newline at end of file -- cgit v1.2.3 From 333822a70640712ac57036d37f7d8ac0787e9cc0 Mon Sep 17 00:00:00 2001 From: HastarTara Date: Mon, 19 Aug 2024 16:19:16 +0100 Subject: bugfixing --- tests/test_extract_lambda.py | 39 +++++++++++++++++++++++---------------- 1 file changed, 23 insertions(+), 16 deletions(-) (limited to 'tests/test_extract_lambda.py') diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 01d7add..a4e8f2b 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -179,29 +179,36 @@ class TestConnectToDatabase: class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): - logger = logging.getLogger() - logger.info('Testing now.') caplog.set_level(logging.INFO) - #### + + # Mock return values for database queries queries = [ "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", - "SELECT * FROM Fruits;", - "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'" - ] - return_values = [[['Fruits']], - [['Vegetable','Sour','Green','2022-11-03 14:20:49.962'],['Berry','Sweet','Red','2022-11-03 14:20:49.962']], - [['Food_type'],['Flavour'],['Colour'],['last_updated']]] # why are individual column names in lists - vals = dict(zip(queries,return_values)) - # {"SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';": [['Fruits']], 'SELECT * FROM Fruits;': [['Vegetable', 'Sour', 'Green'], ['Berry', 'Sweet', 'Red']], "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits'": [['Food_type'], ['Flavour'], ['Colour']]} + "SELECT * FROM Fruits WHERE last_updated > :latest;", + "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits';" + ] + return_values = [ + [['Fruits']], + [], # No new rows with a more recent last_updated timestamp + [['Food_type'], ['Flavour'], ['Colour'], ['last_updated']] + ] + vals = dict(zip(queries, return_values)) + # Patch the database connection and set return values for queries with patch('src.extract_lambda.Connection') as mock_db: mock_db().run.side_effect = return_values s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' - existing_files = {s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red, 2022-11-03 14:20:49.962'} + existing_files = { + s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red,2022-11-03 14:20:49.962' + } + + # Simulate S3 bucket and file setup s3_client.create_bucket(Bucket='test_extract_bucket', - CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) + CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) s3_client.upload_file('tests/dummy_identical.csv', 'test_extract_bucket', s3_key) + + # Run the process_and_upload_tables function process_and_upload_tables(mock_db(), existing_files, client=s3_client) - print('logger', logger.info('hello')) - print('our test', caplog.text) - assert 'No new data' in caplog.text \ No newline at end of file + + # Assert that the log contains "No new data" + assert 'No new data' in caplog.text -- cgit v1.2.3 From 4f629e532a1e989096985dc9cd9e6f03f7b44354 Mon Sep 17 00:00:00 2001 From: Ellie Date: Mon, 19 Aug 2024 16:33:46 +0100 Subject: add working process and upload tables test --- tests/test_extract_lambda.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'tests/test_extract_lambda.py') diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index a4e8f2b..3405743 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -209,6 +209,7 @@ class TestProcessAndUploadTables: # Run the process_and_upload_tables function process_and_upload_tables(mock_db(), existing_files, client=s3_client) - # Assert that the log contains "No new data" assert 'No new data' in caplog.text + + # process and upload tables needs more tests \ No newline at end of file -- cgit v1.2.3 From a42d030fb663ad7eb040498cfc5f0627a27d6cc6 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:11:44 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 4f629e5 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/65 --- src/extract_lambda.py | 8 +++----- tests/test_extract_lambda.py | 34 ++++++++++++++++++++-------------- 2 files changed, 23 insertions(+), 19 deletions(-) (limited to 'tests/test_extract_lambda.py') diff --git a/src/extract_lambda.py b/src/extract_lambda.py index 5a5a631..9b17ef2 100644 --- a/src/extract_lambda.py +++ b/src/extract_lambda.py @@ -151,9 +151,9 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): table_name = table[0] rows = db.run( f"SELECT * FROM {identifier(table_name)} WHERE last_updated >= :latest;", - latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, + latest={datetime.strftime(latest_timestamp, "%Y-%m-%d %H:%M:%S")}, ) - print('rows', rows) + print("rows", rows) # Creating a temporary file path and writing the column name to it followed by each row of data if rows: csv_file_path = f"/tmp/{table_name}.csv" @@ -183,7 +183,5 @@ def process_and_upload_tables(db, existing_files, client=boto3.client("s3")): logger.error(f"Error uploading to S3: {e}") else: load_status["no change"].append(table_name) - logger.info( - f"No new data" - ) + logger.info(f"No new data") return load_status diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 3405743..5a1c5b2 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -12,7 +12,7 @@ from src.extract_lambda import ( DBConnectionException, lambda_handler, process_and_upload_tables, - retrieve_secrets + retrieve_secrets, ) @@ -25,7 +25,9 @@ def mock_config(): "password": "password", "database": "db", } - with patch("src.extract_lambda.retrieve_secrets", return_value=env_vars) as mock_config: + with patch( + "src.extract_lambda.retrieve_secrets", return_value=env_vars + ) as mock_config: yield mock_config @@ -185,31 +187,35 @@ class TestProcessAndUploadTables: queries = [ "SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';", "SELECT * FROM Fruits WHERE last_updated > :latest;", - "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits';" + "SELECT column_name FROM INFORMATION_SCHEMA.COLUMNS where table_name = 'Fruits';", ] return_values = [ - [['Fruits']], + [["Fruits"]], [], # No new rows with a more recent last_updated timestamp - [['Food_type'], ['Flavour'], ['Colour'], ['last_updated']] + [["Food_type"], ["Flavour"], ["Colour"], ["last_updated"]], ] vals = dict(zip(queries, return_values)) # Patch the database connection and set return values for queries - with patch('src.extract_lambda.Connection') as mock_db: + with patch("src.extract_lambda.Connection") as mock_db: mock_db().run.side_effect = return_values - s3_key = 'Fruits/2024/08/15/Fruits_16:46:30.csv' + s3_key = "Fruits/2024/08/15/Fruits_16:46:30.csv" existing_files = { - s3_key: 'Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red,2022-11-03 14:20:49.962' + s3_key: "Food_type,Flavour,Colour,last_updated\nVegetable,Sour,Green,2022-11-03 14:20:49.962\nBerry,Sweet,Red,2022-11-03 14:20:49.962" } # Simulate S3 bucket and file setup - s3_client.create_bucket(Bucket='test_extract_bucket', - CreateBucketConfiguration={'LocationConstraint': 'eu-west-2'}) - s3_client.upload_file('tests/dummy_identical.csv', 'test_extract_bucket', s3_key) - + s3_client.create_bucket( + Bucket="test_extract_bucket", + CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}, + ) + s3_client.upload_file( + "tests/dummy_identical.csv", "test_extract_bucket", s3_key + ) + # Run the process_and_upload_tables function process_and_upload_tables(mock_db(), existing_files, client=s3_client) # Assert that the log contains "No new data" - assert 'No new data' in caplog.text + assert "No new data" in caplog.text - # process and upload tables needs more tests \ No newline at end of file + # process and upload tables needs more tests -- cgit v1.2.3 From b499d78dc660017694ec683c90aba3f558c00669 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Mon, 19 Aug 2024 16:14:07 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in f014d1a according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/65 --- tests/test_extract_lambda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'tests/test_extract_lambda.py') diff --git a/tests/test_extract_lambda.py b/tests/test_extract_lambda.py index 347ef22..3931cfc 100644 --- a/tests/test_extract_lambda.py +++ b/tests/test_extract_lambda.py @@ -180,6 +180,7 @@ class TestConnectToDatabase: connect_to_database() assert "Interface error" in caplog.text + class TestProcessAndUploadTables: def test_error_process_and_upload_tables(mock_conn, s3_client, caplog): caplog.set_level(logging.INFO) @@ -218,4 +219,3 @@ class TestProcessAndUploadTables: process_and_upload_tables(mock_db(), existing_files, client=s3_client) # Assert that the log contains "No new data" assert "No new data" in caplog.text - -- cgit v1.2.3