From 2231ea89329bd500f7371b7395f5208f7a86c20e Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:11:40 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 8e20c5c according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/93 --- tests/test_fact_sales_order.py | 90 +++++++++++++++++++++++++++++++++--------- 1 file changed, 71 insertions(+), 19 deletions(-) (limited to 'tests/test_fact_sales_order.py') diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index 82845d7..87e3ade 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -1,57 +1,109 @@ -from src.fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency +from src.fact_sales_order import ( + create_dim_design, + create_dim_staff, + create_dim_currency, +) import pandas as pd + class TestCreateDimDesign: def test_dim_design_returns_dataframe(self): - d = {"test": ["Hello", "Bye"], "design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], "file_location": ["Hello", "Bye"], "Hello": ["Hello", "Bye"]} + d = { + "test": ["Hello", "Bye"], + "design_id": ["Hello", "Bye"], + "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"], + "Hello": ["Hello", "Bye"], + } test_df = {"design": pd.DataFrame(data=d)} result = create_dim_design(test_df) assert isinstance(result, pd.DataFrame) def test_dim_design_returns_correct_columns_and_values(self): - d = {"test": ["Hello", "Bye"], "design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], "file_location": ["Hello", "Bye"], "Hello": ["Hello", "Bye"]} + d = { + "test": ["Hello", "Bye"], + "design_id": ["Hello", "Bye"], + "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"], + "Hello": ["Hello", "Bye"], + } test_df = {"design": pd.DataFrame(data=d)} result = create_dim_design(test_df) - d2 = {"design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], "file_name": ["Hello", "Bye"], - "file_location": ["Hello", "Bye"]} + d2 = { + "design_id": ["Hello", "Bye"], + "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"], + } expected_df = pd.DataFrame(data=d2) expected_result = expected_df.copy() assert result.equals(expected_result) + class TestCreateDimStaff: def test_dim_staff_returns_dataframe(self): - d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} - d2 = {"department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + d = { + "staff_id": ["Hello", "Bye"], + "first_name": ["Hello", "Bye"], + "last_name": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } + d2 = { + "department_name": ["Hello", "Bye"], + "location": ["Hello", "Bye"], + "email_address": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) - assert isinstance(result, pd.DataFrame) + assert isinstance(result, pd.DataFrame) def test_dim_staff_returns_correct_columns_and_values(self): - d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} - d2 = {"department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + d = { + "staff_id": ["Hello", "Bye"], + "first_name": ["Hello", "Bye"], + "last_name": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } + d2 = { + "department_name": ["Hello", "Bye"], + "location": ["Hello", "Bye"], + "email_address": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) - expected_d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"]} + expected_d = { + "staff_id": ["Hello", "Bye"], + "first_name": ["Hello", "Bye"], + "last_name": ["Hello", "Bye"], + "department_name": ["Hello", "Bye"], + "location": ["Hello", "Bye"], + "email_address": ["Hello", "Bye"], + } expected_df = pd.DataFrame(data=expected_d) expected_result = expected_df.copy() - assert result.equals(expected_result) + assert result.equals(expected_result) + class TestCreateDimCurrency: def test_dim_currency_returns_dataframe(self): d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"]} test_df = {"currency": pd.DataFrame(data=d)} result = create_dim_currency(test_df) - assert isinstance(result, pd.DataFrame) - + assert isinstance(result, pd.DataFrame) + def test_dim_currency_returns_columns_and_values(self): d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"]} test_df = {"currency": pd.DataFrame(data=d)} result = create_dim_currency(test_df) - expected_d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"], "currency_name": ["US Dollar", "Euro", "Pound"]} + expected_d = { + "currency_id": [1, 2, 3], + "currency_code": ["USD", "EUR", "GBP"], + "currency_name": ["US Dollar", "Euro", "Pound"], + } expected_df = pd.DataFrame(data=expected_d) expected_result = expected_df.copy() assert result.equals(expected_result) - - \ No newline at end of file -- cgit v1.2.3 From 3ff2182b8256594dfbfe7d8c7480d2ee70067ce5 Mon Sep 17 00:00:00 2001 From: lian-manonog Date: Fri, 23 Aug 2024 11:46:59 +0100 Subject: trying to resolce git index issue conflicts - commiting was the only solution --- src/transform_lambda.py | 13 ++++--------- tests/test_fact_sales_order.py | 4 ++++ 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'tests/test_fact_sales_order.py') diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 3e74ee0..44454e2 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -6,9 +6,6 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq from src.dataframes import * - -# from src.extract_lambda import extract_bucket, DBConnectionException -import boto3 from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime @@ -34,7 +31,7 @@ logging.basicConfig( logging.getLogger("botocore").setLevel(logging.WARNING) -tables = [ +TABLES = [ "sales_order", "transaction", "payment", @@ -54,12 +51,11 @@ def lambda_handler(event, context): try: db = connect_to_database() - bucket = bucket_name("transform") + bucket = bucket_name('transform') + existing_s3_files = list_existing_s3_files(bucket) - dict_of_df = read_from_s3_subfolder_to_df( - tables, extract_bucket(), client=boto3.client("s3") - ) + dict_of_df = read_from_s3_subfolder_to_df(TABLES, bucket_name('extract'), client=boto3.client("s3")) immutable_df_dict = { "dim_counterparty": create_dim_counterparty(dict_of_df), @@ -134,7 +130,6 @@ def process_to_parquet_and_upload_to_s3( return status - def retrieve_secrets(): secret_name = "bentley-secrets" region_name = "eu-west-2" diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index 87e3ade..c4fc9f4 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -1,8 +1,12 @@ +<<<<<<< Updated upstream from src.fact_sales_order import ( create_dim_design, create_dim_staff, create_dim_currency, ) +======= +from fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency +>>>>>>> Stashed changes import pandas as pd -- cgit v1.2.3 From c3e04ab0415ddeedfa1a304296aa0e34fb5f2a1f Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 10:47:15 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 3ff2182 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/93 --- src/transform_lambda.py | 9 ++++++--- tests/test_fact_sales_order.py | 16 +++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) (limited to 'tests/test_fact_sales_order.py') diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 44454e2..defa15d 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -51,11 +51,13 @@ def lambda_handler(event, context): try: db = connect_to_database() - bucket = bucket_name('transform') - + bucket = bucket_name("transform") + existing_s3_files = list_existing_s3_files(bucket) - dict_of_df = read_from_s3_subfolder_to_df(TABLES, bucket_name('extract'), client=boto3.client("s3")) + dict_of_df = read_from_s3_subfolder_to_df( + TABLES, bucket_name("extract"), client=boto3.client("s3") + ) immutable_df_dict = { "dim_counterparty": create_dim_counterparty(dict_of_df), @@ -130,6 +132,7 @@ def process_to_parquet_and_upload_to_s3( return status + def retrieve_secrets(): secret_name = "bentley-secrets" region_name = "eu-west-2" diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index c4fc9f4..dad245e 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -1,13 +1,13 @@ -<<<<<<< Updated upstream +import pandas as pd +from fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency from src.fact_sales_order import ( create_dim_design, create_dim_staff, create_dim_currency, ) -======= -from fact_sales_order import create_dim_design, create_dim_staff, create_dim_currency ->>>>>>> Stashed changes -import pandas as pd +<< << << < Updated upstream +== == == = +>>>>>> > Stashed changes class TestCreateDimDesign: @@ -60,7 +60,8 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame( + data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) assert isinstance(result, pd.DataFrame) @@ -77,7 +78,8 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame( + data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) expected_d = { "staff_id": ["Hello", "Bye"], -- cgit v1.2.3 From 88f1ef765a9d1113757552ee38ad1bbdb708b629 Mon Sep 17 00:00:00 2001 From: lian-manonog <160282780+lian-manonog@users.noreply.github.com> Date: Fri, 23 Aug 2024 14:53:06 +0100 Subject: Removed redundant empty lines of code --- tests/test_fact_sales_order.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'tests/test_fact_sales_order.py') diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index dad245e..7592f68 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -5,10 +5,6 @@ from src.fact_sales_order import ( create_dim_staff, create_dim_currency, ) -<< << << < Updated upstream -== == == = ->>>>>> > Stashed changes - class TestCreateDimDesign: def test_dim_design_returns_dataframe(self): -- cgit v1.2.3 From 59035d00133eed3f258f75e3a99ce57cae35989d Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Fri, 23 Aug 2024 13:53:17 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 88f1ef7 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/94 --- tests/test_fact_sales_order.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'tests/test_fact_sales_order.py') diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py index 7592f68..48426b4 100644 --- a/tests/test_fact_sales_order.py +++ b/tests/test_fact_sales_order.py @@ -6,6 +6,7 @@ from src.fact_sales_order import ( create_dim_currency, ) + class TestCreateDimDesign: def test_dim_design_returns_dataframe(self): d = { @@ -56,8 +57,7 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame( - data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) assert isinstance(result, pd.DataFrame) @@ -74,8 +74,7 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame( - data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) expected_d = { "staff_id": ["Hello", "Bye"], -- cgit v1.2.3