From 151429859bca904cbacf18f4b169f1f768fa212a Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 12:01:53 +0100 Subject: remove import as not required --- src/load_lambda.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/load_lambda.py b/src/load_lambda.py index 6e6bc80..685c562 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -5,7 +5,6 @@ import pyarrow.parquet as pq from io import BytesIO import logging import json -from src.extract_lambda import retrieve_secrets from sqlalchemy import create_engine @@ -169,7 +168,7 @@ def upload_dfs_to_database(): table_name, con=db_engine, schema="project_team_2", - if_exists="overwrite", + if_exists="append", index=False, ) upload_status["uploaded"].append(table_name) @@ -183,7 +182,7 @@ def upload_dfs_to_database(): table_name, con=db_engine, schema="project_team_2", - if_exists="overwrite", + if_exists="append", index=False, ) upload_status["uploaded"].append(table_name) @@ -195,3 +194,6 @@ def upload_dfs_to_database(): logger.error(f"{file_name} does not correspond with table in database") db_engine.dispose() return upload_status + +if __name__ == "__main__": + lambda_handler(None, None) -- cgit v1.2.3 From 8cd9edde84f4ca706ad93b143c5ff7e3397ce981 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 12:28:58 +0100 Subject: add json.loads to retrieve secrests function --- src/load_lambda.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'src') diff --git a/src/load_lambda.py b/src/load_lambda.py index 685c562..f08e335 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -40,16 +40,19 @@ def lambda_handler(event, context): return {"statusCode": 500, "body": json.dumps("Internal server error.")} -def retrieve_secrets(): - secret_name = "bentley-RDS-credentials" +def retrieve_secrets(client=None, secret_name=None): + session = boto3.session.Session() region_name = "eu-west-2" - # Create a Secrets Manager client - session = boto3.session.Session() - client = session.client(service_name="secretsmanager", region_name=region_name) + if secret_name == None: + secret_name = "bentley-RDS-credentials" + if client == None: + client = session.client(service_name="secretsmanager", region_name=region_name) + try: get_secret_value_response = client.get_secret_value(SecretId=secret_name) + print(get_secret_value_response) except ClientError as e: logger.error(f"Failed to retrieve secret {secret_name}: {str(e)}") raise e @@ -57,7 +60,7 @@ def retrieve_secrets(): logger.error(f"Secret {secret_name} does not contain a SecretString") raise ValueError(f"Secret {secret_name} does not contain a SecretString") - return get_secret_value_response["SecretString"] + return json.loads(get_secret_value_response["SecretString"]) # connect to database, slightly different way of doing it, to allow manipulation through pandas -- cgit v1.2.3 From d623c42a891f2fe8a26493354af0d9e299f3c526 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 27 Aug 2024 15:19:14 +0100 Subject: refactor: add parameter for sm_secret --- src/load_lambda.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/load_lambda.py b/src/load_lambda.py index f08e335..11d1d70 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -49,7 +49,6 @@ def retrieve_secrets(client=None, secret_name=None): if client == None: client = session.client(service_name="secretsmanager", region_name=region_name) - try: get_secret_value_response = client.get_secret_value(SecretId=secret_name) print(get_secret_value_response) @@ -66,9 +65,12 @@ def retrieve_secrets(client=None, secret_name=None): # connect to database, slightly different way of doing it, to allow manipulation through pandas -def connect_to_db_and_return_engine(): +def connect_to_db_and_return_engine(sm_secret=None): + if sm_secret is None: + sm_secret = retrieve_secrets() + try: - secrets = json.loads(retrieve_secrets()) + secrets = json.loads(sm_secret) host = secrets["host"] port = secrets["port"] user = secrets["user"] @@ -198,5 +200,6 @@ def upload_dfs_to_database(): db_engine.dispose() return upload_status + if __name__ == "__main__": lambda_handler(None, None) -- cgit v1.2.3 From cbfc98a9f43b5a0dae95337057c18c9dc2a298e3 Mon Sep 17 00:00:00 2001 From: Alex Schofield Date: Tue, 27 Aug 2024 16:00:29 +0100 Subject: wip: update TestLambdaHandler & lambda_handler function --- src/load_lambda.py | 19 +++++++++++-------- tests/test_load_lambda.py | 12 +++++++++--- 2 files changed, 20 insertions(+), 11 deletions(-) (limited to 'src') diff --git a/src/load_lambda.py b/src/load_lambda.py index 11d1d70..39fa27d 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -23,18 +23,21 @@ logging.getLogger("botocore").setLevel(logging.INFO) def lambda_handler(event, context): try: uploaded_tables = upload_dfs_to_database() - if not uploaded_tables["uploaded"]: + if uploaded_tables["not_uploaded"]: return { "statusCode": 200, "body": json.dumps("No dataframes were uploaded."), } - return { - "statusCode": 200, - "body": json.dumps( - f"""The following dataframes were uploaded successfully: - {uploaded_tables["uploaded"]} .""" - ), - } + + if uploaded_tables["uploaded"]: + return { + "statusCode": 200, + "body": json.dumps( + f"""The following dataframes were uploaded successfully: + {uploaded_tables["uploaded"]} .""" + ), + } + except Exception as e: logger.error(f"Error: {e}", exc_info=True) return {"statusCode": 500, "body": json.dumps("Internal server error.")} diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index a29b75a..9286e48 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -35,7 +35,7 @@ class TestLambdaHandler: def test_lambda_handler_returns_success(self, mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", - return_value={"uploaded": ["table_one", "table_two"]}, + return_value={"uploaded": ["table_one", "table_two"], "not_uploaded": []}, ) result = lambda_handler(None, None) assert result["statusCode"] == 200 @@ -45,14 +45,20 @@ class TestLambdaHandler: def test_lambda_handler_does_not_upload_anything(self, mocker): mocker.patch( "src.load_lambda.upload_dfs_to_database", - return_value={"uploaded": []}, + return_value={"uploaded": [], "not_uploaded": []}, ) result = lambda_handler(None, None) assert result["statusCode"] == 200 assert "No dataframes were uploaded" in result["body"] def test_lambda_handler_returns_exception(self, mocker): - pass + mocker.patch( + "src.load_lambda.upload_dfs_to_database", + return_value={"test": []}, + ) + + with pytest.raises(Exception): + lambda_handler(None, None) class TestRetrieveSecrets: -- cgit v1.2.3 From 27f89b78775f9b6fd8d3d560689c53db2beb1b64 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 16:39:38 +0100 Subject: add logger error to lambda handler --- src/load_lambda.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) (limited to 'src') diff --git a/src/load_lambda.py b/src/load_lambda.py index 39fa27d..9e15af3 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -5,6 +5,7 @@ import pyarrow.parquet as pq from io import BytesIO import logging import json +import traceback from sqlalchemy import create_engine @@ -28,8 +29,7 @@ def lambda_handler(event, context): "statusCode": 200, "body": json.dumps("No dataframes were uploaded."), } - - if uploaded_tables["uploaded"]: + elif uploaded_tables["uploaded"]: return { "statusCode": 200, "body": json.dumps( @@ -37,10 +37,12 @@ def lambda_handler(event, context): {uploaded_tables["uploaded"]} .""" ), } - + else: + logger.error(f"error") + return {"error"} except Exception as e: - logger.error(f"Error: {e}", exc_info=True) - return {"statusCode": 500, "body": json.dumps("Internal server error.")} + logger.error({e}) + return {"statusCode": 500, "body": {e}} def retrieve_secrets(client=None, secret_name=None): -- cgit v1.2.3 From aed1c19a39062e8fe86cf0a531b8d1486b06d1ac Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 27 Aug 2024 12:42:25 +0100 Subject: test: fact transformation function for payment test passes, other fact functions are equivalent, no tests written --- src/dataframes.py | 251 ++++++++++++++--------------------------- tests/test_dataframes.py | 144 +++++++++++++++++++++++ tests/test_fact_sales_order.py | 246 ---------------------------------------- 3 files changed, 229 insertions(+), 412 deletions(-) create mode 100644 tests/test_dataframes.py delete mode 100644 tests/test_fact_sales_order.py (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index ab53063..41f39b8 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -2,7 +2,7 @@ import pandas as pd from bs4 import BeautifulSoup import requests -# Table names: +#Table names: # fact_sales_order # fact_purchase_orders # fact_payment @@ -16,214 +16,133 @@ import requests # dim_counterparty +#no test, same as fact_payment def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" - df_sales["created_date"] = pd.to_datetime(df_sales["created_at"]).dt.date - df_sales["created_time"] = pd.to_datetime(df_sales["created_at"]).dt.time - df_sales["last_updated_date"] = pd.to_datetime(df_sales["last_updated"]).dt.date - df_sales["last_updated_time"] = pd.to_datetime(df_sales["last_updated"]).dt.time - fact_sales_order = df_sales.loc[ - :, - [ - "sales_record_id", - "sales_order_id", - "created_date", - "created_time", - "last_updated_date", - "last_updated_time", - "sales_staff_id", - "counterparty_id", - "units_sold", - "unit_price", - "currency_id", - "design_id", - "agreed_payment_date", - "agreed_delivery_date", - "agreed_delivery_location_id", - ], - ] - return fact_sales_order - - -# fact_purchase_order from purchase_order - - + df_sales["created_date"] = pd.to_datetime(df_sales["created_at"],format='%Y-%m-%d') + df_sales["created_time"] = pd.to_datetime(df_sales["created_at"],format='%H-%M-%S') + df_sales["last_updated_date"] = pd.to_datetime(df_sales["last_updated"],format='%Y-%m-%d') + df_sales["last_updated_time"] = pd.to_datetime(df_sales["last_updated"],format='%H-%M-%S') + df_sales['agreed_delivery_date'] = pd.to_datetime(df_sales['agreed_delivery_date'],format="%Y-%m-%d") + df_sales['agreed_payment_date'] = pd.to_datetime(df_sales['agreed_payment_date'],format="%Y-%m-%d") + df_sales.drop(labels=['created_at','last_updated'],axis=1,inplace=True) + df_sales.reset_index(inplace=True) + return df_sales + +#no test, same as fact_payment def create_fact_purchase_orders(dict_of_df): - df_po = dict_of_df["purchase_order"] - df_po.index.name = "purchase_record_id" - df_po["created_date"] = df_po["created_at"].date() - df_po["created_time"] = df_po["created_at"].dt.time - df_po["last_updated_date"] = df_po["last_updated_at"].date() - df_po["last_updated_time"] = df_po["last_updated_at"].dt.time - df_po["agreed_delivery_date"] = pd.to_datetime( - df_po["agreed_delivery_date"], format="%Y-%m-%d" - ) - df_po["agreed_payment_date"] = pd.to_datetime( - df_po["agreed_payment_date"], format="%Y-%m-%d" - ) - df_po.drop(labels=["created_at", "last_updated_at"], axis=1, inplace=True) + df_po = dict_of_df['purchase_order'] + df_po.index.name = 'purchase_record_id' + df_po['created_date'] = pd.to_datetime(df_po['created_at'],format='%Y-%m-%d') + df_po['created_time'] = pd.to_datetime(df_po['created_at'],format='%H-%M-%S') + df_po['last_updated_date'] = pd.to_datetime(df_po['last_updated'],format='%Y-%m-%d') + df_po['last_updated_time'] = pd.to_datetime(df_po['last_updated'],format='%H-%M-%S') + df_po['agreed_delivery_date'] = pd.to_datetime(df_po['agreed_delivery_date'],format="%Y-%m-%d") + df_po['agreed_payment_date'] = pd.to_datetime(df_po['agreed_payment_date'],format="%Y-%m-%d") + df_po.drop(labels=['created_at','last_updated'],axis=1,inplace=True) + df_po.reset_index(inplace=True) return df_po - +#test passed def create_fact_payment(dict_of_df): df_payment = dict_of_df["payment"] df_payment.index.name = "payment_record_id" - df_payment["created_date"] = df_payment["created_at"].date() - df_payment["created_time"] = df_payment["created_at"].time - df_payment["last_updated_date"] = df_payment["last_updated"].date() - df_payment["last_updated_time"] = df_payment["last_updated"].time - df_payment["payment_date"] = pd.to_datetime( - df_payment["payment_date"], format="%Y-%m-%d" - ) - fact_payment = df_payment.loc[ - :, - [ - "payment_record_id", - "payment_id", - "created_date", - "created_time", - "last_updated_date", - "last_updated_time", - "transaction_id", - "counterparty_id", - "payment_amount", - "currency_id", - "payment_type_id", - "paid", - "payment_date", - ], - ] - return fact_payment - - -# test passed - - + df_payment["created_date"] = pd.to_datetime(df_payment["created_at"],format='%Y-%m-%d') + df_payment["created_time"] = pd.to_datetime(df_payment["created_at"],format='%H-%M-%S') + df_payment["last_updated_date"] = pd.to_datetime(df_payment["last_updated"],format='%Y-%m-%d') + df_payment["last_updated_time"] = pd.to_datetime(df_payment["last_updated"],format='%H-%M-%S') + df_payment['payment_date'] = pd.to_datetime(df_payment['payment_date'],format="%Y-%m-%d") + df_payment.drop(labels=['created_at','last_updated'],axis=1,inplace=True) + df_payment.reset_index(inplace=True) + return df_payment + +#test passed def create_dim_transaction(dict_of_df): - df_transaction = dict_of_df["transaction"].drop( - labels=["created_at", "last_updated"], axis=1 - ) + df_transaction = dict_of_df["transaction"].drop(labels=['created_at', 'last_updated'], axis=1) return df_transaction - -# test passed +#test passed def create_dim_location(dict_of_df): - df_loc = ( - dict_of_df["address"] - .drop(labels=["created_at", "last_updated"], axis=1) - .rename(columns={"address_id": "location_id"}) - ) + df_loc = dict_of_df['address'].drop(labels=['created_at', 'last_updated'], axis=1).rename(columns={'address_id': 'location_id'}) return df_loc def create_dim_counterparty(dict_of_df): - df_prefixed_address = dict_of_df["address"].add_prefix( - "counterparty_legal_", axis=1 - ) - df_cp = pd.merge( - dict_of_df["counterparty"], - df_prefixed_address, - left_on="legal_address_id", - right_on="counterparty_legal_address_id", - how="outer", - ) - df_cp.drop( - columns=["legal_address_id", "counterparty_legal_address_id"], inplace=True - ) + df_prefixed_address = dict_of_df['address'].add_prefix('counterparty_legal_', axis=1) + df_cp = pd.merge(dict_of_df['counterparty'], + df_prefixed_address, + left_on="legal_address_id", + right_on="counterparty_legal_address_id", + how="outer") + df_cp.drop(columns=["legal_address_id","counterparty_legal_address_id"],inplace=True) return df_cp - -# test passed - - +#test passed def create_dim_date(dict_of_df): - fact_dfs = [ - create_fact_payment(dict_of_df), - create_fact_purchase_orders(dict_of_df), - create_fact_sales_order(dict_of_df), - ] - date_col_names = [ - col_name for col_name in list(fact_dfs[0].columns) if "date" in col_name - ] + fact_dfs = [create_fact_payment(dict_of_df), create_fact_purchase_orders(dict_of_df), create_fact_sales_order(dict_of_df)] list_of_date_columns = [] for df in fact_dfs: + date_col_names = [col_name for col_name in list(df.columns) if 'date' in col_name] for col in date_col_names: list_of_date_columns.append(df[col]) - sr_date = pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]") - df_date = pd.DataFrame(data=sr_date, columns=["date_id"]) + sr_date = pd.array(pd.concat(list_of_date_columns),dtype='datetime64[ns]') + df_date = pd.DataFrame(data=sr_date,columns=['date_id']) df_date.drop_duplicates(inplace=True) - df_date["year"] = df_date["date_id"].dt.year - df_date["month"] = df_date["date_id"].dt.month - df_date["day"] = df_date["date_id"].dt.day - df_date["day_of_week"] = df_date["date_id"].dt.dayofweek - df_date["day_name"] = df_date["date_id"].dt.day_name() - df_date["month_name"] = df_date["date_id"].dt.month_name() - df_date["quarter"] = df_date["date_id"].dt.quarter + df_date['year'] = df_date['date_id'].dt.year + df_date['month'] = df_date['date_id'].dt.month + df_date['day'] = df_date['date_id'].dt.day + df_date['day_of_week'] = df_date['date_id'].dt.dayofweek + df_date['day_name'] = df_date['date_id'].dt.day_name() + df_date['month_name'] = df_date['date_id'].dt.month_name() + df_date['quarter'] = df_date['date_id'].dt.quarter return df_date - -# tests passed +#tests passed def scrape_currency_names(): - response = requests.get("https://www.xe.com/currency/").content - soup = BeautifulSoup(response, "html.parser") - currency = [ - item.text for item in soup.findAll("a", attrs={"class": "sc-299dec64-6 fZPTSw"}) - ] + response = requests.get('https://www.xe.com/currency/').content + soup = BeautifulSoup(response,'html.parser') + currency = [item.text for item in soup.findAll('a', attrs={'class' : "sc-299dec64-6 fZPTSw"})] sr = pd.Series(currency) - df_cur = sr.str.split(pat=" - ", expand=True).rename( - {0: "currency_code", 1: "currency_name"}, axis=1 - ) + df_cur = sr.str.split(pat=" - ",expand=True).rename({0:'currency_code',1:'currency_name'},axis=1) return df_cur +#tests passed +def create_dim_currency(dict_of_df,names=scrape_currency_names()): + df_cur = dict_of_df['currency'].drop(labels=['created_at', 'last_updated'], axis=1) + dim_cur = pd.merge(df_cur,names,left_on='currency_code',right_on='currency_code',how='inner') + return dim_cur + +#tests passed +def create_dim_payment_type(dict_of_df): + df_payment_type = dict_of_df["payment_type"] + dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]] + return dim_payment_type + +#tests passed +def create_dim_design(dict_of_df): + df_design = dict_of_df["design"] + dim_design = df_design.loc[:, ["design_id", "design_name", "file_name", "file_location"]] + return dim_design +#tests passed +def create_dim_staff(dict_of_df): + staff_department = pd.merge(dict_of_df["staff"], dict_of_df["department"], on='department_id', how="left") + dim_staff = staff_department.loc[:, ['staff_id', 'first_name', 'last_name', 'department_name', 'location', 'email_address']] + return dim_staff + + -# tests passed -def create_dim_currency(dict_of_df, names=scrape_currency_names()): - df_cur = dict_of_df["currency"].drop(labels=["created_at", "last_updated"], axis=1) - dim_cur = pd.merge( - df_cur, names, left_on="currency_code", right_on="currency_code", how="inner" - ) - return dim_cur -# tests passed -def create_dim_payment_type(dict_of_df): - df_payment_type = dict_of_df["payment_type"] - dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]] - return dim_payment_type -# tests passed -def create_dim_design(dict_of_df): - df_design = dict_of_df["design"] - dim_design = df_design.loc[ - :, ["design_id", "design_name", "file_name", "file_location"] - ] - return dim_design -# tests passed -def create_dim_staff(dict_of_df): - staff_department = pd.merge( - dict_of_df["staff"], dict_of_df["department"], on="department_id", how="left" - ) - dim_staff = staff_department.loc[ - :, - [ - "staff_id", - "first_name", - "last_name", - "department_name", - "location", - "email_address", - ], - ] - return dim_staff diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py new file mode 100644 index 0000000..8f32b1d --- /dev/null +++ b/tests/test_dataframes.py @@ -0,0 +1,144 @@ +from src.dataframes import * +import pandas as pd +from unittest.mock import patch +from datetime import datetime as dt + +class TestCreateDimDesign: + def test_dim_design_returns_dataframe(self): + d = {"test": ["Hello", "Bye"], "design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], "file_location": ["Hello", "Bye"], "Hello": ["Hello", "Bye"]} + test_df = {"design": pd.DataFrame(data=d)} + result = create_dim_design(test_df) + assert isinstance(result, pd.DataFrame) + + def test_dim_design_returns_correct_columns_and_values(self): + d = {"test": ["Hello", "Bye"], "design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], "file_location": ["Hello", "Bye"], "Hello": ["Hello", "Bye"]} + test_df = {"design": pd.DataFrame(data=d)} + result = create_dim_design(test_df) + d2 = {"design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"]} + expected_df = pd.DataFrame(data=d2) + expected_result = expected_df.copy() + assert result.equals(expected_result) + +class TestCreateDimStaff: + def test_dim_staff_returns_dataframe(self): + d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + d2 = {"department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + result = create_dim_staff(test_df) + assert isinstance(result, pd.DataFrame) + + def test_dim_staff_returns_correct_columns_and_values(self): + d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + d2 = {"department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + result = create_dim_staff(test_df) + expected_d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"]} + expected_df = pd.DataFrame(data=expected_d) + expected_result = expected_df.copy() + assert result.equals(expected_result) + +class TestCreatePaymentType: + def test_create_dim_payment_type_returns_correct_columns_and_values(self): + d = {"payment_type_id": ["Hello", "Bye"], "payment_type_name": ["Hello", "Bye"]} + test_df = {"payment_type": pd.DataFrame(data=d)} + result = create_dim_payment_type(test_df) + expected_columns = ["payment_type_id", "payment_type_name"] + expected_d = {"payment_type_id": ["Hello", "Bye"], "payment_type_name": ["Hello", "Bye"]} + expected_df = pd.DataFrame(data=expected_d) + assert isinstance(result, pd.DataFrame) + assert list(result.columns) == expected_columns + assert result.equals(expected_df) + +class TestCreateDimCounterparty: + + def test_create_dim_counterparty_type_returns_correct_columns_and_object(self): + data_l = pd.DataFrame(data={"counterparty_id": ["Hello", "Bye"], + "counterparty_legal_name": ["Hello", "Bye"], + "commercial_contact": ["Hello", "Bye"], + "legal_address_id": ["bond street", "regent street"]}) + data_a = pd.DataFrame(data={"address_id":["bond street", "regent street"], + "postcode":[98365,93753]}) + test_df = {"address": data_a,"counterparty":data_l} + result = create_dim_counterparty(test_df) + + expected_columns = ["counterparty_id", "counterparty_legal_name", + "commercial_contact", "counterparty_legal_postcode"] + print(data_l) + print(data_a) + assert isinstance(result, pd.DataFrame) + assert list(result.columns) == expected_columns + +class TestCreateDimCurrency: + + def test_dim_currency_returns_columns_and_values(self): + nones = [None,None,None] + d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"],"created_at":nones,"last_updated":nones} + test_df = {"currency": pd.DataFrame(data=d)} + scraper_output = pd.DataFrame({"currency_code":["RUS","USD","PHP","GBP","EUR"],"currency_name":["Rubble","US Dollar","Peso","Pound","Euro"]}) + result = create_dim_currency(test_df,names=scraper_output).sort_values(by="currency_code",axis=0) + expected_d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"], "currency_name": ["US Dollar", "Euro", "Pound"]} + expected_df = pd.DataFrame(data=expected_d).sort_values(by="currency_code",axis=0) + assert isinstance(result, pd.DataFrame) + assert result.equals(expected_df) + + def test_scrape_currency_names_returns_dataframe_with_correct_collumns(self): + result = scrape_currency_names() + assert isinstance(result,pd.DataFrame) + assert list(result.columns) == ['currency_code', 'currency_name'] + +class TestCreateDimDate: + + def test_returns_required_columns(self): + df_one = pd.DataFrame(data={'updated_date':dt(2020, 5, 17),'created_date':dt(2021, 5, 13),'not_dat':None},index=[0]) + df_two = pd.DataFrame(data={'updated_date':dt(2020, 5, 17),'created_date':dt(2021, 9, 13)},index=[0]) + df_three = pd.DataFrame(data={'updated_date':dt(2022, 5, 17),'created_date':dt(2023, 5, 13)},index=[0]) + expected_df = pd.DataFrame(data= + [[dt(2020,5,17),2020,5,17,6,'Sunday','May',2], + [dt(2021,5,13),2021,5,13,3,'Thursday','May',2], + [dt(2021,9,13),2021,9,13,0,'Monday','September',3], + [dt(2022,5,17),2022,5,17,1,'Tuesday','May',2], + [dt(2023,5,13),2023,5,13,5,'Saturday','May',2]], + columns=['date_id','year','month','day','day_of_week','day_name','month_name','quarter']) + with patch("src.dataframes.create_fact_payment") as mock_fp: + with patch("src.dataframes.create_fact_purchase_orders") as mock_fpo: + with patch("src.dataframes.create_fact_sales_order") as mock_fso: + mock_fp.return_value = df_one + mock_fpo.return_value = df_two + mock_fso.return_value = df_three + result = create_dim_date({'dum':0}) + result.reset_index(inplace=True,drop=True) + assert result.eq(expected_df, axis="columns").all(axis=None) + +class TestCreateDimLocation: + + def test_returns_correct_columns_lo(self): + dict_df = {'address':pd.DataFrame(data=[['some_time','some_other_time',1,'SE18 9QO']], + columns=['created_at','last_updated','address_id','postal_code'])} + result = create_dim_location(dict_df) + assert list(result.columns) == ['location_id','postal_code'] + +class TestCreateDimTransaction: + def test_returns_correct_columns_tr(self): + dict_df = {'transaction':pd.DataFrame(data=[['some_time','some_other_time',1,'SE18 9QO']], + columns=['created_at','last_updated','transaction_id','some_other_id'])} + result = create_dim_transaction(dict_df) + assert list(result.columns) == ['transaction_id','some_other_id'] + +class TestCreateFactPayment: + def test_returns_correct_columns_payment(self): + dict_df = {'payment':pd.DataFrame(data=[[dt(2020,5,17,6,15,20),dt(2020,5,20,8,19,30),1,'SE18 9QO','2020-7-16']], + columns=['created_at','last_updated','payment_id','some_other_id','payment_date'])} + expected_cols = ['payment_record_id','created_date','created_time','last_updated_date', + 'last_updated_time','payment_date','payment_id','some_other_id'] + result = create_fact_payment(dict_df) + assert isinstance(result,pd.DataFrame) + for col in list(result.columns): + assert col in expected_cols + for col in expected_cols: + if 'date' in col: + assert result[col].dtype == 'datetime64[ns]' + + \ No newline at end of file diff --git a/tests/test_fact_sales_order.py b/tests/test_fact_sales_order.py deleted file mode 100644 index a245379..0000000 --- a/tests/test_fact_sales_order.py +++ /dev/null @@ -1,246 +0,0 @@ -from src.dataframes import * -import pandas as pd -from unittest.mock import patch -from datetime import datetime as dt - - -class TestCreateDimDesign: - def test_dim_design_returns_dataframe(self): - d = { - "test": ["Hello", "Bye"], - "design_id": ["Hello", "Bye"], - "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], - "file_location": ["Hello", "Bye"], - "Hello": ["Hello", "Bye"], - } - test_df = {"design": pd.DataFrame(data=d)} - result = create_dim_design(test_df) - assert isinstance(result, pd.DataFrame) - - def test_dim_design_returns_correct_columns_and_values(self): - d = { - "test": ["Hello", "Bye"], - "design_id": ["Hello", "Bye"], - "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], - "file_location": ["Hello", "Bye"], - "Hello": ["Hello", "Bye"], - } - test_df = {"design": pd.DataFrame(data=d)} - result = create_dim_design(test_df) - d2 = { - "design_id": ["Hello", "Bye"], - "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], - "file_location": ["Hello", "Bye"], - } - expected_df = pd.DataFrame(data=d2) - expected_result = expected_df.copy() - assert result.equals(expected_result) - - -class TestCreateDimStaff: - def test_dim_staff_returns_dataframe(self): - d = { - "staff_id": ["Hello", "Bye"], - "first_name": ["Hello", "Bye"], - "last_name": ["Hello", "Bye"], - "department_id": ["Hello", "Bye"], - } - d2 = { - "department_name": ["Hello", "Bye"], - "location": ["Hello", "Bye"], - "email_address": ["Hello", "Bye"], - "department_id": ["Hello", "Bye"], - } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} - result = create_dim_staff(test_df) - assert isinstance(result, pd.DataFrame) - - def test_dim_staff_returns_correct_columns_and_values(self): - d = { - "staff_id": ["Hello", "Bye"], - "first_name": ["Hello", "Bye"], - "last_name": ["Hello", "Bye"], - "department_id": ["Hello", "Bye"], - } - d2 = { - "department_name": ["Hello", "Bye"], - "location": ["Hello", "Bye"], - "email_address": ["Hello", "Bye"], - "department_id": ["Hello", "Bye"], - } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} - result = create_dim_staff(test_df) - expected_d = { - "staff_id": ["Hello", "Bye"], - "first_name": ["Hello", "Bye"], - "last_name": ["Hello", "Bye"], - "department_name": ["Hello", "Bye"], - "location": ["Hello", "Bye"], - "email_address": ["Hello", "Bye"], - } - expected_df = pd.DataFrame(data=expected_d) - expected_result = expected_df.copy() - assert result.equals(expected_result) - - -class TestCreatePaymentType: - def test_create_dim_payment_type_returns_correct_columns_and_values(self): - d = {"payment_type_id": ["Hello", "Bye"], "payment_type_name": ["Hello", "Bye"]} - test_df = {"payment_type": pd.DataFrame(data=d)} - result = create_dim_payment_type(test_df) - expected_columns = ["payment_type_id", "payment_type_name"] - expected_d = { - "payment_type_id": ["Hello", "Bye"], - "payment_type_name": ["Hello", "Bye"], - } - expected_df = pd.DataFrame(data=expected_d) - assert isinstance(result, pd.DataFrame) - assert list(result.columns) == expected_columns - assert result.equals(expected_df) - - -class TestCreateDimCounterparty: - def test_create_dim_counterparty_type_returns_correct_columns_and_object(self): - data_l = pd.DataFrame( - data={ - "counterparty_id": ["Hello", "Bye"], - "counterparty_legal_name": ["Hello", "Bye"], - "commercial_contact": ["Hello", "Bye"], - "legal_address_id": ["bond street", "regent street"], - } - ) - data_a = pd.DataFrame( - data={ - "address_id": ["bond street", "regent street"], - "postcode": [98365, 93753], - } - ) - test_df = {"address": data_a, "counterparty": data_l} - result = create_dim_counterparty(test_df) - - expected_columns = [ - "counterparty_id", - "counterparty_legal_name", - "commercial_contact", - "counterparty_legal_postcode", - ] - print(data_l) - print(data_a) - assert isinstance(result, pd.DataFrame) - assert list(result.columns) == expected_columns - - -class TestCreateDimCurrency: - def test_dim_currency_returns_columns_and_values(self): - nones = [None, None, None] - d = { - "currency_id": [1, 2, 3], - "currency_code": ["USD", "EUR", "GBP"], - "created_at": nones, - "last_updated": nones, - } - test_df = {"currency": pd.DataFrame(data=d)} - scraper_output = pd.DataFrame( - { - "currency_code": ["RUS", "USD", "PHP", "GBP", "EUR"], - "currency_name": ["Rubble", "US Dollar", "Peso", "Pound", "Euro"], - } - ) - result = create_dim_currency(test_df, names=scraper_output).sort_values( - by="currency_code", axis=0 - ) - expected_d = { - "currency_id": [1, 2, 3], - "currency_code": ["USD", "EUR", "GBP"], - "currency_name": ["US Dollar", "Euro", "Pound"], - } - expected_df = pd.DataFrame(data=expected_d).sort_values( - by="currency_code", axis=0 - ) - assert isinstance(result, pd.DataFrame) - assert result.equals(expected_df) - - def test_scrape_currency_names_returns_dataframe_with_correct_collumns(self): - result = scrape_currency_names() - assert isinstance(result, pd.DataFrame) - assert list(result.columns) == ["currency_code", "currency_name"] - - -class TestCreateDimDate: - def test_returns_required_columns(self): - df_one = pd.DataFrame( - data={ - "updated_date": dt(2020, 5, 17), - "created_date": dt(2021, 5, 13), - "not_dat": None, - }, - index=[0], - ) - df_two = pd.DataFrame( - data={"updated_date": dt(2020, 5, 17), "created_date": dt(2021, 9, 13)}, - index=[0], - ) - df_three = pd.DataFrame( - data={"updated_date": dt(2022, 5, 17), "created_date": dt(2023, 5, 13)}, - index=[0], - ) - expected_df = pd.DataFrame( - data=[ - [dt(2020, 5, 17), 2020, 5, 17, 6, "Sunday", "May", 2], - [dt(2021, 5, 13), 2021, 5, 13, 3, "Thursday", "May", 2], - [dt(2021, 9, 13), 2021, 9, 13, 0, "Monday", "September", 3], - [dt(2022, 5, 17), 2022, 5, 17, 1, "Tuesday", "May", 2], - [dt(2023, 5, 13), 2023, 5, 13, 5, "Saturday", "May", 2], - ], - columns=[ - "date_id", - "year", - "month", - "day", - "day_of_week", - "day_name", - "month_name", - "quarter", - ], - ) - with patch("src.dataframes.create_fact_payment") as mock_fp: - with patch("src.dataframes.create_fact_purchase_orders") as mock_fpo: - with patch("src.dataframes.create_fact_sales_order") as mock_fso: - mock_fp.return_value = df_one - mock_fpo.return_value = df_two - mock_fso.return_value = df_three - result = create_dim_date({"dum": 0}) - result.reset_index(inplace=True, drop=True) - assert result.eq(expected_df, axis="columns").all(axis=None) - - -class TestCreateDimLocation: - def test_returns_correct_columns_lo(self): - dict_df = { - "address": pd.DataFrame( - data=[["some_time", "some_other_time", 1, "SE18 9QO"]], - columns=["created_at", "last_updated", "address_id", "postal_code"], - ) - } - result = create_dim_location(dict_df) - assert list(result.columns) == ["location_id", "postal_code"] - - -class TestCreateDimTransaction: - def test_returns_correct_columns_tr(self): - dict_df = { - "transaction": pd.DataFrame( - data=[["some_time", "some_other_time", 1, "SE18 9QO"]], - columns=[ - "created_at", - "last_updated", - "transaction_id", - "some_other_id", - ], - ) - } - result = create_dim_transaction(dict_df) - assert list(result.columns) == ["transaction_id", "some_other_id"] -- cgit v1.2.3 From 8588d4b318d7732d33a59bc6c8b93870310668c5 Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 27 Aug 2024 15:18:54 +0100 Subject: test: refactored fact functions with test passing --- src/dataframes.py | 24 ++++++++++++------------ tests/test_dataframes.py | 9 +++++++-- 2 files changed, 19 insertions(+), 14 deletions(-) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index 41f39b8..1f445a4 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -20,10 +20,10 @@ import requests def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" - df_sales["created_date"] = pd.to_datetime(df_sales["created_at"],format='%Y-%m-%d') - df_sales["created_time"] = pd.to_datetime(df_sales["created_at"],format='%H-%M-%S') - df_sales["last_updated_date"] = pd.to_datetime(df_sales["last_updated"],format='%Y-%m-%d') - df_sales["last_updated_time"] = pd.to_datetime(df_sales["last_updated"],format='%H-%M-%S') + df_sales["created_date"] = pd.to_datetime(df_sales["created_at"].dt.date,format='%Y-%m-%d') + df_sales["created_time"] = df_sales["created_at"].dt.floor('s').dt.time + df_sales["last_updated_date"] = pd.to_datetime(df_sales["last_updated"].dt.date,format='%Y-%m-%d') + df_sales["last_updated_time"] = df_sales["last_updated"].dt.floor('s').dt.time df_sales['agreed_delivery_date'] = pd.to_datetime(df_sales['agreed_delivery_date'],format="%Y-%m-%d") df_sales['agreed_payment_date'] = pd.to_datetime(df_sales['agreed_payment_date'],format="%Y-%m-%d") df_sales.drop(labels=['created_at','last_updated'],axis=1,inplace=True) @@ -34,10 +34,10 @@ def create_fact_sales_order(dict_of_df): def create_fact_purchase_orders(dict_of_df): df_po = dict_of_df['purchase_order'] df_po.index.name = 'purchase_record_id' - df_po['created_date'] = pd.to_datetime(df_po['created_at'],format='%Y-%m-%d') - df_po['created_time'] = pd.to_datetime(df_po['created_at'],format='%H-%M-%S') - df_po['last_updated_date'] = pd.to_datetime(df_po['last_updated'],format='%Y-%m-%d') - df_po['last_updated_time'] = pd.to_datetime(df_po['last_updated'],format='%H-%M-%S') + df_po['created_date'] = pd.to_datetime(df_po['created_at'].dt.date,format='%Y-%m-%d') + df_po['created_time'] = df_po['created_at'].dt.floor('s').dt.time + df_po['last_updated_date'] = pd.to_datetime(df_po['last_updated'].dt.date,format='%Y-%m-%d') + df_po['last_updated_time'] = df_po['last_updated'].dt.floor('s').dt.time df_po['agreed_delivery_date'] = pd.to_datetime(df_po['agreed_delivery_date'],format="%Y-%m-%d") df_po['agreed_payment_date'] = pd.to_datetime(df_po['agreed_payment_date'],format="%Y-%m-%d") df_po.drop(labels=['created_at','last_updated'],axis=1,inplace=True) @@ -48,10 +48,10 @@ def create_fact_purchase_orders(dict_of_df): def create_fact_payment(dict_of_df): df_payment = dict_of_df["payment"] df_payment.index.name = "payment_record_id" - df_payment["created_date"] = pd.to_datetime(df_payment["created_at"],format='%Y-%m-%d') - df_payment["created_time"] = pd.to_datetime(df_payment["created_at"],format='%H-%M-%S') - df_payment["last_updated_date"] = pd.to_datetime(df_payment["last_updated"],format='%Y-%m-%d') - df_payment["last_updated_time"] = pd.to_datetime(df_payment["last_updated"],format='%H-%M-%S') + df_payment["created_date"] = pd.to_datetime(df_payment["created_at"].dt.date,format='%Y-%m-%d') + df_payment["created_time"] = df_payment["created_at"].dt.floor('s').dt.time + df_payment["last_updated_date"] = pd.to_datetime(df_payment["last_updated"].dt.date,format='%Y-%m-%d') + df_payment["last_updated_time"] = df_payment["last_updated"].dt.floor('s').dt.time df_payment['payment_date'] = pd.to_datetime(df_payment['payment_date'],format="%Y-%m-%d") df_payment.drop(labels=['created_at','last_updated'],axis=1,inplace=True) df_payment.reset_index(inplace=True) diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py index 8f32b1d..70aefe8 100644 --- a/tests/test_dataframes.py +++ b/tests/test_dataframes.py @@ -129,7 +129,8 @@ class TestCreateDimTransaction: class TestCreateFactPayment: def test_returns_correct_columns_payment(self): - dict_df = {'payment':pd.DataFrame(data=[[dt(2020,5,17,6,15,20),dt(2020,5,20,8,19,30),1,'SE18 9QO','2020-7-16']], + dict_df = {'payment':pd.DataFrame(data=[[dt.strptime('2022-11-03 14:20:49.962846','%Y-%m-%d %H:%M:%S.%f'), + dt.strptime('2022-12-14 16:20:49.962194','%Y-%m-%d %H:%M:%S.%f'),1,'SE18 9QO','2020-07-16']], columns=['created_at','last_updated','payment_id','some_other_id','payment_date'])} expected_cols = ['payment_record_id','created_date','created_time','last_updated_date', 'last_updated_time','payment_date','payment_id','some_other_id'] @@ -138,7 +139,11 @@ class TestCreateFactPayment: for col in list(result.columns): assert col in expected_cols for col in expected_cols: - if 'date' in col: + if '_date' in col: + print(col) assert result[col].dtype == 'datetime64[ns]' + if '_time' in col: + print(col) + assert result[col].dtype == 'O' #<< O for object \ No newline at end of file -- cgit v1.2.3 From efab1eccd4e2f0a8069ff4f1c968807a9c1ce05f Mon Sep 17 00:00:00 2001 From: Ang Bel Date: Tue, 27 Aug 2024 17:00:04 +0100 Subject: test: transform refactoring - it now loads parquet files into s3 bucket --- src/dataframes.py | 32 ++++++++++++++++---------------- src/transform_lambda.py | 6 +++--- tests/test_dataframes.py | 10 +++------- 3 files changed, 22 insertions(+), 26 deletions(-) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index 1f445a4..9d0f2ac 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -20,13 +20,13 @@ import requests def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" - df_sales["created_date"] = pd.to_datetime(df_sales["created_at"].dt.date,format='%Y-%m-%d') - df_sales["created_time"] = df_sales["created_at"].dt.floor('s').dt.time - df_sales["last_updated_date"] = pd.to_datetime(df_sales["last_updated"].dt.date,format='%Y-%m-%d') - df_sales["last_updated_time"] = df_sales["last_updated"].dt.floor('s').dt.time + df_sales["created_date"] = df_sales["created_at"].astype('datetime64[ns]').dt.date + df_sales["created_time"] = df_sales["created_at"].astype('datetime64[ns]').dt.floor('s').dt.time + df_sales["last_updated_date"] = df_sales["last_updated"].astype('datetime64[ns]').dt.date + df_sales["last_updated_time"] = df_sales["last_updated"].astype('datetime64[ns]').dt.floor('s').dt.time df_sales['agreed_delivery_date'] = pd.to_datetime(df_sales['agreed_delivery_date'],format="%Y-%m-%d") df_sales['agreed_payment_date'] = pd.to_datetime(df_sales['agreed_payment_date'],format="%Y-%m-%d") - df_sales.drop(labels=['created_at','last_updated'],axis=1,inplace=True) + df_sales = df_sales.drop(labels=['created_at','last_updated'],axis=1) df_sales.reset_index(inplace=True) return df_sales @@ -34,13 +34,13 @@ def create_fact_sales_order(dict_of_df): def create_fact_purchase_orders(dict_of_df): df_po = dict_of_df['purchase_order'] df_po.index.name = 'purchase_record_id' - df_po['created_date'] = pd.to_datetime(df_po['created_at'].dt.date,format='%Y-%m-%d') - df_po['created_time'] = df_po['created_at'].dt.floor('s').dt.time - df_po['last_updated_date'] = pd.to_datetime(df_po['last_updated'].dt.date,format='%Y-%m-%d') - df_po['last_updated_time'] = df_po['last_updated'].dt.floor('s').dt.time + df_po['created_date'] = df_po['created_at'].astype('datetime64[ns]').dt.date + df_po['created_time'] = df_po['created_at'].astype('datetime64[ns]').dt.floor('s').dt.time + df_po['last_updated_date'] = df_po['last_updated'].astype('datetime64[ns]').dt.date + df_po['last_updated_time'] = df_po['last_updated'].astype('datetime64[ns]').dt.floor('s').dt.time df_po['agreed_delivery_date'] = pd.to_datetime(df_po['agreed_delivery_date'],format="%Y-%m-%d") df_po['agreed_payment_date'] = pd.to_datetime(df_po['agreed_payment_date'],format="%Y-%m-%d") - df_po.drop(labels=['created_at','last_updated'],axis=1,inplace=True) + df_po = df_po.drop(labels=['created_at','last_updated'],axis=1) df_po.reset_index(inplace=True) return df_po @@ -48,12 +48,12 @@ def create_fact_purchase_orders(dict_of_df): def create_fact_payment(dict_of_df): df_payment = dict_of_df["payment"] df_payment.index.name = "payment_record_id" - df_payment["created_date"] = pd.to_datetime(df_payment["created_at"].dt.date,format='%Y-%m-%d') - df_payment["created_time"] = df_payment["created_at"].dt.floor('s').dt.time - df_payment["last_updated_date"] = pd.to_datetime(df_payment["last_updated"].dt.date,format='%Y-%m-%d') - df_payment["last_updated_time"] = df_payment["last_updated"].dt.floor('s').dt.time + df_payment["created_date"] = df_payment["created_at"].astype('datetime64[ns]').dt.date + df_payment["created_time"] = df_payment["created_at"].astype('datetime64[ns]').dt.floor('s').dt.time + df_payment["last_updated_date"] = df_payment["last_updated"].astype('datetime64[ns]').dt.date + df_payment["last_updated_time"] = df_payment["last_updated"].astype('datetime64[ns]').dt.floor('s').dt.time df_payment['payment_date'] = pd.to_datetime(df_payment['payment_date'],format="%Y-%m-%d") - df_payment.drop(labels=['created_at','last_updated'],axis=1,inplace=True) + df_payment = df_payment.drop(labels=['created_at','last_updated'],axis=1) df_payment.reset_index(inplace=True) return df_payment @@ -83,7 +83,7 @@ def create_dim_date(dict_of_df): fact_dfs = [create_fact_payment(dict_of_df), create_fact_purchase_orders(dict_of_df), create_fact_sales_order(dict_of_df)] list_of_date_columns = [] for df in fact_dfs: - date_col_names = [col_name for col_name in list(df.columns) if 'date' in col_name] + date_col_names = [col_name for col_name in list(df.columns) if '_date' in col_name] for col in date_col_names: list_of_date_columns.append(df[col]) sr_date = pd.array(pd.concat(list_of_date_columns),dtype='datetime64[ns]') diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 2cd9272..ccf90e5 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -117,7 +117,7 @@ def process_to_parquet_and_upload_to_s3( parquet_file = df.to_parquet( f"{table_name}.parquet", engine="pyarrow" ) # or fastparquet - client.upload_file(parquet_file, bucket, f"{table_name}.parquet") + client.upload_file(f"{table_name}.parquet", bucket, f"{table_name}.parquet") #changed parquet_file variable to the file name status["uploaded"].append(table_name) for table_name, df in mutable_df_dict.items(): @@ -127,7 +127,7 @@ def process_to_parquet_and_upload_to_s3( parquet_file = df.to_parquet( f"{table_name}.parquet", engine="pyarrow" ) # or fastparquet - client.upload_file(parquet_file, bucket, s3_key) + client.upload_file(f"{table_name}.parquet", bucket, s3_key) status["uploaded"].append(table_name) return status @@ -203,7 +203,7 @@ def list_existing_s3_files(bucket_name, client=boto3.client("s3")): existing_files = [obj["Key"] for obj in response["Contents"]] else: logger.error("The bucket is empty") - return None + return [] #changed from None to [] so it is an iterable except ClientError as e: logger.error(f"Error listing S3 objects: {e}") diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py index 70aefe8..adbb5ed 100644 --- a/tests/test_dataframes.py +++ b/tests/test_dataframes.py @@ -139,11 +139,7 @@ class TestCreateFactPayment: for col in list(result.columns): assert col in expected_cols for col in expected_cols: - if '_date' in col: - print(col) - assert result[col].dtype == 'datetime64[ns]' - if '_time' in col: - print(col) - assert result[col].dtype == 'O' #<< O for object - + if '_date' or '_time' in col: + assert result[col].dtype == 'O' + \ No newline at end of file -- cgit v1.2.3 From 26902dc234c114382c2926923820c3537490c30e Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 16:01:11 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 1a145a3 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/101 --- src/dataframes.py | 236 ++++++++++++++++++++++++++------------- src/transform_lambda.py | 5 +- tests/test_dataframes.py | 282 +++++++++++++++++++++++++++++++++++------------ 3 files changed, 375 insertions(+), 148 deletions(-) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index 9d0f2ac..f122368 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -2,7 +2,7 @@ import pandas as pd from bs4 import BeautifulSoup import requests -#Table names: +# Table names: # fact_sales_order # fact_purchase_orders # fact_payment @@ -16,133 +16,213 @@ import requests # dim_counterparty -#no test, same as fact_payment +# no test, same as fact_payment def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" - df_sales["created_date"] = df_sales["created_at"].astype('datetime64[ns]').dt.date - df_sales["created_time"] = df_sales["created_at"].astype('datetime64[ns]').dt.floor('s').dt.time - df_sales["last_updated_date"] = df_sales["last_updated"].astype('datetime64[ns]').dt.date - df_sales["last_updated_time"] = df_sales["last_updated"].astype('datetime64[ns]').dt.floor('s').dt.time - df_sales['agreed_delivery_date'] = pd.to_datetime(df_sales['agreed_delivery_date'],format="%Y-%m-%d") - df_sales['agreed_payment_date'] = pd.to_datetime(df_sales['agreed_payment_date'],format="%Y-%m-%d") - df_sales = df_sales.drop(labels=['created_at','last_updated'],axis=1) + df_sales["created_date"] = df_sales["created_at"].astype("datetime64[ns]").dt.date + df_sales["created_time"] = ( + df_sales["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_sales["last_updated_date"] = ( + df_sales["last_updated"].astype("datetime64[ns]").dt.date + ) + df_sales["last_updated_time"] = ( + df_sales["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_sales["agreed_delivery_date"] = pd.to_datetime( + df_sales["agreed_delivery_date"], format="%Y-%m-%d" + ) + df_sales["agreed_payment_date"] = pd.to_datetime( + df_sales["agreed_payment_date"], format="%Y-%m-%d" + ) + df_sales = df_sales.drop(labels=["created_at", "last_updated"], axis=1) df_sales.reset_index(inplace=True) return df_sales -#no test, same as fact_payment + +# no test, same as fact_payment + + def create_fact_purchase_orders(dict_of_df): - df_po = dict_of_df['purchase_order'] - df_po.index.name = 'purchase_record_id' - df_po['created_date'] = df_po['created_at'].astype('datetime64[ns]').dt.date - df_po['created_time'] = df_po['created_at'].astype('datetime64[ns]').dt.floor('s').dt.time - df_po['last_updated_date'] = df_po['last_updated'].astype('datetime64[ns]').dt.date - df_po['last_updated_time'] = df_po['last_updated'].astype('datetime64[ns]').dt.floor('s').dt.time - df_po['agreed_delivery_date'] = pd.to_datetime(df_po['agreed_delivery_date'],format="%Y-%m-%d") - df_po['agreed_payment_date'] = pd.to_datetime(df_po['agreed_payment_date'],format="%Y-%m-%d") - df_po = df_po.drop(labels=['created_at','last_updated'],axis=1) + df_po = dict_of_df["purchase_order"] + df_po.index.name = "purchase_record_id" + df_po["created_date"] = df_po["created_at"].astype("datetime64[ns]").dt.date + df_po["created_time"] = ( + df_po["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_po["last_updated_date"] = df_po["last_updated"].astype("datetime64[ns]").dt.date + df_po["last_updated_time"] = ( + df_po["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_po["agreed_delivery_date"] = pd.to_datetime( + df_po["agreed_delivery_date"], format="%Y-%m-%d" + ) + df_po["agreed_payment_date"] = pd.to_datetime( + df_po["agreed_payment_date"], format="%Y-%m-%d" + ) + df_po = df_po.drop(labels=["created_at", "last_updated"], axis=1) df_po.reset_index(inplace=True) return df_po -#test passed + +# test passed + + def create_fact_payment(dict_of_df): df_payment = dict_of_df["payment"] df_payment.index.name = "payment_record_id" - df_payment["created_date"] = df_payment["created_at"].astype('datetime64[ns]').dt.date - df_payment["created_time"] = df_payment["created_at"].astype('datetime64[ns]').dt.floor('s').dt.time - df_payment["last_updated_date"] = df_payment["last_updated"].astype('datetime64[ns]').dt.date - df_payment["last_updated_time"] = df_payment["last_updated"].astype('datetime64[ns]').dt.floor('s').dt.time - df_payment['payment_date'] = pd.to_datetime(df_payment['payment_date'],format="%Y-%m-%d") - df_payment = df_payment.drop(labels=['created_at','last_updated'],axis=1) + df_payment["created_date"] = ( + df_payment["created_at"].astype("datetime64[ns]").dt.date + ) + df_payment["created_time"] = ( + df_payment["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_payment["last_updated_date"] = ( + df_payment["last_updated"].astype("datetime64[ns]").dt.date + ) + df_payment["last_updated_time"] = ( + df_payment["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_payment["payment_date"] = pd.to_datetime( + df_payment["payment_date"], format="%Y-%m-%d" + ) + df_payment = df_payment.drop(labels=["created_at", "last_updated"], axis=1) df_payment.reset_index(inplace=True) return df_payment -#test passed + +# test passed + + def create_dim_transaction(dict_of_df): - df_transaction = dict_of_df["transaction"].drop(labels=['created_at', 'last_updated'], axis=1) + df_transaction = dict_of_df["transaction"].drop( + labels=["created_at", "last_updated"], axis=1 + ) return df_transaction -#test passed + +# test passed + + def create_dim_location(dict_of_df): - df_loc = dict_of_df['address'].drop(labels=['created_at', 'last_updated'], axis=1).rename(columns={'address_id': 'location_id'}) + df_loc = ( + dict_of_df["address"] + .drop(labels=["created_at", "last_updated"], axis=1) + .rename(columns={"address_id": "location_id"}) + ) return df_loc def create_dim_counterparty(dict_of_df): - df_prefixed_address = dict_of_df['address'].add_prefix('counterparty_legal_', axis=1) - df_cp = pd.merge(dict_of_df['counterparty'], - df_prefixed_address, - left_on="legal_address_id", - right_on="counterparty_legal_address_id", - how="outer") - df_cp.drop(columns=["legal_address_id","counterparty_legal_address_id"],inplace=True) + df_prefixed_address = dict_of_df["address"].add_prefix( + "counterparty_legal_", axis=1 + ) + df_cp = pd.merge( + dict_of_df["counterparty"], + df_prefixed_address, + left_on="legal_address_id", + right_on="counterparty_legal_address_id", + how="outer", + ) + df_cp.drop( + columns=["legal_address_id", "counterparty_legal_address_id"], inplace=True + ) return df_cp -#test passed + +# test passed + + def create_dim_date(dict_of_df): - fact_dfs = [create_fact_payment(dict_of_df), create_fact_purchase_orders(dict_of_df), create_fact_sales_order(dict_of_df)] + fact_dfs = [ + create_fact_payment(dict_of_df), + create_fact_purchase_orders(dict_of_df), + create_fact_sales_order(dict_of_df), + ] list_of_date_columns = [] for df in fact_dfs: - date_col_names = [col_name for col_name in list(df.columns) if '_date' in col_name] + date_col_names = [ + col_name for col_name in list(df.columns) if "_date" in col_name + ] for col in date_col_names: list_of_date_columns.append(df[col]) - sr_date = pd.array(pd.concat(list_of_date_columns),dtype='datetime64[ns]') - df_date = pd.DataFrame(data=sr_date,columns=['date_id']) + sr_date = pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]") + df_date = pd.DataFrame(data=sr_date, columns=["date_id"]) df_date.drop_duplicates(inplace=True) - df_date['year'] = df_date['date_id'].dt.year - df_date['month'] = df_date['date_id'].dt.month - df_date['day'] = df_date['date_id'].dt.day - df_date['day_of_week'] = df_date['date_id'].dt.dayofweek - df_date['day_name'] = df_date['date_id'].dt.day_name() - df_date['month_name'] = df_date['date_id'].dt.month_name() - df_date['quarter'] = df_date['date_id'].dt.quarter + df_date["year"] = df_date["date_id"].dt.year + df_date["month"] = df_date["date_id"].dt.month + df_date["day"] = df_date["date_id"].dt.day + df_date["day_of_week"] = df_date["date_id"].dt.dayofweek + df_date["day_name"] = df_date["date_id"].dt.day_name() + df_date["month_name"] = df_date["date_id"].dt.month_name() + df_date["quarter"] = df_date["date_id"].dt.quarter return df_date -#tests passed -def scrape_currency_names(): - response = requests.get('https://www.xe.com/currency/').content - soup = BeautifulSoup(response,'html.parser') - currency = [item.text for item in soup.findAll('a', attrs={'class' : "sc-299dec64-6 fZPTSw"})] - sr = pd.Series(currency) - df_cur = sr.str.split(pat=" - ",expand=True).rename({0:'currency_code',1:'currency_name'},axis=1) - return df_cur - -#tests passed -def create_dim_currency(dict_of_df,names=scrape_currency_names()): - df_cur = dict_of_df['currency'].drop(labels=['created_at', 'last_updated'], axis=1) - dim_cur = pd.merge(df_cur,names,left_on='currency_code',right_on='currency_code',how='inner') - return dim_cur -#tests passed -def create_dim_payment_type(dict_of_df): - df_payment_type = dict_of_df["payment_type"] - dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]] - return dim_payment_type +# tests passed -#tests passed -def create_dim_design(dict_of_df): - df_design = dict_of_df["design"] - dim_design = df_design.loc[:, ["design_id", "design_name", "file_name", "file_location"]] - return dim_design -#tests passed -def create_dim_staff(dict_of_df): - staff_department = pd.merge(dict_of_df["staff"], dict_of_df["department"], on='department_id', how="left") - dim_staff = staff_department.loc[:, ['staff_id', 'first_name', 'last_name', 'department_name', 'location', 'email_address']] - return dim_staff +def scrape_currency_names(): + response = requests.get("https://www.xe.com/currency/").content + soup = BeautifulSoup(response, "html.parser") + currency = [ + item.text for item in soup.findAll("a", attrs={"class": "sc-299dec64-6 fZPTSw"}) + ] + sr = pd.Series(currency) + df_cur = sr.str.split(pat=" - ", expand=True).rename( + {0: "currency_code", 1: "currency_name"}, axis=1 + ) + return df_cur +# tests passed +def create_dim_currency(dict_of_df, names=scrape_currency_names()): + df_cur = dict_of_df["currency"].drop(labels=["created_at", "last_updated"], axis=1) + dim_cur = pd.merge( + df_cur, names, left_on="currency_code", right_on="currency_code", how="inner" + ) + return dim_cur +# tests passed +def create_dim_payment_type(dict_of_df): + df_payment_type = dict_of_df["payment_type"] + dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]] + return dim_payment_type +# tests passed +def create_dim_design(dict_of_df): + df_design = dict_of_df["design"] + dim_design = df_design.loc[ + :, ["design_id", "design_name", "file_name", "file_location"] + ] + return dim_design +# tests passed +def create_dim_staff(dict_of_df): + staff_department = pd.merge( + dict_of_df["staff"], dict_of_df["department"], on="department_id", how="left" + ) + dim_staff = staff_department.loc[ + :, + [ + "staff_id", + "first_name", + "last_name", + "department_name", + "location", + "email_address", + ], + ] + return dim_staff diff --git a/src/transform_lambda.py b/src/transform_lambda.py index ccf90e5..93b2284 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -117,7 +117,8 @@ def process_to_parquet_and_upload_to_s3( parquet_file = df.to_parquet( f"{table_name}.parquet", engine="pyarrow" ) # or fastparquet - client.upload_file(f"{table_name}.parquet", bucket, f"{table_name}.parquet") #changed parquet_file variable to the file name + # changed parquet_file variable to the file name + client.upload_file(f"{table_name}.parquet", bucket, f"{table_name}.parquet") status["uploaded"].append(table_name) for table_name, df in mutable_df_dict.items(): @@ -203,7 +204,7 @@ def list_existing_s3_files(bucket_name, client=boto3.client("s3")): existing_files = [obj["Key"] for obj in response["Contents"]] else: logger.error("The bucket is empty") - return [] #changed from None to [] so it is an iterable + return [] # changed from None to [] so it is an iterable except ClientError as e: logger.error(f"Error listing S3 objects: {e}") diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py index adbb5ed..c9ff43f 100644 --- a/tests/test_dataframes.py +++ b/tests/test_dataframes.py @@ -3,42 +3,88 @@ import pandas as pd from unittest.mock import patch from datetime import datetime as dt + class TestCreateDimDesign: def test_dim_design_returns_dataframe(self): - d = {"test": ["Hello", "Bye"], "design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], "file_location": ["Hello", "Bye"], "Hello": ["Hello", "Bye"]} + d = { + "test": ["Hello", "Bye"], + "design_id": ["Hello", "Bye"], + "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"], + "Hello": ["Hello", "Bye"], + } test_df = {"design": pd.DataFrame(data=d)} result = create_dim_design(test_df) assert isinstance(result, pd.DataFrame) def test_dim_design_returns_correct_columns_and_values(self): - d = {"test": ["Hello", "Bye"], "design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], - "file_name": ["Hello", "Bye"], "file_location": ["Hello", "Bye"], "Hello": ["Hello", "Bye"]} + d = { + "test": ["Hello", "Bye"], + "design_id": ["Hello", "Bye"], + "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"], + "Hello": ["Hello", "Bye"], + } test_df = {"design": pd.DataFrame(data=d)} result = create_dim_design(test_df) - d2 = {"design_id": ["Hello", "Bye"], "design_name": ["Hello", "Bye"], "file_name": ["Hello", "Bye"], - "file_location": ["Hello", "Bye"]} + d2 = { + "design_id": ["Hello", "Bye"], + "design_name": ["Hello", "Bye"], + "file_name": ["Hello", "Bye"], + "file_location": ["Hello", "Bye"], + } expected_df = pd.DataFrame(data=d2) expected_result = expected_df.copy() assert result.equals(expected_result) + class TestCreateDimStaff: def test_dim_staff_returns_dataframe(self): - d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} - d2 = {"department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + d = { + "staff_id": ["Hello", "Bye"], + "first_name": ["Hello", "Bye"], + "last_name": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } + d2 = { + "department_name": ["Hello", "Bye"], + "location": ["Hello", "Bye"], + "email_address": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) - assert isinstance(result, pd.DataFrame) + assert isinstance(result, pd.DataFrame) def test_dim_staff_returns_correct_columns_and_values(self): - d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} - d2 = {"department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"]} + d = { + "staff_id": ["Hello", "Bye"], + "first_name": ["Hello", "Bye"], + "last_name": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } + d2 = { + "department_name": ["Hello", "Bye"], + "location": ["Hello", "Bye"], + "email_address": ["Hello", "Bye"], + "department_id": ["Hello", "Bye"], + } test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) - expected_d = {"staff_id": ["Hello", "Bye"], "first_name": ["Hello", "Bye"], "last_name": ["Hello", "Bye"], "department_name": ["Hello", "Bye"], "location": ["Hello", "Bye"], "email_address": ["Hello", "Bye"]} + expected_d = { + "staff_id": ["Hello", "Bye"], + "first_name": ["Hello", "Bye"], + "last_name": ["Hello", "Bye"], + "department_name": ["Hello", "Bye"], + "location": ["Hello", "Bye"], + "email_address": ["Hello", "Bye"], + } expected_df = pd.DataFrame(data=expected_d) expected_result = expected_df.copy() - assert result.equals(expected_result) + assert result.equals(expected_result) + class TestCreatePaymentType: def test_create_dim_payment_type_returns_correct_columns_and_values(self): @@ -46,100 +92,200 @@ class TestCreatePaymentType: test_df = {"payment_type": pd.DataFrame(data=d)} result = create_dim_payment_type(test_df) expected_columns = ["payment_type_id", "payment_type_name"] - expected_d = {"payment_type_id": ["Hello", "Bye"], "payment_type_name": ["Hello", "Bye"]} + expected_d = { + "payment_type_id": ["Hello", "Bye"], + "payment_type_name": ["Hello", "Bye"], + } expected_df = pd.DataFrame(data=expected_d) assert isinstance(result, pd.DataFrame) assert list(result.columns) == expected_columns assert result.equals(expected_df) + class TestCreateDimCounterparty: - def test_create_dim_counterparty_type_returns_correct_columns_and_object(self): - data_l = pd.DataFrame(data={"counterparty_id": ["Hello", "Bye"], - "counterparty_legal_name": ["Hello", "Bye"], - "commercial_contact": ["Hello", "Bye"], - "legal_address_id": ["bond street", "regent street"]}) - data_a = pd.DataFrame(data={"address_id":["bond street", "regent street"], - "postcode":[98365,93753]}) - test_df = {"address": data_a,"counterparty":data_l} + data_l = pd.DataFrame( + data={ + "counterparty_id": ["Hello", "Bye"], + "counterparty_legal_name": ["Hello", "Bye"], + "commercial_contact": ["Hello", "Bye"], + "legal_address_id": ["bond street", "regent street"], + } + ) + data_a = pd.DataFrame( + data={ + "address_id": ["bond street", "regent street"], + "postcode": [98365, 93753], + } + ) + test_df = {"address": data_a, "counterparty": data_l} result = create_dim_counterparty(test_df) - expected_columns = ["counterparty_id", "counterparty_legal_name", - "commercial_contact", "counterparty_legal_postcode"] + expected_columns = [ + "counterparty_id", + "counterparty_legal_name", + "commercial_contact", + "counterparty_legal_postcode", + ] print(data_l) print(data_a) assert isinstance(result, pd.DataFrame) assert list(result.columns) == expected_columns + class TestCreateDimCurrency: - def test_dim_currency_returns_columns_and_values(self): - nones = [None,None,None] - d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"],"created_at":nones,"last_updated":nones} + nones = [None, None, None] + d = { + "currency_id": [1, 2, 3], + "currency_code": ["USD", "EUR", "GBP"], + "created_at": nones, + "last_updated": nones, + } test_df = {"currency": pd.DataFrame(data=d)} - scraper_output = pd.DataFrame({"currency_code":["RUS","USD","PHP","GBP","EUR"],"currency_name":["Rubble","US Dollar","Peso","Pound","Euro"]}) - result = create_dim_currency(test_df,names=scraper_output).sort_values(by="currency_code",axis=0) - expected_d = {"currency_id": [1, 2, 3], "currency_code": ["USD", "EUR", "GBP"], "currency_name": ["US Dollar", "Euro", "Pound"]} - expected_df = pd.DataFrame(data=expected_d).sort_values(by="currency_code",axis=0) - assert isinstance(result, pd.DataFrame) - assert result.equals(expected_df) + scraper_output = pd.DataFrame( + { + "currency_code": ["RUS", "USD", "PHP", "GBP", "EUR"], + "currency_name": ["Rubble", "US Dollar", "Peso", "Pound", "Euro"], + } + ) + result = create_dim_currency(test_df, names=scraper_output).sort_values( + by="currency_code", axis=0 + ) + expected_d = { + "currency_id": [1, 2, 3], + "currency_code": ["USD", "EUR", "GBP"], + "currency_name": ["US Dollar", "Euro", "Pound"], + } + expected_df = pd.DataFrame(data=expected_d).sort_values( + by="currency_code", axis=0 + ) + assert isinstance(result, pd.DataFrame) + assert result.equals(expected_df) def test_scrape_currency_names_returns_dataframe_with_correct_collumns(self): result = scrape_currency_names() - assert isinstance(result,pd.DataFrame) - assert list(result.columns) == ['currency_code', 'currency_name'] + assert isinstance(result, pd.DataFrame) + assert list(result.columns) == ["currency_code", "currency_name"] -class TestCreateDimDate: +class TestCreateDimDate: def test_returns_required_columns(self): - df_one = pd.DataFrame(data={'updated_date':dt(2020, 5, 17),'created_date':dt(2021, 5, 13),'not_dat':None},index=[0]) - df_two = pd.DataFrame(data={'updated_date':dt(2020, 5, 17),'created_date':dt(2021, 9, 13)},index=[0]) - df_three = pd.DataFrame(data={'updated_date':dt(2022, 5, 17),'created_date':dt(2023, 5, 13)},index=[0]) - expected_df = pd.DataFrame(data= - [[dt(2020,5,17),2020,5,17,6,'Sunday','May',2], - [dt(2021,5,13),2021,5,13,3,'Thursday','May',2], - [dt(2021,9,13),2021,9,13,0,'Monday','September',3], - [dt(2022,5,17),2022,5,17,1,'Tuesday','May',2], - [dt(2023,5,13),2023,5,13,5,'Saturday','May',2]], - columns=['date_id','year','month','day','day_of_week','day_name','month_name','quarter']) + df_one = pd.DataFrame( + data={ + "updated_date": dt(2020, 5, 17), + "created_date": dt(2021, 5, 13), + "not_dat": None, + }, + index=[0], + ) + df_two = pd.DataFrame( + data={"updated_date": dt(2020, 5, 17), "created_date": dt(2021, 9, 13)}, + index=[0], + ) + df_three = pd.DataFrame( + data={"updated_date": dt(2022, 5, 17), "created_date": dt(2023, 5, 13)}, + index=[0], + ) + expected_df = pd.DataFrame( + data=[ + [dt(2020, 5, 17), 2020, 5, 17, 6, "Sunday", "May", 2], + [dt(2021, 5, 13), 2021, 5, 13, 3, "Thursday", "May", 2], + [dt(2021, 9, 13), 2021, 9, 13, 0, "Monday", "September", 3], + [dt(2022, 5, 17), 2022, 5, 17, 1, "Tuesday", "May", 2], + [dt(2023, 5, 13), 2023, 5, 13, 5, "Saturday", "May", 2], + ], + columns=[ + "date_id", + "year", + "month", + "day", + "day_of_week", + "day_name", + "month_name", + "quarter", + ], + ) with patch("src.dataframes.create_fact_payment") as mock_fp: with patch("src.dataframes.create_fact_purchase_orders") as mock_fpo: with patch("src.dataframes.create_fact_sales_order") as mock_fso: mock_fp.return_value = df_one mock_fpo.return_value = df_two mock_fso.return_value = df_three - result = create_dim_date({'dum':0}) - result.reset_index(inplace=True,drop=True) + result = create_dim_date({"dum": 0}) + result.reset_index(inplace=True, drop=True) assert result.eq(expected_df, axis="columns").all(axis=None) - -class TestCreateDimLocation: + +class TestCreateDimLocation: def test_returns_correct_columns_lo(self): - dict_df = {'address':pd.DataFrame(data=[['some_time','some_other_time',1,'SE18 9QO']], - columns=['created_at','last_updated','address_id','postal_code'])} + dict_df = { + "address": pd.DataFrame( + data=[["some_time", "some_other_time", 1, "SE18 9QO"]], + columns=["created_at", "last_updated", "address_id", "postal_code"], + ) + } result = create_dim_location(dict_df) - assert list(result.columns) == ['location_id','postal_code'] - + assert list(result.columns) == ["location_id", "postal_code"] + + class TestCreateDimTransaction: - def test_returns_correct_columns_tr(self): - dict_df = {'transaction':pd.DataFrame(data=[['some_time','some_other_time',1,'SE18 9QO']], - columns=['created_at','last_updated','transaction_id','some_other_id'])} + def test_returns_correct_columns_tr(self): + dict_df = { + "transaction": pd.DataFrame( + data=[["some_time", "some_other_time", 1, "SE18 9QO"]], + columns=[ + "created_at", + "last_updated", + "transaction_id", + "some_other_id", + ], + ) + } result = create_dim_transaction(dict_df) - assert list(result.columns) == ['transaction_id','some_other_id'] + assert list(result.columns) == ["transaction_id", "some_other_id"] + class TestCreateFactPayment: def test_returns_correct_columns_payment(self): - dict_df = {'payment':pd.DataFrame(data=[[dt.strptime('2022-11-03 14:20:49.962846','%Y-%m-%d %H:%M:%S.%f'), - dt.strptime('2022-12-14 16:20:49.962194','%Y-%m-%d %H:%M:%S.%f'),1,'SE18 9QO','2020-07-16']], - columns=['created_at','last_updated','payment_id','some_other_id','payment_date'])} - expected_cols = ['payment_record_id','created_date','created_time','last_updated_date', - 'last_updated_time','payment_date','payment_id','some_other_id'] + dict_df = { + "payment": pd.DataFrame( + data=[ + [ + dt.strptime( + "2022-11-03 14:20:49.962846", "%Y-%m-%d %H:%M:%S.%f" + ), + dt.strptime( + "2022-12-14 16:20:49.962194", "%Y-%m-%d %H:%M:%S.%f" + ), + 1, + "SE18 9QO", + "2020-07-16", + ] + ], + columns=[ + "created_at", + "last_updated", + "payment_id", + "some_other_id", + "payment_date", + ], + ) + } + expected_cols = [ + "payment_record_id", + "created_date", + "created_time", + "last_updated_date", + "last_updated_time", + "payment_date", + "payment_id", + "some_other_id", + ] result = create_fact_payment(dict_df) - assert isinstance(result,pd.DataFrame) + assert isinstance(result, pd.DataFrame) for col in list(result.columns): assert col in expected_cols for col in expected_cols: - if '_date' or '_time' in col: - assert result[col].dtype == 'O' - - \ No newline at end of file + if "_date" or "_time" in col: + assert result[col].dtype == "O" -- cgit v1.2.3 From f8988db9372802053db60e311960f5da4defba02 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Tue, 27 Aug 2024 11:44:00 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in a05a371 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/98 --- src/dataframes.py | 50 ++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_dataframes.py | 13 +++++++++++++ 2 files changed, 63 insertions(+) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index f122368..36361d2 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -20,6 +20,7 @@ import requests def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" +<<<<<<< HEAD df_sales["created_date"] = df_sales["created_at"].astype("datetime64[ns]").dt.date df_sales["created_time"] = ( df_sales["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time @@ -29,6 +30,15 @@ def create_fact_sales_order(dict_of_df): ) df_sales["last_updated_time"] = ( df_sales["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time +======= + df_sales["created_date"] = pd.to_datetime(df_sales["created_at"], format="%Y-%m-%d") + df_sales["created_time"] = pd.to_datetime(df_sales["created_at"], format="%H-%M-%S") + df_sales["last_updated_date"] = pd.to_datetime( + df_sales["last_updated"], format="%Y-%m-%d" + ) + df_sales["last_updated_time"] = pd.to_datetime( + df_sales["last_updated"], format="%H-%M-%S" +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) ) df_sales["agreed_delivery_date"] = pd.to_datetime( df_sales["agreed_delivery_date"], format="%Y-%m-%d" @@ -36,7 +46,11 @@ def create_fact_sales_order(dict_of_df): df_sales["agreed_payment_date"] = pd.to_datetime( df_sales["agreed_payment_date"], format="%Y-%m-%d" ) +<<<<<<< HEAD df_sales = df_sales.drop(labels=["created_at", "last_updated"], axis=1) +======= + df_sales.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) df_sales.reset_index(inplace=True) return df_sales @@ -47,6 +61,7 @@ def create_fact_sales_order(dict_of_df): def create_fact_purchase_orders(dict_of_df): df_po = dict_of_df["purchase_order"] df_po.index.name = "purchase_record_id" +<<<<<<< HEAD df_po["created_date"] = df_po["created_at"].astype("datetime64[ns]").dt.date df_po["created_time"] = ( df_po["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time @@ -54,6 +69,15 @@ def create_fact_purchase_orders(dict_of_df): df_po["last_updated_date"] = df_po["last_updated"].astype("datetime64[ns]").dt.date df_po["last_updated_time"] = ( df_po["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time +======= + df_po["created_date"] = pd.to_datetime(df_po["created_at"], format="%Y-%m-%d") + df_po["created_time"] = pd.to_datetime(df_po["created_at"], format="%H-%M-%S") + df_po["last_updated_date"] = pd.to_datetime( + df_po["last_updated"], format="%Y-%m-%d" + ) + df_po["last_updated_time"] = pd.to_datetime( + df_po["last_updated"], format="%H-%M-%S" +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) ) df_po["agreed_delivery_date"] = pd.to_datetime( df_po["agreed_delivery_date"], format="%Y-%m-%d" @@ -61,7 +85,11 @@ def create_fact_purchase_orders(dict_of_df): df_po["agreed_payment_date"] = pd.to_datetime( df_po["agreed_payment_date"], format="%Y-%m-%d" ) +<<<<<<< HEAD df_po = df_po.drop(labels=["created_at", "last_updated"], axis=1) +======= + df_po.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) df_po.reset_index(inplace=True) return df_po @@ -72,6 +100,7 @@ def create_fact_purchase_orders(dict_of_df): def create_fact_payment(dict_of_df): df_payment = dict_of_df["payment"] df_payment.index.name = "payment_record_id" +<<<<<<< HEAD df_payment["created_date"] = ( df_payment["created_at"].astype("datetime64[ns]").dt.date ) @@ -83,11 +112,28 @@ def create_fact_payment(dict_of_df): ) df_payment["last_updated_time"] = ( df_payment["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time +======= + df_payment["created_date"] = pd.to_datetime( + df_payment["created_at"], format="%Y-%m-%d" + ) + df_payment["created_time"] = pd.to_datetime( + df_payment["created_at"], format="%H-%M-%S" + ) + df_payment["last_updated_date"] = pd.to_datetime( + df_payment["last_updated"], format="%Y-%m-%d" + ) + df_payment["last_updated_time"] = pd.to_datetime( + df_payment["last_updated"], format="%H-%M-%S" +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) ) df_payment["payment_date"] = pd.to_datetime( df_payment["payment_date"], format="%Y-%m-%d" ) +<<<<<<< HEAD df_payment = df_payment.drop(labels=["created_at", "last_updated"], axis=1) +======= + df_payment.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) df_payment.reset_index(inplace=True) return df_payment @@ -143,7 +189,11 @@ def create_dim_date(dict_of_df): list_of_date_columns = [] for df in fact_dfs: date_col_names = [ +<<<<<<< HEAD col_name for col_name in list(df.columns) if "_date" in col_name +======= + col_name for col_name in list(df.columns) if "date" in col_name +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) ] for col in date_col_names: list_of_date_columns.append(df[col]) diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py index c9ff43f..cc133fe 100644 --- a/tests/test_dataframes.py +++ b/tests/test_dataframes.py @@ -252,6 +252,7 @@ class TestCreateFactPayment: "payment": pd.DataFrame( data=[ [ +<<<<<<< HEAD dt.strptime( "2022-11-03 14:20:49.962846", "%Y-%m-%d %H:%M:%S.%f" ), @@ -261,6 +262,13 @@ class TestCreateFactPayment: 1, "SE18 9QO", "2020-07-16", +======= + dt(2020, 5, 17, 6, 15, 20), + dt(2020, 5, 20, 8, 19, 30), + 1, + "SE18 9QO", + "2020-7-16", +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) ] ], columns=[ @@ -287,5 +295,10 @@ class TestCreateFactPayment: for col in list(result.columns): assert col in expected_cols for col in expected_cols: +<<<<<<< HEAD if "_date" or "_time" in col: assert result[col].dtype == "O" +======= + if "date" in col: + assert result[col].dtype == "datetime64[ns]" +>>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) -- cgit v1.2.3 From 102575af5e1ac3f12b3f7e1c459a3a06bc5ec80a Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 17:24:47 +0100 Subject: amend to inner join --- src/dataframes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index 36361d2..4b32b36 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -161,7 +161,7 @@ def create_dim_location(dict_of_df): def create_dim_counterparty(dict_of_df): - df_prefixed_address = dict_of_df["address"].add_prefix( + df_prefixed_address = dict_of_df["address"].drop(labels=["created_at", "last_updated"], axis=1).add_prefix( "counterparty_legal_", axis=1 ) df_cp = pd.merge( @@ -169,10 +169,10 @@ def create_dim_counterparty(dict_of_df): df_prefixed_address, left_on="legal_address_id", right_on="counterparty_legal_address_id", - how="outer", + how="inner", ) df_cp.drop( - columns=["legal_address_id", "counterparty_legal_address_id"], inplace=True + columns=["legal_address_id", "counterparty_legal_address_id", ], inplace=True ) return df_cp -- cgit v1.2.3 From 0915d4fe4e151d6b593467129b51a1322398fc04 Mon Sep 17 00:00:00 2001 From: Ellie Date: Tue, 27 Aug 2024 17:27:21 +0100 Subject: add json.loads --- src/load_lambda.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'src') diff --git a/src/load_lambda.py b/src/load_lambda.py index 9e15af3..7339ab9 100644 --- a/src/load_lambda.py +++ b/src/load_lambda.py @@ -64,7 +64,7 @@ def retrieve_secrets(client=None, secret_name=None): logger.error(f"Secret {secret_name} does not contain a SecretString") raise ValueError(f"Secret {secret_name} does not contain a SecretString") - return json.loads(get_secret_value_response["SecretString"]) + return get_secret_value_response["SecretString"] # connect to database, slightly different way of doing it, to allow manipulation through pandas @@ -72,10 +72,10 @@ def retrieve_secrets(client=None, secret_name=None): def connect_to_db_and_return_engine(sm_secret=None): if sm_secret is None: - sm_secret = retrieve_secrets() + sm_secret = json.loads(retrieve_secrets()) try: - secrets = json.loads(sm_secret) + secrets = sm_secret host = secrets["host"] port = secrets["port"] user = secrets["user"] @@ -171,13 +171,14 @@ def upload_dfs_to_database(): ] for file_name, df in dict_of_dfs.items(): + print(df) if file_name in immutable_df_dict: table_name = file_name.split(".")[0] + print(table_name, "<<<<<") try: df.to_sql( table_name, con=db_engine, - schema="project_team_2", if_exists="append", index=False, ) -- cgit v1.2.3 From 95935534931b5ff6e617ba74c86cb7a6718128e4 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 08:24:21 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in 08c971f according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/102 --- src/dataframes.py | 182 ++++++++++++++++++++++++---------------------- tests/test_dataframes.py | 43 ++++++----- tests/test_load_lambda.py | 2 - 3 files changed, 123 insertions(+), 104 deletions(-) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index 4b32b36..43facd6 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -20,8 +20,11 @@ import requests def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" -<<<<<<< HEAD - df_sales["created_date"] = df_sales["created_at"].astype("datetime64[ns]").dt.date + + +<< << << < HEAD + df_sales["created_date"] = df_sales["created_at"].astype( + "datetime64[ns]").dt.date df_sales["created_time"] = ( df_sales["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) @@ -30,27 +33,29 @@ def create_fact_sales_order(dict_of_df): ) df_sales["last_updated_time"] = ( df_sales["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time -======= - df_sales["created_date"] = pd.to_datetime(df_sales["created_at"], format="%Y-%m-%d") - df_sales["created_time"] = pd.to_datetime(df_sales["created_at"], format="%H-%M-%S") - df_sales["last_updated_date"] = pd.to_datetime( +== == == = + df_sales["created_date"]=pd.to_datetime( + df_sales["created_at"], format="%Y-%m-%d") + df_sales["created_time"]=pd.to_datetime( + df_sales["created_at"], format="%H-%M-%S") + df_sales["last_updated_date"]=pd.to_datetime( df_sales["last_updated"], format="%Y-%m-%d" ) - df_sales["last_updated_time"] = pd.to_datetime( + df_sales["last_updated_time"]=pd.to_datetime( df_sales["last_updated"], format="%H-%M-%S" ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) ) - df_sales["agreed_delivery_date"] = pd.to_datetime( + df_sales["agreed_delivery_date"]=pd.to_datetime( df_sales["agreed_delivery_date"], format="%Y-%m-%d" ) - df_sales["agreed_payment_date"] = pd.to_datetime( + df_sales["agreed_payment_date"]=pd.to_datetime( df_sales["agreed_payment_date"], format="%Y-%m-%d" ) -<<<<<<< HEAD - df_sales = df_sales.drop(labels=["created_at", "last_updated"], axis=1) -======= +<< << << < HEAD + df_sales=df_sales.drop(labels=["created_at", "last_updated"], axis=1) +== == == = df_sales.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) df_sales.reset_index(inplace=True) return df_sales @@ -59,37 +64,40 @@ def create_fact_sales_order(dict_of_df): def create_fact_purchase_orders(dict_of_df): - df_po = dict_of_df["purchase_order"] - df_po.index.name = "purchase_record_id" -<<<<<<< HEAD - df_po["created_date"] = df_po["created_at"].astype("datetime64[ns]").dt.date - df_po["created_time"] = ( + df_po=dict_of_df["purchase_order"] + df_po.index.name="purchase_record_id" +<< << << < HEAD + df_po["created_date"]=df_po["created_at"].astype("datetime64[ns]").dt.date + df_po["created_time"]=( df_po["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_po["last_updated_date"] = df_po["last_updated"].astype("datetime64[ns]").dt.date - df_po["last_updated_time"] = ( + df_po["last_updated_date"]=df_po["last_updated"].astype( + "datetime64[ns]").dt.date + df_po["last_updated_time"]=( df_po["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time -======= - df_po["created_date"] = pd.to_datetime(df_po["created_at"], format="%Y-%m-%d") - df_po["created_time"] = pd.to_datetime(df_po["created_at"], format="%H-%M-%S") - df_po["last_updated_date"] = pd.to_datetime( +== == == = + df_po["created_date"]=pd.to_datetime( + df_po["created_at"], format="%Y-%m-%d") + df_po["created_time"]=pd.to_datetime( + df_po["created_at"], format="%H-%M-%S") + df_po["last_updated_date"]=pd.to_datetime( df_po["last_updated"], format="%Y-%m-%d" ) - df_po["last_updated_time"] = pd.to_datetime( + df_po["last_updated_time"]=pd.to_datetime( df_po["last_updated"], format="%H-%M-%S" ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) ) - df_po["agreed_delivery_date"] = pd.to_datetime( + df_po["agreed_delivery_date"]=pd.to_datetime( df_po["agreed_delivery_date"], format="%Y-%m-%d" ) - df_po["agreed_payment_date"] = pd.to_datetime( + df_po["agreed_payment_date"]=pd.to_datetime( df_po["agreed_payment_date"], format="%Y-%m-%d" ) -<<<<<<< HEAD - df_po = df_po.drop(labels=["created_at", "last_updated"], axis=1) -======= +<< << << < HEAD + df_po=df_po.drop(labels=["created_at", "last_updated"], axis=1) +== == == = df_po.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) df_po.reset_index(inplace=True) return df_po @@ -98,42 +106,44 @@ def create_fact_purchase_orders(dict_of_df): def create_fact_payment(dict_of_df): - df_payment = dict_of_df["payment"] - df_payment.index.name = "payment_record_id" -<<<<<<< HEAD - df_payment["created_date"] = ( + df_payment=dict_of_df["payment"] + df_payment.index.name="payment_record_id" +<< << << < HEAD + df_payment["created_date"]=( df_payment["created_at"].astype("datetime64[ns]").dt.date ) - df_payment["created_time"] = ( + df_payment["created_time"]=( df_payment["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_payment["last_updated_date"] = ( + df_payment["last_updated_date"]=( df_payment["last_updated"].astype("datetime64[ns]").dt.date ) - df_payment["last_updated_time"] = ( - df_payment["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time -======= - df_payment["created_date"] = pd.to_datetime( + df_payment["last_updated_time"]=( + df_payment["last_updated"].astype( + "datetime64[ns]").dt.floor("s").dt.time +== == == = + df_payment["created_date"]=pd.to_datetime( df_payment["created_at"], format="%Y-%m-%d" ) - df_payment["created_time"] = pd.to_datetime( + df_payment["created_time"]=pd.to_datetime( df_payment["created_at"], format="%H-%M-%S" ) - df_payment["last_updated_date"] = pd.to_datetime( + df_payment["last_updated_date"]=pd.to_datetime( df_payment["last_updated"], format="%Y-%m-%d" ) - df_payment["last_updated_time"] = pd.to_datetime( + df_payment["last_updated_time"]=pd.to_datetime( df_payment["last_updated"], format="%H-%M-%S" ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) ) - df_payment["payment_date"] = pd.to_datetime( + df_payment["payment_date"]=pd.to_datetime( df_payment["payment_date"], format="%Y-%m-%d" ) -<<<<<<< HEAD - df_payment = df_payment.drop(labels=["created_at", "last_updated"], axis=1) -======= - df_payment.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +<< << << < HEAD + df_payment=df_payment.drop(labels=["created_at", "last_updated"], axis=1) +== == == = + df_payment.drop( + labels=["created_at", "last_updated"], axis=1, inplace=True) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) df_payment.reset_index(inplace=True) return df_payment @@ -142,7 +152,7 @@ def create_fact_payment(dict_of_df): def create_dim_transaction(dict_of_df): - df_transaction = dict_of_df["transaction"].drop( + df_transaction=dict_of_df["transaction"].drop( labels=["created_at", "last_updated"], axis=1 ) return df_transaction @@ -152,7 +162,7 @@ def create_dim_transaction(dict_of_df): def create_dim_location(dict_of_df): - df_loc = ( + df_loc=( dict_of_df["address"] .drop(labels=["created_at", "last_updated"], axis=1) .rename(columns={"address_id": "location_id"}) @@ -161,10 +171,10 @@ def create_dim_location(dict_of_df): def create_dim_counterparty(dict_of_df): - df_prefixed_address = dict_of_df["address"].drop(labels=["created_at", "last_updated"], axis=1).add_prefix( + df_prefixed_address=dict_of_df["address"].drop(labels=["created_at", "last_updated"], axis=1).add_prefix( "counterparty_legal_", axis=1 ) - df_cp = pd.merge( + df_cp=pd.merge( dict_of_df["counterparty"], df_prefixed_address, left_on="legal_address_id", @@ -181,32 +191,32 @@ def create_dim_counterparty(dict_of_df): def create_dim_date(dict_of_df): - fact_dfs = [ + fact_dfs=[ create_fact_payment(dict_of_df), create_fact_purchase_orders(dict_of_df), create_fact_sales_order(dict_of_df), ] - list_of_date_columns = [] + list_of_date_columns=[] for df in fact_dfs: - date_col_names = [ -<<<<<<< HEAD + date_col_names=[ +<< << << < HEAD col_name for col_name in list(df.columns) if "_date" in col_name -======= +== == == = col_name for col_name in list(df.columns) if "date" in col_name ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) +>> >>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) ] for col in date_col_names: list_of_date_columns.append(df[col]) - sr_date = pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]") - df_date = pd.DataFrame(data=sr_date, columns=["date_id"]) + sr_date=pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]") + df_date=pd.DataFrame(data=sr_date, columns=["date_id"]) df_date.drop_duplicates(inplace=True) - df_date["year"] = df_date["date_id"].dt.year - df_date["month"] = df_date["date_id"].dt.month - df_date["day"] = df_date["date_id"].dt.day - df_date["day_of_week"] = df_date["date_id"].dt.dayofweek - df_date["day_name"] = df_date["date_id"].dt.day_name() - df_date["month_name"] = df_date["date_id"].dt.month_name() - df_date["quarter"] = df_date["date_id"].dt.quarter + df_date["year"]=df_date["date_id"].dt.year + df_date["month"]=df_date["date_id"].dt.month + df_date["day"]=df_date["date_id"].dt.day + df_date["day_of_week"]=df_date["date_id"].dt.dayofweek + df_date["day_name"]=df_date["date_id"].dt.day_name() + df_date["month_name"]=df_date["date_id"].dt.month_name() + df_date["quarter"]=df_date["date_id"].dt.quarter return df_date @@ -214,13 +224,13 @@ def create_dim_date(dict_of_df): def scrape_currency_names(): - response = requests.get("https://www.xe.com/currency/").content - soup = BeautifulSoup(response, "html.parser") - currency = [ + response=requests.get("https://www.xe.com/currency/").content + soup=BeautifulSoup(response, "html.parser") + currency=[ item.text for item in soup.findAll("a", attrs={"class": "sc-299dec64-6 fZPTSw"}) ] - sr = pd.Series(currency) - df_cur = sr.str.split(pat=" - ", expand=True).rename( + sr=pd.Series(currency) + df_cur=sr.str.split(pat=" - ", expand=True).rename( {0: "currency_code", 1: "currency_name"}, axis=1 ) return df_cur @@ -230,8 +240,9 @@ def scrape_currency_names(): def create_dim_currency(dict_of_df, names=scrape_currency_names()): - df_cur = dict_of_df["currency"].drop(labels=["created_at", "last_updated"], axis=1) - dim_cur = pd.merge( + df_cur=dict_of_df["currency"].drop( + labels=["created_at", "last_updated"], axis=1) + dim_cur=pd.merge( df_cur, names, left_on="currency_code", right_on="currency_code", how="inner" ) return dim_cur @@ -241,8 +252,9 @@ def create_dim_currency(dict_of_df, names=scrape_currency_names()): def create_dim_payment_type(dict_of_df): - df_payment_type = dict_of_df["payment_type"] - dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]] + df_payment_type=dict_of_df["payment_type"] + dim_payment_type=df_payment_type.loc[:, [ + "payment_type_id", "payment_type_name"]] return dim_payment_type @@ -250,8 +262,8 @@ def create_dim_payment_type(dict_of_df): def create_dim_design(dict_of_df): - df_design = dict_of_df["design"] - dim_design = df_design.loc[ + df_design=dict_of_df["design"] + dim_design=df_design.loc[ :, ["design_id", "design_name", "file_name", "file_location"] ] return dim_design @@ -261,10 +273,10 @@ def create_dim_design(dict_of_df): def create_dim_staff(dict_of_df): - staff_department = pd.merge( + staff_department=pd.merge( dict_of_df["staff"], dict_of_df["department"], on="department_id", how="left" ) - dim_staff = staff_department.loc[ + dim_staff=staff_department.loc[ :, [ "staff_id", diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py index cc133fe..785a3fd 100644 --- a/tests/test_dataframes.py +++ b/tests/test_dataframes.py @@ -54,7 +54,8 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame( + data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) assert isinstance(result, pd.DataFrame) @@ -71,7 +72,8 @@ class TestCreateDimStaff: "email_address": ["Hello", "Bye"], "department_id": ["Hello", "Bye"], } - test_df = {"staff": pd.DataFrame(data=d), "department": pd.DataFrame(data=d2)} + test_df = {"staff": pd.DataFrame( + data=d), "department": pd.DataFrame(data=d2)} result = create_dim_staff(test_df) expected_d = { "staff_id": ["Hello", "Bye"], @@ -88,7 +90,8 @@ class TestCreateDimStaff: class TestCreatePaymentType: def test_create_dim_payment_type_returns_correct_columns_and_values(self): - d = {"payment_type_id": ["Hello", "Bye"], "payment_type_name": ["Hello", "Bye"]} + d = {"payment_type_id": ["Hello", "Bye"], + "payment_type_name": ["Hello", "Bye"]} test_df = {"payment_type": pd.DataFrame(data=d)} result = create_dim_payment_type(test_df) expected_columns = ["payment_type_id", "payment_type_name"] @@ -180,11 +183,13 @@ class TestCreateDimDate: index=[0], ) df_two = pd.DataFrame( - data={"updated_date": dt(2020, 5, 17), "created_date": dt(2021, 9, 13)}, + data={"updated_date": dt(2020, 5, 17), + "created_date": dt(2021, 9, 13)}, index=[0], ) df_three = pd.DataFrame( - data={"updated_date": dt(2022, 5, 17), "created_date": dt(2023, 5, 13)}, + data={"updated_date": dt(2022, 5, 17), + "created_date": dt(2023, 5, 13)}, index=[0], ) expected_df = pd.DataFrame( @@ -214,7 +219,8 @@ class TestCreateDimDate: mock_fso.return_value = df_three result = create_dim_date({"dum": 0}) result.reset_index(inplace=True, drop=True) - assert result.eq(expected_df, axis="columns").all(axis=None) + assert result.eq( + expected_df, axis="columns").all(axis=None) class TestCreateDimLocation: @@ -222,7 +228,8 @@ class TestCreateDimLocation: dict_df = { "address": pd.DataFrame( data=[["some_time", "some_other_time", 1, "SE18 9QO"]], - columns=["created_at", "last_updated", "address_id", "postal_code"], + columns=["created_at", "last_updated", + "address_id", "postal_code"], ) } result = create_dim_location(dict_df) @@ -252,7 +259,7 @@ class TestCreateFactPayment: "payment": pd.DataFrame( data=[ [ -<<<<<<< HEAD + << << << < HEAD dt.strptime( "2022-11-03 14:20:49.962846", "%Y-%m-%d %H:%M:%S.%f" ), @@ -262,13 +269,13 @@ class TestCreateFactPayment: 1, "SE18 9QO", "2020-07-16", -======= + == == === dt(2020, 5, 17, 6, 15, 20), dt(2020, 5, 20, 8, 19, 30), 1, "SE18 9QO", "2020-7-16", ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) + >>>>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) ] ], columns=[ @@ -295,10 +302,12 @@ class TestCreateFactPayment: for col in list(result.columns): assert col in expected_cols for col in expected_cols: -<<<<<<< HEAD - if "_date" or "_time" in col: - assert result[col].dtype == "O" -======= - if "date" in col: - assert result[col].dtype == "datetime64[ns]" ->>>>>>> 5db3f61 (style: format code with Autopep8, Black and Ruff Formatter) + + +<< << << < HEAD +if "_date" or "_time" in col: + assert result[col].dtype == "O" +== == == = +if "date" in col: + assert result[col].dtype == "datetime64[ns]" +>>>>>> > 5db3f61(style: format code with Autopep8, Black and Ruff Formatter) diff --git a/tests/test_load_lambda.py b/tests/test_load_lambda.py index 02cf2c0..65106f7 100644 --- a/tests/test_load_lambda.py +++ b/tests/test_load_lambda.py @@ -62,8 +62,6 @@ class TestLambdaHandler: assert result == {"error"} - - class TestRetrieveSecrets: def test_retrieve_secrets_returns_dictionary(self, mock_sm_client): secret = { -- cgit v1.2.3 From 4bd3f408a185d16f9580294755621156ad850ab4 Mon Sep 17 00:00:00 2001 From: "deepsource-autofix[bot]" <62050782+deepsource-autofix[bot]@users.noreply.github.com> Date: Wed, 28 Aug 2024 08:36:33 +0000 Subject: style: format code with Autopep8, Black and Ruff Formatter This commit fixes the style issues introduced in d0b0fa9 according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/102 --- src/dataframes.py | 118 +++++++++++++++++++++++------------------------ tests/test_dataframes.py | 2 - 2 files changed, 59 insertions(+), 61 deletions(-) (limited to 'src') diff --git a/src/dataframes.py b/src/dataframes.py index ab32fff..2a46bd6 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -20,9 +20,8 @@ import requests def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" - - df_sales["created_date"] = df_sales["created_at"].astype( - "datetime64[ns]").dt.date + + df_sales["created_date"] = df_sales["created_at"].astype("datetime64[ns]").dt.date df_sales["created_time"] = ( df_sales["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) @@ -32,13 +31,13 @@ def create_fact_sales_order(dict_of_df): df_sales["last_updated_time"] = ( df_sales["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_sales["agreed_delivery_date"]=pd.to_datetime( + df_sales["agreed_delivery_date"] = pd.to_datetime( df_sales["agreed_delivery_date"], format="%Y-%m-%d" ) - df_sales["agreed_payment_date"]=pd.to_datetime( + df_sales["agreed_payment_date"] = pd.to_datetime( df_sales["agreed_payment_date"], format="%Y-%m-%d" ) - df_sales=df_sales.drop(labels=["created_at", "last_updated"], axis=1) + df_sales = df_sales.drop(labels=["created_at", "last_updated"], axis=1) df_sales.reset_index(inplace=True) return df_sales @@ -68,25 +67,23 @@ def create_fact_sales_order(dict_of_df): def create_fact_purchase_orders(dict_of_df): - df_po=dict_of_df["purchase_order"] - df_po.index.name="purchase_record_id" - df_po["created_date"]=df_po["created_at"].astype("datetime64[ns]").dt.date - df_po["created_time"]=( + df_po = dict_of_df["purchase_order"] + df_po.index.name = "purchase_record_id" + df_po["created_date"] = df_po["created_at"].astype("datetime64[ns]").dt.date + df_po["created_time"] = ( df_po["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_po["last_updated_date"]=df_po["last_updated"].astype( - "datetime64[ns]").dt.date - df_po["last_updated_time"]=( + df_po["last_updated_date"] = df_po["last_updated"].astype("datetime64[ns]").dt.date + df_po["last_updated_time"] = ( df_po["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time - ) - df_po["agreed_delivery_date"]=pd.to_datetime( + df_po["agreed_delivery_date"] = pd.to_datetime( df_po["agreed_delivery_date"], format="%Y-%m-%d" ) - df_po["agreed_payment_date"]=pd.to_datetime( + df_po["agreed_payment_date"] = pd.to_datetime( df_po["agreed_payment_date"], format="%Y-%m-%d" ) - df_po=df_po.drop(labels=["created_at", "last_updated"], axis=1) + df_po = df_po.drop(labels=["created_at", "last_updated"], axis=1) df_po.reset_index(inplace=True) return df_po @@ -95,26 +92,25 @@ def create_fact_purchase_orders(dict_of_df): def create_fact_payment(dict_of_df): - df_payment=dict_of_df["payment"] - df_payment.index.name="payment_record_id" - df_payment["created_date"]=( + df_payment = dict_of_df["payment"] + df_payment.index.name = "payment_record_id" + df_payment["created_date"] = ( df_payment["created_at"].astype("datetime64[ns]").dt.date ) - df_payment["created_time"]=( + df_payment["created_time"] = ( df_payment["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_payment["last_updated_date"]=( + df_payment["last_updated_date"] = ( df_payment["last_updated"].astype("datetime64[ns]").dt.date ) - df_payment["last_updated_time"]=( - df_payment["last_updated"].astype( - "datetime64[ns]").dt.floor("s").dt.time + df_payment["last_updated_time"] = ( + df_payment["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_payment["payment_date"]=pd.to_datetime( + df_payment["payment_date"] = pd.to_datetime( df_payment["payment_date"], format="%Y-%m-%d" ) - df_payment=df_payment.drop(labels=["created_at", "last_updated"], axis=1) - + df_payment = df_payment.drop(labels=["created_at", "last_updated"], axis=1) + df_payment.reset_index(inplace=True) return df_payment @@ -123,7 +119,7 @@ def create_fact_payment(dict_of_df): def create_dim_transaction(dict_of_df): - df_transaction=dict_of_df["transaction"].drop( + df_transaction = dict_of_df["transaction"].drop( labels=["created_at", "last_updated"], axis=1 ) return df_transaction @@ -133,7 +129,7 @@ def create_dim_transaction(dict_of_df): def create_dim_location(dict_of_df): - df_loc=( + df_loc = ( dict_of_df["address"] .drop(labels=["created_at", "last_updated"], axis=1) .rename(columns={"address_id": "location_id"}) @@ -142,10 +138,12 @@ def create_dim_location(dict_of_df): def create_dim_counterparty(dict_of_df): - df_prefixed_address=dict_of_df["address"].drop(labels=["created_at", "last_updated"], axis=1).add_prefix( - "counterparty_legal_", axis=1 + df_prefixed_address = ( + dict_of_df["address"] + .drop(labels=["created_at", "last_updated"], axis=1) + .add_prefix("counterparty_legal_", axis=1) ) - df_cp=pd.merge( + df_cp = pd.merge( dict_of_df["counterparty"], df_prefixed_address, left_on="legal_address_id", @@ -153,7 +151,11 @@ def create_dim_counterparty(dict_of_df): how="inner", ) df_cp.drop( - columns=["legal_address_id", "counterparty_legal_address_id", ], inplace=True + columns=[ + "legal_address_id", + "counterparty_legal_address_id", + ], + inplace=True, ) return df_cp @@ -162,7 +164,7 @@ def create_dim_counterparty(dict_of_df): def create_dim_date(dict_of_df): - fact_dfs=[ + fact_dfs = [ create_fact_payment(dict_of_df), create_fact_purchase_orders(dict_of_df), create_fact_sales_order(dict_of_df), @@ -174,16 +176,16 @@ def create_dim_date(dict_of_df): ] for col in date_col_names: list_of_date_columns.append(df[col]) - sr_date=pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]") - df_date=pd.DataFrame(data=sr_date, columns=["date_id"]) + sr_date = pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]") + df_date = pd.DataFrame(data=sr_date, columns=["date_id"]) df_date.drop_duplicates(inplace=True) - df_date["year"]=df_date["date_id"].dt.year - df_date["month"]=df_date["date_id"].dt.month - df_date["day"]=df_date["date_id"].dt.day - df_date["day_of_week"]=df_date["date_id"].dt.dayofweek - df_date["day_name"]=df_date["date_id"].dt.day_name() - df_date["month_name"]=df_date["date_id"].dt.month_name() - df_date["quarter"]=df_date["date_id"].dt.quarter + df_date["year"] = df_date["date_id"].dt.year + df_date["month"] = df_date["date_id"].dt.month + df_date["day"] = df_date["date_id"].dt.day + df_date["day_of_week"] = df_date["date_id"].dt.dayofweek + df_date["day_name"] = df_date["date_id"].dt.day_name() + df_date["month_name"] = df_date["date_id"].dt.month_name() + df_date["quarter"] = df_date["date_id"].dt.quarter return df_date @@ -191,13 +193,13 @@ def create_dim_date(dict_of_df): def scrape_currency_names(): - response=requests.get("https://www.xe.com/currency/").content - soup=BeautifulSoup(response, "html.parser") - currency=[ + response = requests.get("https://www.xe.com/currency/").content + soup = BeautifulSoup(response, "html.parser") + currency = [ item.text for item in soup.findAll("a", attrs={"class": "sc-299dec64-6 fZPTSw"}) ] - sr=pd.Series(currency) - df_cur=sr.str.split(pat=" - ", expand=True).rename( + sr = pd.Series(currency) + df_cur = sr.str.split(pat=" - ", expand=True).rename( {0: "currency_code", 1: "currency_name"}, axis=1 ) return df_cur @@ -207,9 +209,8 @@ def scrape_currency_names(): def create_dim_currency(dict_of_df, names=scrape_currency_names()): - df_cur=dict_of_df["currency"].drop( - labels=["created_at", "last_updated"], axis=1) - dim_cur=pd.merge( + df_cur = dict_of_df["currency"].drop(labels=["created_at", "last_updated"], axis=1) + dim_cur = pd.merge( df_cur, names, left_on="currency_code", right_on="currency_code", how="inner" ) return dim_cur @@ -219,9 +220,8 @@ def create_dim_currency(dict_of_df, names=scrape_currency_names()): def create_dim_payment_type(dict_of_df): - df_payment_type=dict_of_df["payment_type"] - dim_payment_type=df_payment_type.loc[:, [ - "payment_type_id", "payment_type_name"]] + df_payment_type = dict_of_df["payment_type"] + dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]] return dim_payment_type @@ -229,8 +229,8 @@ def create_dim_payment_type(dict_of_df): def create_dim_design(dict_of_df): - df_design=dict_of_df["design"] - dim_design=df_design.loc[ + df_design = dict_of_df["design"] + dim_design = df_design.loc[ :, ["design_id", "design_name", "file_name", "file_location"] ] return dim_design @@ -240,10 +240,10 @@ def create_dim_design(dict_of_df): def create_dim_staff(dict_of_df): - staff_department=pd.merge( + staff_department = pd.merge( dict_of_df["staff"], dict_of_df["department"], on="department_id", how="left" ) - dim_staff=staff_department.loc[ + dim_staff = staff_department.loc[ :, [ "staff_id", diff --git a/tests/test_dataframes.py b/tests/test_dataframes.py index ff282eb..ea7bad1 100644 --- a/tests/test_dataframes.py +++ b/tests/test_dataframes.py @@ -227,7 +227,6 @@ class TestCreateDimDate: expected_df, axis="columns").all(axis=None) - class TestCreateDimLocation: def test_returns_correct_columns_lo(self): dict_df = { @@ -302,6 +301,5 @@ class TestCreateFactPayment: for col in expected_cols: - if "_date" or "_time" in col: assert result[col].dtype == "O" -- cgit v1.2.3