aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authordeepsource-autofix[bot] <62050782+deepsource-autofix[bot]@users.noreply.github.com>2024-08-27 09:46:39 +0000
committerGitHub <noreply@github.com>2024-08-27 09:46:39 +0000
commite51e9fc3c7fa886fe5e753bd123d45c8871673bc (patch)
tree9a7d397028986693923c9b7169d142c34a783cb0
parentc68f63fa3aebcf9b77c24d6e2aec91a4ff4950bb (diff)
downloadde-project-bentley-e51e9fc3c7fa886fe5e753bd123d45c8871673bc.tar.gz
de-project-bentley-e51e9fc3c7fa886fe5e753bd123d45c8871673bc.zip
style: format code with Autopep8, Black and Ruff Formatter
This commit fixes the style issues introduced in c68f63f according to the output from Autopep8, Black and Ruff Formatter. Details: https://github.com/ajschofield/de-project-bentley/pull/97
-rw-r--r--src/dataframes.py74
-rw-r--r--src/transform_lambda.py6
-rw-r--r--tests/test_transform_lambda.py44
3 files changed, 65 insertions, 59 deletions
diff --git a/src/dataframes.py b/src/dataframes.py
index 94eb509..ab53063 100644
--- a/src/dataframes.py
+++ b/src/dataframes.py
@@ -21,10 +21,8 @@ def create_fact_sales_order(dict_of_df):
df_sales.index.name = "sales_record_id"
df_sales["created_date"] = pd.to_datetime(df_sales["created_at"]).dt.date
df_sales["created_time"] = pd.to_datetime(df_sales["created_at"]).dt.time
- df_sales["last_updated_date"] = pd.to_datetime(
- df_sales["last_updated"]).dt.date
- df_sales["last_updated_time"] = pd.to_datetime(
- df_sales["last_updated"]).dt.time
+ df_sales["last_updated_date"] = pd.to_datetime(df_sales["last_updated"]).dt.date
+ df_sales["last_updated_time"] = pd.to_datetime(df_sales["last_updated"]).dt.time
fact_sales_order = df_sales.loc[
:,
[
@@ -76,7 +74,8 @@ def create_fact_payment(dict_of_df):
df_payment["last_updated_date"] = df_payment["last_updated"].date()
df_payment["last_updated_time"] = df_payment["last_updated"].time
df_payment["payment_date"] = pd.to_datetime(
- df_payment["payment_date"], format="%Y-%m-%d")
+ df_payment["payment_date"], format="%Y-%m-%d"
+ )
fact_payment = df_payment.loc[
:,
[
@@ -113,16 +112,16 @@ def create_dim_location(dict_of_df):
df_loc = (
dict_of_df["address"]
.drop(labels=["created_at", "last_updated"], axis=1)
- .rename(columns={"address_id": "location_id"}))
+ .rename(columns={"address_id": "location_id"})
+ )
return df_loc
-
def create_dim_counterparty(dict_of_df):
- df_prefixed_address=dict_of_df["address"].add_prefix(
+ df_prefixed_address = dict_of_df["address"].add_prefix(
"counterparty_legal_", axis=1
)
- df_cp=pd.merge(
+ df_cp = pd.merge(
dict_of_df["counterparty"],
df_prefixed_address,
left_on="legal_address_id",
@@ -139,51 +138,51 @@ def create_dim_counterparty(dict_of_df):
def create_dim_date(dict_of_df):
- fact_dfs=[
+ fact_dfs = [
create_fact_payment(dict_of_df),
create_fact_purchase_orders(dict_of_df),
create_fact_sales_order(dict_of_df),
]
- date_col_names=[
+ date_col_names = [
col_name for col_name in list(fact_dfs[0].columns) if "date" in col_name
]
- list_of_date_columns=[]
+ list_of_date_columns = []
for df in fact_dfs:
for col in date_col_names:
list_of_date_columns.append(df[col])
- sr_date=pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]")
- df_date=pd.DataFrame(data=sr_date, columns=["date_id"])
+ sr_date = pd.array(pd.concat(list_of_date_columns), dtype="datetime64[ns]")
+ df_date = pd.DataFrame(data=sr_date, columns=["date_id"])
df_date.drop_duplicates(inplace=True)
- df_date["year"]=df_date["date_id"].dt.year
- df_date["month"]=df_date["date_id"].dt.month
- df_date["day"]=df_date["date_id"].dt.day
- df_date["day_of_week"]=df_date["date_id"].dt.dayofweek
- df_date["day_name"]=df_date["date_id"].dt.day_name()
- df_date["month_name"]=df_date["date_id"].dt.month_name()
- df_date["quarter"]=df_date["date_id"].dt.quarter
+ df_date["year"] = df_date["date_id"].dt.year
+ df_date["month"] = df_date["date_id"].dt.month
+ df_date["day"] = df_date["date_id"].dt.day
+ df_date["day_of_week"] = df_date["date_id"].dt.dayofweek
+ df_date["day_name"] = df_date["date_id"].dt.day_name()
+ df_date["month_name"] = df_date["date_id"].dt.month_name()
+ df_date["quarter"] = df_date["date_id"].dt.quarter
return df_date
# tests passed
def scrape_currency_names():
- response=requests.get("https://www.xe.com/currency/").content
- soup=BeautifulSoup(response, "html.parser")
- currency=[
+ response = requests.get("https://www.xe.com/currency/").content
+ soup = BeautifulSoup(response, "html.parser")
+ currency = [
item.text for item in soup.findAll("a", attrs={"class": "sc-299dec64-6 fZPTSw"})
]
- sr=pd.Series(currency)
- df_cur=sr.str.split(pat=" - ", expand=True).rename(
+ sr = pd.Series(currency)
+ df_cur = sr.str.split(pat=" - ", expand=True).rename(
{0: "currency_code", 1: "currency_name"}, axis=1
)
return df_cur
+
# tests passed
def create_dim_currency(dict_of_df, names=scrape_currency_names()):
- df_cur=dict_of_df["currency"].drop(
- labels=["created_at", "last_updated"], axis=1)
- dim_cur=pd.merge(
+ df_cur = dict_of_df["currency"].drop(labels=["created_at", "last_updated"], axis=1)
+ dim_cur = pd.merge(
df_cur, names, left_on="currency_code", right_on="currency_code", how="inner"
)
return dim_cur
@@ -191,33 +190,32 @@ def create_dim_currency(dict_of_df, names=scrape_currency_names()):
# tests passed
+
def create_dim_payment_type(dict_of_df):
- df_payment_type=dict_of_df["payment_type"]
- dim_payment_type=df_payment_type.loc[:, [
- "payment_type_id", "payment_type_name"]]
+ df_payment_type = dict_of_df["payment_type"]
+ dim_payment_type = df_payment_type.loc[:, ["payment_type_id", "payment_type_name"]]
return dim_payment_type
-
# tests passed
def create_dim_design(dict_of_df):
- df_design=dict_of_df["design"]
- dim_design=df_design.loc[
+ df_design = dict_of_df["design"]
+ dim_design = df_design.loc[
:, ["design_id", "design_name", "file_name", "file_location"]
]
return dim_design
-
# tests passed
+
def create_dim_staff(dict_of_df):
- staff_department=pd.merge(
+ staff_department = pd.merge(
dict_of_df["staff"], dict_of_df["department"], on="department_id", how="left"
)
- dim_staff=staff_department.loc[
+ dim_staff = staff_department.loc[
:,
[
"staff_id",
diff --git a/src/transform_lambda.py b/src/transform_lambda.py
index 565b4ee..2cd9272 100644
--- a/src/transform_lambda.py
+++ b/src/transform_lambda.py
@@ -11,7 +11,6 @@ from pg8000.native import Connection, InterfaceError
from datetime import datetime
-
class DBConnectionException(Exception):
"""Wraps pg8000.native Error or DatabaseError."""
@@ -212,5 +211,6 @@ def list_existing_s3_files(bucket_name, client=boto3.client("s3")):
return existing_files
-if __name__ == '__main__':
- lambda_handler({}, '') \ No newline at end of file
+
+if __name__ == "__main__":
+ lambda_handler({}, "")
diff --git a/tests/test_transform_lambda.py b/tests/test_transform_lambda.py
index 00f3d83..5ed743e 100644
--- a/tests/test_transform_lambda.py
+++ b/tests/test_transform_lambda.py
@@ -1,4 +1,8 @@
-from src.transform_lambda import read_from_s3_subfolder_to_df, list_existing_s3_files, bucket_name
+from src.transform_lambda import (
+ read_from_s3_subfolder_to_df,
+ list_existing_s3_files,
+ bucket_name,
+)
from moto import mock_aws
import pytest
import pandas as pd
@@ -6,14 +10,15 @@ import os
import boto3
from botocore.exceptions import ClientError
import numpy as np
+
# import caplog
import logging
-
logger = logging.getLogger()
logger.setLevel(logging.INFO)
+
@pytest.fixture(scope="class")
def aws_credentials():
os.environ["AWS_ACCESS_KEY_ID"] = "testing"
@@ -92,42 +97,45 @@ class TestReadFromS3:
assert result["Foods"].eq(expected_foods_df, axis="columns").all(axis=None)
assert result["Cars"].eq(expected_cars_df, axis="columns").all(axis=None)
+
class TestListExistingFiles:
def test_functions_receives_error_if_no_bucket(self, s3_client, caplog):
caplog.set_level(logging.INFO)
with pytest.raises(ClientError):
- list_existing_s3_files('rando_bucket', client=s3_client)
+ list_existing_s3_files("rando_bucket", client=s3_client)
- assert "Error listing S3 objects: An error occurred (NoSuchBucket) when calling the ListObjectsV2 operation: The specified bucket does not exist" in caplog.text
+ assert (
+ "Error listing S3 objects: An error occurred (NoSuchBucket) when calling the ListObjectsV2 operation: The specified bucket does not exist"
+ in caplog.text
+ )
def test_recieves_logger_error_if_no_files_listed(self, s3_client, caplog):
caplog.set_level(logging.INFO)
s3_client.create_bucket(
- Bucket='mock_bucket',
- CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}
+ Bucket="mock_bucket",
+ CreateBucketConfiguration={"LocationConstraint": "eu-west-2"},
)
- response = list_existing_s3_files('mock_bucket', client=s3_client)
- assert 'The bucket is empty' in caplog.text
+ response = list_existing_s3_files("mock_bucket", client=s3_client)
+ assert "The bucket is empty" in caplog.text
def test_retrieves_existing_files(self, s3_client, caplog):
caplog.set_level(logging.INFO)
- s3_client.upload_file(
- "tests/dummy.txt", 'mock_bucket', "dummy.txt"
- )
- result = list_existing_s3_files('mock_bucket', client=s3_client)
+ s3_client.upload_file("tests/dummy.txt", "mock_bucket", "dummy.txt")
+ result = list_existing_s3_files("mock_bucket", client=s3_client)
assert result == ["dummy.txt"]
+
class TestBucketName:
def test_functions_retrieves_bucket(self, s3_client):
s3_client.create_bucket(
- Bucket='mock_bucket',
- CreateBucketConfiguration={"LocationConstraint": "eu-west-2"}
+ Bucket="mock_bucket",
+ CreateBucketConfiguration={"LocationConstraint": "eu-west-2"},
)
-
- bucket = bucket_name('mock_bucket', s3_client)
- assert bucket == 'mock_bucket'
- # def test_ \ No newline at end of file
+ bucket = bucket_name("mock_bucket", s3_client)
+ assert bucket == "mock_bucket"
+
+ # def test_
git.ajschof.me — hosted by ajschofield — powered by cgit