diff options
| author | T-Aji <tolujbd2@gmail.com> | 2024-08-28 17:52:45 +0100 |
|---|---|---|
| committer | T-Aji <tolujbd2@gmail.com> | 2024-08-28 17:52:45 +0100 |
| commit | 6235a2bb04b60d57a41196b07bbf0296920c6980 (patch) | |
| tree | a9c768be6a4503df12b8547eb8f21757a37ff3f5 /src/transform_lambda | |
| parent | d064b2ec2c7393f8de50560a7edfe100851bfea3 (diff) | |
| download | de-project-bentley-6235a2bb04b60d57a41196b07bbf0296920c6980.tar.gz de-project-bentley-6235a2bb04b60d57a41196b07bbf0296920c6980.zip | |
wip commit
Diffstat (limited to 'src/transform_lambda')
| -rw-r--r-- | src/transform_lambda/dataframes.py | 8 | ||||
| -rw-r--r-- | src/transform_lambda/transform_lambda.py | 2 |
2 files changed, 6 insertions, 4 deletions
diff --git a/src/transform_lambda/dataframes.py b/src/transform_lambda/dataframes.py index bf0556b..e89a6b2 100644 --- a/src/transform_lambda/dataframes.py +++ b/src/transform_lambda/dataframes.py @@ -18,7 +18,7 @@ import requests # no test, same as fact_payment def create_fact_sales_order(dict_of_df): - df_sales = dict_of_df["sales_order"] + df_sales = dict_of_df["sales_order"].rename(columns={"staff_id": "sales_staff_id"}) df_sales.index.name = "sales_record_id" df_sales["created_date"] = df_sales["created_at"].astype("datetime64[ns]").dt.date @@ -44,7 +44,7 @@ def create_fact_sales_order(dict_of_df): "created_time", "last_updated_date", "last_updated_time", - "staff_id", + "sales_staff_id", "counterparty_id", "units_sold", "unit_price", @@ -55,7 +55,7 @@ def create_fact_sales_order(dict_of_df): "agreed_delivery_location_id" ], ] - fact_sales.rename(columns={"staff_id": "sales_staff_id"}).reset_index(inplace=True) + fact_sales.reset_index(inplace=True) return fact_sales @@ -253,6 +253,8 @@ def create_dim_currency(dict_of_df, names=scrape_currency_names()): df_cur, names, left_on="currency_code", right_on="currency_code", how="left" ) dim_currency.drop_duplicates(inplace=True) + dim_currency.astype({"currency_name": "string", "currency_code": "string"}) + print(dim_currency.dtypes, "<<<<<<<<<Dtype") return dim_currency diff --git a/src/transform_lambda/transform_lambda.py b/src/transform_lambda/transform_lambda.py index 1453c6c..0b5748b 100644 --- a/src/transform_lambda/transform_lambda.py +++ b/src/transform_lambda/transform_lambda.py @@ -5,7 +5,7 @@ import logging import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -from dataframes import * +from src.transform_lambda.dataframes import * from botocore.exceptions import ClientError from pg8000.native import Connection, InterfaceError from datetime import datetime |