diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/dataframes.py | 51 | ||||
| -rw-r--r-- | src/transform_lambda.py | 7 |
2 files changed, 32 insertions, 26 deletions
diff --git a/src/dataframes.py b/src/dataframes.py index ce15872..f122368 100644 --- a/src/dataframes.py +++ b/src/dataframes.py @@ -20,22 +20,23 @@ import requests def create_fact_sales_order(dict_of_df): df_sales = dict_of_df["sales_order"] df_sales.index.name = "sales_record_id" - df_sales["created_date"] = pd.to_datetime( - df_sales["created_at"].dt.date, format="%Y-%m-%d" + df_sales["created_date"] = df_sales["created_at"].astype("datetime64[ns]").dt.date + df_sales["created_time"] = ( + df_sales["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_sales["created_time"] = df_sales["created_at"].dt.floor("s").dt.time - df_sales["last_updated_date"] = pd.to_datetime( - df_sales["last_updated"].dt.date, format="%Y-%m-%d" + df_sales["last_updated_date"] = ( + df_sales["last_updated"].astype("datetime64[ns]").dt.date + ) + df_sales["last_updated_time"] = ( + df_sales["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_sales["last_updated_time"] = df_sales["last_updated"].dt.floor("s").dt.time - df_sales["agreed_delivery_date"] = pd.to_datetime( df_sales["agreed_delivery_date"], format="%Y-%m-%d" ) df_sales["agreed_payment_date"] = pd.to_datetime( df_sales["agreed_payment_date"], format="%Y-%m-%d" ) - df_sales.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) + df_sales = df_sales.drop(labels=["created_at", "last_updated"], axis=1) df_sales.reset_index(inplace=True) return df_sales @@ -46,21 +47,21 @@ def create_fact_sales_order(dict_of_df): def create_fact_purchase_orders(dict_of_df): df_po = dict_of_df["purchase_order"] df_po.index.name = "purchase_record_id" - df_po["created_date"] = pd.to_datetime( - df_po["created_at"].dt.date, format="%Y-%m-%d" + df_po["created_date"] = df_po["created_at"].astype("datetime64[ns]").dt.date + df_po["created_time"] = ( + df_po["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_po["created_time"] = df_po["created_at"].dt.floor("s").dt.time - df_po["last_updated_date"] = pd.to_datetime( - df_po["last_updated"].dt.date, format="%Y-%m-%d" + df_po["last_updated_date"] = df_po["last_updated"].astype("datetime64[ns]").dt.date + df_po["last_updated_time"] = ( + df_po["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_po["last_updated_time"] = df_po["last_updated"].dt.floor("s").dt.time df_po["agreed_delivery_date"] = pd.to_datetime( df_po["agreed_delivery_date"], format="%Y-%m-%d" ) df_po["agreed_payment_date"] = pd.to_datetime( df_po["agreed_payment_date"], format="%Y-%m-%d" ) - df_po.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) + df_po = df_po.drop(labels=["created_at", "last_updated"], axis=1) df_po.reset_index(inplace=True) return df_po @@ -71,18 +72,22 @@ def create_fact_purchase_orders(dict_of_df): def create_fact_payment(dict_of_df): df_payment = dict_of_df["payment"] df_payment.index.name = "payment_record_id" - df_payment["created_date"] = pd.to_datetime( - df_payment["created_at"].dt.date, format="%Y-%m-%d" + df_payment["created_date"] = ( + df_payment["created_at"].astype("datetime64[ns]").dt.date + ) + df_payment["created_time"] = ( + df_payment["created_at"].astype("datetime64[ns]").dt.floor("s").dt.time + ) + df_payment["last_updated_date"] = ( + df_payment["last_updated"].astype("datetime64[ns]").dt.date ) - df_payment["created_time"] = df_payment["created_at"].dt.floor("s").dt.time - df_payment["last_updated_date"] = pd.to_datetime( - df_payment["last_updated"].dt.date, format="%Y-%m-%d" + df_payment["last_updated_time"] = ( + df_payment["last_updated"].astype("datetime64[ns]").dt.floor("s").dt.time ) - df_payment["last_updated_time"] = df_payment["last_updated"].dt.floor("s").dt.time df_payment["payment_date"] = pd.to_datetime( df_payment["payment_date"], format="%Y-%m-%d" ) - df_payment.drop(labels=["created_at", "last_updated"], axis=1, inplace=True) + df_payment = df_payment.drop(labels=["created_at", "last_updated"], axis=1) df_payment.reset_index(inplace=True) return df_payment @@ -138,7 +143,7 @@ def create_dim_date(dict_of_df): list_of_date_columns = [] for df in fact_dfs: date_col_names = [ - col_name for col_name in list(df.columns) if "date" in col_name + col_name for col_name in list(df.columns) if "_date" in col_name ] for col in date_col_names: list_of_date_columns.append(df[col]) diff --git a/src/transform_lambda.py b/src/transform_lambda.py index 2cd9272..93b2284 100644 --- a/src/transform_lambda.py +++ b/src/transform_lambda.py @@ -117,7 +117,8 @@ def process_to_parquet_and_upload_to_s3( parquet_file = df.to_parquet( f"{table_name}.parquet", engine="pyarrow" ) # or fastparquet - client.upload_file(parquet_file, bucket, f"{table_name}.parquet") + # changed parquet_file variable to the file name + client.upload_file(f"{table_name}.parquet", bucket, f"{table_name}.parquet") status["uploaded"].append(table_name) for table_name, df in mutable_df_dict.items(): @@ -127,7 +128,7 @@ def process_to_parquet_and_upload_to_s3( parquet_file = df.to_parquet( f"{table_name}.parquet", engine="pyarrow" ) # or fastparquet - client.upload_file(parquet_file, bucket, s3_key) + client.upload_file(f"{table_name}.parquet", bucket, s3_key) status["uploaded"].append(table_name) return status @@ -203,7 +204,7 @@ def list_existing_s3_files(bucket_name, client=boto3.client("s3")): existing_files = [obj["Key"] for obj in response["Contents"]] else: logger.error("The bucket is empty") - return None + return [] # changed from None to [] so it is an iterable except ClientError as e: logger.error(f"Error listing S3 objects: {e}") |