aboutsummaryrefslogtreecommitdiffstats
path: root/src/transform_lambda.py
diff options
context:
space:
mode:
Diffstat (limited to 'src/transform_lambda.py')
-rw-r--r--src/transform_lambda.py36
1 files changed, 20 insertions, 16 deletions
diff --git a/src/transform_lambda.py b/src/transform_lambda.py
index 3a7cf43..b176ccc 100644
--- a/src/transform_lambda.py
+++ b/src/transform_lambda.py
@@ -1,4 +1,4 @@
-#from src.extract_lambda import extract_bucket
+# from src.extract_lambda import extract_bucket
import json
import boto3
import re
@@ -6,29 +6,33 @@ import io
from io import StringIO
import pandas as pd
+
def lambda_handler(event, context):
pass
-tables = ['sales_order',
- 'transaction',
- 'payment',
- 'counterparty',
- 'address',
- 'staff',
- 'purchase_order',
- 'department',
- 'currency',
- 'design',
- 'payment_type']
+tables = [
+ "sales_order",
+ "transaction",
+ "payment",
+ "counterparty",
+ "address",
+ "staff",
+ "purchase_order",
+ "department",
+ "currency",
+ "design",
+ "payment_type",
+]
+
-def read_from_s3_subfolder_to_df(tables, bucket, client=boto3.client('s3')):
+def read_from_s3_subfolder_to_df(tables, bucket, client=boto3.client("s3")):
table_dfs = {}
for table in tables:
response = client.list_objects_v2(Bucket=bucket, Prefix=table)
- list_of_keys = ['s3://'+bucket+'/'+object['Key'] for object in response['Contents']]
+ list_of_keys = [
+ "s3://" + bucket + "/" + object["Key"] for object in response["Contents"]
+ ]
list_of_df = [pd.read_csv(key) for key in list_of_keys]
table_dfs[table] = pd.concat(list_of_df)
return table_dfs
-
-
git.ajschof.me — hosted by ajschofield — powered by cgit