dumps data to csv

1 files changed, 27 insertions, 5 deletions

diff --git a/src/extract_lambda.py b/src/extract_lambda.py
index 7d56c66..8317ef8 100644
--- a/src/extract_lambda.py
+++ b/src/extract_lambda.py
@@ -1,22 +1,25 @@
 from pg8000.native import Connection, Error, DatabaseError, InterfaceError
 from dotenv import load_dotenv
 import os
+import boto3
+import csv
+from botocore.exceptions import ClientError
 
 load_dotenv()
 
-def extract():
+def lambda_handler(event, context):
 
+    client = boto3.client('s3')
 # temporary credentials for dev- will not have access when uploaded
-
+    
     database = os.getenv('database')
     user = os.getenv('user')
     password = os.getenv('password')
     host = os.getenv('host')
     port = os.getenv('port')
 
-
     try:
-        db = Connection.run(
+        db = Connection(
         database=database,
         user=user,
         password=password,
@@ -27,6 +30,25 @@ def extract():
         print(e)
     except InterfaceError as i:
         print(i)
-
+    #replace prints with upload to cloudwatch logs 
+
+    tables = db.run("SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';")
+    for table in tables:
+        table_name = table[0]
+        rows = db.run(f"SELECT * FROM {table_name};")
+        # this saves the csv files to the repo root before writing to s3, this is unnecessary. how will the lambda behave when it attempts to save files?
+        with open(f"{table_name}.csv", "w", newline='') as file:
+            writer = csv.writer(file)
+            writer.writerow([desc["name"] for desc in db.columns(f"SELECT * FROM {table_name};")])
+            writer.writerows(rows)
+            try:  
+                client.upload_file(file, Bucket='ingestion-bucket', Object_name=table_name)
+            
+            except ClientError as e:
+                print(e)
+            #replace print with upload to cloudwatch logs 
+    
+    if db:
+        db.close()
 
     
\ No newline at end of file