blob: 8317ef8c0ea1840479745e17c3f276396c6c26a7 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
|
from pg8000.native import Connection, Error, DatabaseError, InterfaceError
from dotenv import load_dotenv
import os
import boto3
import csv
from botocore.exceptions import ClientError
load_dotenv()
def lambda_handler(event, context):
client = boto3.client('s3')
# temporary credentials for dev- will not have access when uploaded
database = os.getenv('database')
user = os.getenv('user')
password = os.getenv('password')
host = os.getenv('host')
port = os.getenv('port')
try:
db = Connection(
database=database,
user=user,
password=password,
host=host,
port=port
)
except DatabaseError as e:
print(e)
except InterfaceError as i:
print(i)
#replace prints with upload to cloudwatch logs
tables = db.run("SELECT table_name FROM information_schema.tables WHERE table_schema='public' AND table_type='BASE TABLE';")
for table in tables:
table_name = table[0]
rows = db.run(f"SELECT * FROM {table_name};")
# this saves the csv files to the repo root before writing to s3, this is unnecessary. how will the lambda behave when it attempts to save files?
with open(f"{table_name}.csv", "w", newline='') as file:
writer = csv.writer(file)
writer.writerow([desc["name"] for desc in db.columns(f"SELECT * FROM {table_name};")])
writer.writerows(rows)
try:
client.upload_file(file, Bucket='ingestion-bucket', Object_name=table_name)
except ClientError as e:
print(e)
#replace print with upload to cloudwatch logs
if db:
db.close()
|
