Dev sync ago

sync-ago.py

@@ -10,6 +10,7 @@
 import petl as etl
 import pyproj
 import shapely.wkt
+import citygeo_secrets
 from shapely.ops import transform as shapely_transformer
 from config import *
 import click
@@ -31,13 +32,16 @@ def sync(day):
                                                always_xy=True)
 
     print('Connecting to AGO...')
-    org = GIS(url='https://phl.maps.arcgis.com',
-                username='maps.phl.data',
-                password=MAPS_PASSWORD,
+    ago_creds = citygeo_secrets.connect_with_secrets(connect_maps_ago, "maps.phl.data")
+    org = GIS(url=ago_creds.get('url'),
+                username=ago_creds.get('login'),
+                password=ago_creds.get('password'),
                 verify_cert=False)
-    print('Connected to AGO.\n')
 
-    flayer = org.content.get(SALESFORCE_AGO_ITEMID)
+    print('Connected to AGO.\n')
+
+    salesforce_creds = citygeo_secrets.connect_with_secrets(connect_salesforce, "salesforce API copy" )
+    flayer = org.content.get(salesforce_creds.get('AGO_item_id'))
     LAYER_OBJECT = flayer.layers[0]
     print(LAYER_OBJECT)
 
@@ -49,21 +53,9 @@ def sync(day):
     else:
         raise AssertionError('Item is not geometric.\n')
 
-
-    def database_connect():
-        user = 'sde'
-        password = DATABRIDGE_SDE_PASSWORD
-        host = DATABRIDGE_HOST
-        service_name = 'GISDBP'
-        port = 1521
-        dsn = cx_Oracle.makedsn(host, port, service_name)
-        # Connect to database
-        db_connect = cx_Oracle.connect(user, password, dsn, encoding="UTF-8")
-        print('Connected to %s' % db_connect)
-        cursor = db_connect.cursor()
-        return cursor
-
-    cursor = database_connect()
+
+    sde_connection = citygeo_secrets.connect_with_secrets(connect_sde, "SDE","databridge-oracle/hostname")
+    cursor = sde_connection.cursor()
     print("Connected to Oracle!\n")
 
 

sync.py

@@ -18,6 +18,7 @@
 import geopetl
 import requests
 from requests.adapters import HTTPAdapter, Retry
+import citygeo_secrets
 from common import *
 from config import *
 
@@ -30,13 +31,13 @@
 
 # if this is set to true in the config
 if TEST:
-    DEST_DB_CONN_STRING = f'{DEST_DB_ACCOUNT}/{THREEONEONE_PASSWORD}@{DEST_TEST_DSN}'
+    #connect to test DB
+    dest_conn = citygeo_secrets.connect_with_secrets(connect_311_test, "GIS_311","databridge-oracle/hostname-testing")
 else:
-    DEST_DB_CONN_STRING = f'{DEST_DB_ACCOUNT}/{THREEONEONE_PASSWORD}@{DEST_PROD_DSN}'
+    # connect to prod DB
+    dest_conn = citygeo_secrets.connect_with_secrets(connect_311, "GIS_311","databridge-oracle/hostname")
+
 
-# Connect to database
-#print("Connecting to oracle, DNS: {}".format(DEST_DB_CONN_STRING)) 
-dest_conn = cx_Oracle.connect(DEST_DB_CONN_STRING)
 dest_conn.autocommit = True
 # 1200000 ms is a 20 minute timeout for really big queries
 # Nvm, 0 for infinite
@@ -59,10 +60,11 @@ def write_log(msg):
 @click.option('--year_refresh', '-y', default=None, help='Retrieve records that were updated in a specific year, then upsert them. Ex: 2017')
 @click.option('--date_column', '-c', default='LastModifiedDate', help='Date column to select cases by from Salesforce. Default is "LastModifiedDate". You can consider using "CreatedDate" when doing full refreshes.')
 def sync(day_refresh, year_refresh, month_refresh, date_column):
+        salesforce_creds = citygeo_secrets.connect_with_secrets(connect_salesforce, "salesforce API copy" )
         # Connect to Salesforce
-        sf = Salesforce(username=SF_USER, \
-                        password=SF_PASSWORD, \
-                        security_token=SF_TOKEN)
+        sf = Salesforce(username=salesforce_creds.get('login'), \
+                        password=salesforce_creds.get('password'), \
+                        security_token=salesforce_creds.get('token'))
         # supposedly SalesForce() takes a timeout parameter, but it doesn't appear to work.
         # Instead, we can apparently set the tmeout anyway by inserting our own request session
         #session = requests.Session()
@@ -371,4 +373,4 @@ def process_rows(sf_rows):
 
 if __name__ == '__main__':
     sync()
-    print('Done.')
+    print('Done.')