Add unique id key

cobalt-uoft · Apr 17, 2016 · e859cff · e859cff
1 parent f5668fc
commit e859cff
Show file tree

Hide file tree

Showing 3 changed files with 22 additions and 7 deletions.
diff --git a/README.md b/README.md
@@ -471,7 +471,8 @@ uoftscrapers.Athletics
 
 ##### Output format
 ```js
-{  
+{
+  "id": String,
   "date": String,
   "events":[{
     "title": String,

diff --git a/uoftscrapers/scrapers/athletics/utm.py b/uoftscrapers/scrapers/athletics/utm.py
@@ -29,6 +29,7 @@ def scrape(location='.', month=None):
         for tr in calendar.find_all('tr', class_='single-day'):
             for td in tr.find_all('td'):
                 date = td.get('data-date')
+                id_ = UTMAthletics.get_id(date)
 
                 if not UTMAthletics.date_in_month(date, month):
                     continue
@@ -56,13 +57,14 @@ def scrape(location='.', month=None):
                         ('end_time', end)
                     ]))
 
-                athletics[date] = OrderedDict([
+                athletics[id_] = OrderedDict([
+                    ('id', id_),
                     ('date', date),
                     ('events', events)
                 ])
 
-        for date, doc in athletics.items():
-            Scraper.save_json(doc, location, date)
+        for id_, doc in athletics.items():
+            Scraper.save_json(doc, location, id_)
 
         Scraper.logger.info('UTMAthletics completed.')
 
@@ -71,6 +73,11 @@ def get_month(m):
         now = datetime.now()
         return '%s-%s' % (now.year, now.month)
 
+    @staticmethod
+    def get_id(d):
+        day = datetime.strptime(d, '%Y-%m-%d').day
+        return '%s%s' % (str(day).zfill(2), 'M')
+
     @staticmethod
     def date_in_month(d, m):
         d = datetime.strptime(d, '%Y-%m-%d')

diff --git a/uoftscrapers/scrapers/athletics/utsc.py b/uoftscrapers/scrapers/athletics/utsc.py
@@ -29,6 +29,7 @@ def scrape(location='.', month=None):
         for tr in calendar.find_all('tr', class_='single-day'):
             for td in tr.find_all('td'):
                 date = td.get('data-date')
+                id_ = UTSCAthletics.get_id(date)
 
                 if not UTSCAthletics.date_in_month(date, month):
                     continue
@@ -55,13 +56,14 @@ def scrape(location='.', month=None):
                         ('end_time', end)
                     ]))
 
-                athletics[date] = OrderedDict([
+                athletics[id_] = OrderedDict([
+                    ('id', id_),
                     ('date', date),
                     ('events', events)
                 ])
 
-        for date, doc in athletics.items():
-            Scraper.save_json(doc, location, date)
+        for id_, doc in athletics.items():
+            Scraper.save_json(doc, location, id_)
 
         Scraper.logger.info('UTSCAthletics completed.')
 
@@ -70,6 +72,11 @@ def get_month(m):
         now = datetime.now()
         return '%s-%s' % (now.year, now.month)
 
+    @staticmethod
+    def get_id(d):
+        day = datetime.strptime(d, '%Y-%m-%d').day
+        return '%s%s' % (str(day).zfill(2), 'SC')
+
     @staticmethod
     def date_in_month(d, m):
         d = datetime.strptime(d, '%Y-%m-%d')