[WIP] set a custom user agent for python requests

chsdi/lib/helpers.py

@@ -62,7 +62,7 @@ def make_geoadmin_url(request, agnostic=False):
     return base_url
 
 
-def resource_exists(path, headers={}):
+def resource_exists(path, headers={'User-Agent': 'mf-geoadmin/python'}):
     try:
         r = requests.head(path, headers=headers)
     except ConnectionError:
@@ -266,7 +266,7 @@ def parseHydroXML(id, root):
 def imagesize_from_metafile(tileUrlBasePath, bvnummer):
     width = None
     height = None
-    headers = {'Referer': 'http://admin.ch'}
+    headers = {'Referer': 'http://admin.ch', 'User-Agent': 'mf-geoadmin/python'}
     metaurl = tileUrlBasePath + '/' + bvnummer + '/tilemapresource.xml'
     s = requests.Session()
     response = s.get(metaurl, headers=headers)

chsdi/templates/htmlpopup/cadastralwebmap_opendata.mako

@@ -50,7 +50,7 @@ ${partials.table_body_cadastral(c, lang, fallbackLang, clickCoord)}
 </tr>
 <tr>
     <td class="cell-left">${_('ch.swisstopo-vd.amtliche-vermessung.shape')}</td><td>
-% if requests.get(shp_url).status_code == 200:
+% if requests.head(shp_url, headers={'User-Agent': 'mf-geoadmin/python'}).status_code == 200:
         <a href="${shp_url}" target="_blank">
             SHP</a>
 % else:
@@ -60,7 +60,7 @@ ${partials.table_body_cadastral(c, lang, fallbackLang, clickCoord)}
 </tr>
 <tr>
     <td class="cell-left">${_('ch.swisstopo-vd.amtliche-vermessung.itf')}</td><td>
-% if requests.get(itf_url).status_code == 200:
+% if requests.head(itf_url, headers={'User-Agent': 'mf-geoadmin/python'}).status_code == 200:
         <a href="${itf_url}" target="_blank">
             ITF</a>
 % else:

chsdi/templates/htmlpopup/windatlas50.mako

@@ -5,7 +5,7 @@ def getAltitude(baseUrl, center):
     fullUrl = 'http:' + baseUrl +'/rest/services/height'
     response = requests.get(
         fullUrl + '?easting=%s&northing=%s&elevation_model=COMB' % (center[0], center[1]),
-        headers={'Referer': fullUrl})
+        headers={'Referer': fullUrl, 'User-Agent': 'mf-geoadmin/python'})
     result = response.json()
     return result['height']
 %>
@@ -23,7 +23,7 @@ grid = Grid(gridSpec.get('extent'), gridSpec.get('resolutionX'), gridSpec.get('r
 extent = grid.cellExtent(col,row)
 center = [(extent[0] + extent[2])/2,(extent[1] + extent[3])/2]
 baseUrl = request.registry.settings['api_url']
-dhm_altitude = int(round(float(getAltitude(baseUrl, center)),0))
+dhm_altitude = int(round(float(getAltitude(request.registry.settings['host'], center)),0))
 center = '2%s, 1%s' % (str(int(round(center[0], 0))), str(int(round(center[1], 0))))
 
 props = c['attributes']
@@ -75,7 +75,7 @@ bottomLeft = coordinates[0]
 topRight = coordinates[2]
 center = [(bottomLeft[0] + topRight[0]) / 2, (bottomLeft[1] + topRight[1]) / 2]
 baseUrl = request.registry.settings['api_url']
-dhm_altitude = int(round(float(getAltitude(baseUrl, center)),0))
+dhm_altitude = int(round(float(getAltitude(request.registry.settings['host'], center)),0))
 center = '2%s, 1%s' % (str(int(round(center[0], 0))), str(int(round(center[1], 0))))
 altitude = int(c['layerBodId'].split('ch.bfe.windenergie-geschwindigkeit_h')[1])
 

chsdi/tests/e2e/test_varnish.py

@@ -47,14 +47,14 @@ class TestHeight(TestVarnish):
     def test_height_no_referer(self):
 
         payload = {'easting': 600000.0, 'northing': 200000.0, '_id': self.hash()}
-        resp = requests.get(self.api_url + '/rest/services/height', params=payload)
+        resp = requests.get(self.api_url + '/rest/services/height', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertEqual(resp.status_code, 403)
 
     def test_height_good_referer(self):
 
         payload = {'easting': 600000.0, 'northing': 200000.0, '_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         resp = requests.get(self.api_url + '/rest/services/height', params=payload, headers=headers)
 
         self.assertEqual(resp.status_code, 200)
@@ -65,29 +65,29 @@ class TestProfile(TestVarnish):
     def test_profile_json_no_referer(self):
 
         payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()}
-        resp = requests.get(self.api_url + '/rest/services/profile.json', params=payload)
+        resp = requests.get(self.api_url + '/rest/services/profile.json', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertEqual(resp.status_code, 403)
 
     def test_profile_json_good_referer(self):
 
         payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         resp = requests.get(self.api_url + '/rest/services/profile.json', params=payload, headers=headers)
 
         self.assertEqual(resp.status_code, 200)
 
     def test_profile_csv_no_referer(self):
 
         payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()}
-        resp = requests.get(self.api_url + '/rest/services/profile.csv', params=payload)
+        resp = requests.get(self.api_url + '/rest/services/profile.csv', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertEqual(resp.status_code, 403)
 
     def test_profile_csv_good_referer(self):
 
         payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         resp = requests.get(self.api_url + '/rest/services/profile.csv', params=payload, headers=headers)
 
         self.assertEqual(resp.status_code, 200)
@@ -98,7 +98,7 @@ class TestLocation(TestVarnish):
     def test_locations_no_referer(self):
 
         payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne', '_id': self.hash()}
-        r = requests.get(self.api_url + '/rest/services/api/SearchServer', params=payload)
+        r = requests.get(self.api_url + '/rest/services/api/SearchServer', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         returned_attrs = r.json()['results'][0]['attrs'].keys()
 
@@ -108,7 +108,7 @@ def test_locations_no_referer(self):
     def test_locations_good_referer(self):
 
         payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne', '_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         r = requests.get(self.api_url + '/rest/services/api/SearchServer', params=payload, headers=headers)
 
@@ -120,7 +120,7 @@ def test_locations_good_referer(self):
     def test_location_cached_no_referer(self):
 
         payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne'}
-        r = requests.get(self.api_url + '/%d/rest/services/api/SearchServer' % self.timestamp(), params=payload)
+        r = requests.get(self.api_url + '/%d/rest/services/api/SearchServer' % self.timestamp(), params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         returned_attrs = r.json()['results'][0]['attrs'].keys()
 
@@ -130,7 +130,7 @@ def test_location_cached_no_referer(self):
     def test_location_cached_good_referer(self):
 
         payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne'}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         r = requests.get(self.api_url + '/%d/rest/services/api/SearchServer' % self.timestamp(), params=payload, headers=headers)
 
@@ -151,21 +151,21 @@ class TestGebaeude(TestVarnish):
     def test_gebaude_no_referer(self):
 
         payload = {'_id': self.hash()}
-        r = requests.get(self.api_url + '/rest/services/ech/MapServer/ch.bfs.gebaeude_wohnungs_register/490830_0', params=payload)
+        r = requests.get(self.api_url + '/rest/services/ech/MapServer/ch.bfs.gebaeude_wohnungs_register/490830_0', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertNotIn('geometry', r.json()['feature'].keys())
 
     def test_find_gebaude_no_referer(self):
 
         payload = {'layer': 'ch.bfs.gebaeude_wohnungs_register', 'searchText': 'berges', 'searchField': 'strname1', '_id': self.hash()}
-        r = requests.get(self.api_url + '/rest/services/ech/MapServer/find', params=payload)
+        r = requests.get(self.api_url + '/rest/services/ech/MapServer/find', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertNotIn('geometry', r.json()['results'][0].keys())
 
     def test_gebaude_good_referer(self):
 
         payload = {'type': 'location', 'searchText': 'dorf', '_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         r = requests.get(self.api_url + '/rest/services/ech/MapServer/ch.bfs.gebaeude_wohnungs_register/490830_0', params=payload, headers=headers)
 
@@ -174,7 +174,7 @@ def test_gebaude_good_referer(self):
     def test_find_gebaude_good_referer(self):
 
         payload = {'layer': 'ch.bfs.gebaeude_wohnungs_register', 'searchText': 'berges', 'searchField': 'strname1', '_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         r = requests.get(self.api_url + '/rest/services/ech/MapServer/find', params=payload, headers=headers)
 
         self.assertIn('geometry', r.json()['results'][0].keys())
@@ -189,14 +189,14 @@ def test_mapproxy_no_referer(self):
 
         payload = {'_id': self.hash()}
 
-        resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload)
+        resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertEqual(resp.status_code, 403)
 
     def test_mapproxy_bad_referer(self):
 
         payload = {'_id': self.hash()}
-        headers = {'referer': 'http://gooffy-referer.ch'}
+        headers = {'referer': 'http://gooffy-referer.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload, headers=headers)
 
@@ -205,7 +205,7 @@ def test_mapproxy_bad_referer(self):
     def test_mapproxy_good_referer(self):
 
         payload = {'_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload, headers=headers)
 
@@ -221,14 +221,14 @@ def test_ogcproxy_no_referer(self):
 
         payload = {'_id': self.hash()}
 
-        resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload)
+        resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertEqual(resp.status_code, 403)
 
     def test_ogcproxy_bad_referer(self):
 
         payload = {'_id': self.hash()}
-        headers = {'referer': 'http://goofy-referer.ch'}
+        headers = {'referer': 'http://goofy-referer.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload, headers=headers)
 
@@ -237,7 +237,7 @@ def test_ogcproxy_bad_referer(self):
     def test_ogcproxy_good_referer(self):
 
         payload = {'_id': self.hash()}
-        headers = {'referer': 'http://unittest.geo.admin.ch'}
+        headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
 
         resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload, headers=headers)
 
@@ -248,13 +248,13 @@ class TestFilestorage(TestVarnish):
 
     def test_post_filestorage_no_referer(self):
 
-        resp = requests.post(self.api_url + '/files', VALID_KML)
+        resp = requests.post(self.api_url + '/files', VALID_KML, headers={'User-Agent': 'mf-geoadmin/python'})
 
         self.assertEqual(resp.status_code, 403)
 
     def test_post_filestorage_good_referer(self):
 
-        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch'}
+        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         resp = requests.post(self.api_url + '/files', VALID_KML, headers=headers)
 
         self.assertEqual(resp.status_code, 200)
@@ -263,28 +263,28 @@ def test_post_filestorage_good_referer(self):
 
     def test_post_filestorage_wrong_referer(self):
 
-        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://foo.bar'}
+        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://foo.bar', 'User-Agent': 'mf-geaodmin/python'}
         resp = requests.post(self.api_url + '/files', VALID_KML, headers=headers)
 
         self.assertEqual(resp.status_code, 403)
 
     def test_post_filestorage_wrong_content_type(self):
 
-        headers = {'Content-Type': 'application/xml', 'Referer': 'http://unittest.geo.admin.ch'}
+        headers = {'Content-Type': 'application/xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         resp = requests.post(self.api_url + '/files', VALID_KML, headers=headers)
 
         self.assertEqual(resp.status_code, 415)
 
     def test_post_filestorage_not_well_formed(self):
 
-        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch'}
+        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         resp = requests.post(self.api_url + '/files', NOT_WELL_FORMED_KML, headers=headers)
 
         self.assertEqual(resp.status_code, 415)
 
     def test_post_filestorage_too_big(self):
 
-        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch'}
+        headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
         current_dir = os.path.dirname(os.path.abspath(__file__))
         with open(os.path.join(current_dir, '../integration', 'big.kml')) as f:
             data = f.read()

chsdi/tests/e2e/test_wms.py

@@ -23,7 +23,7 @@
 
 
 def get_wms_layers():
-    resp = requests.get(api_url + '/rest/services/all/MapServer/layersConfig')
+    resp = requests.get(api_url + '/rest/services/all/MapServer/layersConfig', headers={'User-Agent': 'mf-geoadmin/python'})
     layers = resp.json()
 
     return [layers[k] for k in layers.keys() if layers[k]['type'] == 'wms']
@@ -58,7 +58,7 @@ def check_status_code(url, scheme):
     s = requests.Session()
     a = requests.adapters.HTTPAdapter(max_retries=max_retry)
     s.mount('%s://' % scheme, a)
-    resp = s.get(url)
+    resp = s.get(url, headers={'User-Agent': 'mf-geoadmin/python'})
     assert resp.status_code in [200, 204, 304], resp.status_code
     assert resp.headers['content-type'] in ['image/png', 'image/jpeg'], resp.headers['content-type']
     if geodata_staging == 'prod':

chsdi/tests/e2e/test_wmts.py

@@ -14,23 +14,23 @@
 
 
 def test_getcap_21781():
-    resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.xml?lang=de')
+    resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.xml?lang=de', headers={'User-Agent': 'mf-geoadmin/python'})
 
     words = ['Alpenkonvention', 'Schweiz', 'Verkehrswege ', 'Flachmoor']
     for w in words:
         assert w in resp.text, w
 
 
 def test_getcap_de():
-    resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=de')
+    resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=de', headers={'User-Agent': 'mf-geoadmin/python'})
 
     words = ['Alpenkonvention', 'Schweiz', 'Verkehrswege ', 'Flachmoor']
     for w in words:
         assert w in resp.text, w
 
 
 def test_getcap_fr():
-    resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=fr')
+    resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=fr', headers={'User-Agent': 'mf-geoadmin/python'})
 
     words = ['Convention', 'Suisse', 'voies', u'marécage']
     for w in words:

chsdi/tests/e2e/test_wmtscapabilities_auth.py

@@ -23,10 +23,10 @@ def tearDown(self):
     def check_status_code(self, url, referer, code):
         headers = None
         if referer:
-            headers = {'Referer': referer}
+            headers = {'Referer': referer, 'User-Agent': 'mf-geoadmin/python'}
             resp = requests.get(url, params={'_id': self.mp.hash()}, headers=headers)
         else:
-            resp = requests.get(url, params={'_id': self.mp.hash()})
+            resp = requests.get(url, params={'_id': self.mp.hash()}, headers={'User-Agent': 'mf-geoadmin/python'})
 
         assert (resp.status_code == code), 'Called Url: ' + url + ' [referer: ' + str(referer) + '] with return code: ' + str(resp.status_code)
 

chsdi/tests/e2e/test_wmtsgettile.py

@@ -35,13 +35,13 @@ def rotateUrl(url):
 HEADER_RESULTS = [{
     # NOTE Varnish transforms all non 200 status code into 204 (even 404)!
     'Results': [200, 204, 304],
-    'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://unittest.geo.admin.ch'}
+    'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'}
 }, {
     'Results': [403],
-    'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': None}
+    'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': None, 'User-Agent': 'mf-geaodmin/python'}
 }, {
     'Results': [403],
-    'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://foonogood.ch'}
+    'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://foonogood.ch', 'User-Agent': 'mf-geoadmin/python'}
 }
 ]
 

chsdi/views/admin.py

@@ -33,7 +33,7 @@ def kml_load(api_url='//api3.geo.admin.ch', bucket_name='public.geo.admin.ch'):
     results = table.query_2(bucket__eq=bucket_name, timestamp__beginswith=date, index='bucketTimestampIndex', limit=LIMIT * 4, reverse=True)
     for f in results:
         try:
-            resp = requests.head("http:" + api_url + "/files/" + f['fileId'])
+            resp = requests.head("http:" + api_url + "/files/" + f['fileId'], headers={'User-Agent': 'mf-geoadmin/python'})
             if int(resp.status_code) == 200:
                 fileids.append((f['fileId'], f['adminId'], f['timestamp']))
         except RequestException: