diff --git a/chsdi/lib/helpers.py b/chsdi/lib/helpers.py index 2a20cf3c34..7ba1aaace3 100644 --- a/chsdi/lib/helpers.py +++ b/chsdi/lib/helpers.py @@ -62,7 +62,7 @@ def make_geoadmin_url(request, agnostic=False): return base_url -def resource_exists(path, headers={}): +def resource_exists(path, headers={'User-Agent': 'mf-geoadmin/python'}): try: r = requests.head(path, headers=headers) except ConnectionError: @@ -266,7 +266,7 @@ def parseHydroXML(id, root): def imagesize_from_metafile(tileUrlBasePath, bvnummer): width = None height = None - headers = {'Referer': 'http://admin.ch'} + headers = {'Referer': 'http://admin.ch', 'User-Agent': 'mf-geoadmin/python'} metaurl = tileUrlBasePath + '/' + bvnummer + '/tilemapresource.xml' s = requests.Session() response = s.get(metaurl, headers=headers) diff --git a/chsdi/templates/htmlpopup/cadastralwebmap_opendata.mako b/chsdi/templates/htmlpopup/cadastralwebmap_opendata.mako index 738eb7f5c1..11c2a84a85 100644 --- a/chsdi/templates/htmlpopup/cadastralwebmap_opendata.mako +++ b/chsdi/templates/htmlpopup/cadastralwebmap_opendata.mako @@ -50,7 +50,7 @@ ${partials.table_body_cadastral(c, lang, fallbackLang, clickCoord)} ${_('ch.swisstopo-vd.amtliche-vermessung.shape')} -% if requests.get(shp_url).status_code == 200: +% if requests.head(shp_url, headers={'User-Agent': 'mf-geoadmin/python'}).status_code == 200: SHP % else: @@ -60,7 +60,7 @@ ${partials.table_body_cadastral(c, lang, fallbackLang, clickCoord)} ${_('ch.swisstopo-vd.amtliche-vermessung.itf')} -% if requests.get(itf_url).status_code == 200: +% if requests.head(itf_url, headers={'User-Agent': 'mf-geoadmin/python'}).status_code == 200: ITF % else: diff --git a/chsdi/templates/htmlpopup/windatlas50.mako b/chsdi/templates/htmlpopup/windatlas50.mako index 829e8dc63a..4a51f1dd71 100644 --- a/chsdi/templates/htmlpopup/windatlas50.mako +++ b/chsdi/templates/htmlpopup/windatlas50.mako @@ -5,7 +5,7 @@ def getAltitude(baseUrl, center): fullUrl = 'http:' + baseUrl +'/rest/services/height' response = requests.get( fullUrl + '?easting=%s&northing=%s&elevation_model=COMB' % (center[0], center[1]), - headers={'Referer': fullUrl}) + headers={'Referer': fullUrl, 'User-Agent': 'mf-geoadmin/python'}) result = response.json() return result['height'] %> @@ -23,7 +23,7 @@ grid = Grid(gridSpec.get('extent'), gridSpec.get('resolutionX'), gridSpec.get('r extent = grid.cellExtent(col,row) center = [(extent[0] + extent[2])/2,(extent[1] + extent[3])/2] baseUrl = request.registry.settings['api_url'] -dhm_altitude = int(round(float(getAltitude(baseUrl, center)),0)) +dhm_altitude = int(round(float(getAltitude(request.registry.settings['host'], center)),0)) center = '2%s, 1%s' % (str(int(round(center[0], 0))), str(int(round(center[1], 0)))) props = c['attributes'] @@ -75,7 +75,7 @@ bottomLeft = coordinates[0] topRight = coordinates[2] center = [(bottomLeft[0] + topRight[0]) / 2, (bottomLeft[1] + topRight[1]) / 2] baseUrl = request.registry.settings['api_url'] -dhm_altitude = int(round(float(getAltitude(baseUrl, center)),0)) +dhm_altitude = int(round(float(getAltitude(request.registry.settings['host'], center)),0)) center = '2%s, 1%s' % (str(int(round(center[0], 0))), str(int(round(center[1], 0)))) altitude = int(c['layerBodId'].split('ch.bfe.windenergie-geschwindigkeit_h')[1]) diff --git a/chsdi/tests/e2e/test_varnish.py b/chsdi/tests/e2e/test_varnish.py index 46ac93c3f0..9f556fdea0 100644 --- a/chsdi/tests/e2e/test_varnish.py +++ b/chsdi/tests/e2e/test_varnish.py @@ -47,14 +47,14 @@ class TestHeight(TestVarnish): def test_height_no_referer(self): payload = {'easting': 600000.0, 'northing': 200000.0, '_id': self.hash()} - resp = requests.get(self.api_url + '/rest/services/height', params=payload) + resp = requests.get(self.api_url + '/rest/services/height', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertEqual(resp.status_code, 403) def test_height_good_referer(self): payload = {'easting': 600000.0, 'northing': 200000.0, '_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.api_url + '/rest/services/height', params=payload, headers=headers) self.assertEqual(resp.status_code, 200) @@ -65,14 +65,14 @@ class TestProfile(TestVarnish): def test_profile_json_no_referer(self): payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()} - resp = requests.get(self.api_url + '/rest/services/profile.json', params=payload) + resp = requests.get(self.api_url + '/rest/services/profile.json', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertEqual(resp.status_code, 403) def test_profile_json_good_referer(self): payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.api_url + '/rest/services/profile.json', params=payload, headers=headers) self.assertEqual(resp.status_code, 200) @@ -80,14 +80,14 @@ def test_profile_json_good_referer(self): def test_profile_csv_no_referer(self): payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()} - resp = requests.get(self.api_url + '/rest/services/profile.csv', params=payload) + resp = requests.get(self.api_url + '/rest/services/profile.csv', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertEqual(resp.status_code, 403) def test_profile_csv_good_referer(self): payload = {'geom': '{"type":"LineString","coordinates":[[550050,206550],[556950,204150],[561050,207950]]}', '_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.api_url + '/rest/services/profile.csv', params=payload, headers=headers) self.assertEqual(resp.status_code, 200) @@ -98,7 +98,7 @@ class TestLocation(TestVarnish): def test_locations_no_referer(self): payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne', '_id': self.hash()} - r = requests.get(self.api_url + '/rest/services/api/SearchServer', params=payload) + r = requests.get(self.api_url + '/rest/services/api/SearchServer', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) returned_attrs = r.json()['results'][0]['attrs'].keys() @@ -108,7 +108,7 @@ def test_locations_no_referer(self): def test_locations_good_referer(self): payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne', '_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} r = requests.get(self.api_url + '/rest/services/api/SearchServer', params=payload, headers=headers) @@ -120,7 +120,7 @@ def test_locations_good_referer(self): def test_location_cached_no_referer(self): payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne'} - r = requests.get(self.api_url + '/%d/rest/services/api/SearchServer' % self.timestamp(), params=payload) + r = requests.get(self.api_url + '/%d/rest/services/api/SearchServer' % self.timestamp(), params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) returned_attrs = r.json()['results'][0]['attrs'].keys() @@ -130,7 +130,7 @@ def test_location_cached_no_referer(self): def test_location_cached_good_referer(self): payload = {'type': 'locations', 'searchText': 'fontenay 10 lausanne'} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} r = requests.get(self.api_url + '/%d/rest/services/api/SearchServer' % self.timestamp(), params=payload, headers=headers) @@ -151,21 +151,21 @@ class TestGebaeude(TestVarnish): def test_gebaude_no_referer(self): payload = {'_id': self.hash()} - r = requests.get(self.api_url + '/rest/services/ech/MapServer/ch.bfs.gebaeude_wohnungs_register/490830_0', params=payload) + r = requests.get(self.api_url + '/rest/services/ech/MapServer/ch.bfs.gebaeude_wohnungs_register/490830_0', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertNotIn('geometry', r.json()['feature'].keys()) def test_find_gebaude_no_referer(self): payload = {'layer': 'ch.bfs.gebaeude_wohnungs_register', 'searchText': 'berges', 'searchField': 'strname1', '_id': self.hash()} - r = requests.get(self.api_url + '/rest/services/ech/MapServer/find', params=payload) + r = requests.get(self.api_url + '/rest/services/ech/MapServer/find', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertNotIn('geometry', r.json()['results'][0].keys()) def test_gebaude_good_referer(self): payload = {'type': 'location', 'searchText': 'dorf', '_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} r = requests.get(self.api_url + '/rest/services/ech/MapServer/ch.bfs.gebaeude_wohnungs_register/490830_0', params=payload, headers=headers) @@ -174,7 +174,7 @@ def test_gebaude_good_referer(self): def test_find_gebaude_good_referer(self): payload = {'layer': 'ch.bfs.gebaeude_wohnungs_register', 'searchText': 'berges', 'searchField': 'strname1', '_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} r = requests.get(self.api_url + '/rest/services/ech/MapServer/find', params=payload, headers=headers) self.assertIn('geometry', r.json()['results'][0].keys()) @@ -189,14 +189,14 @@ def test_mapproxy_no_referer(self): payload = {'_id': self.hash()} - resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload) + resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertEqual(resp.status_code, 403) def test_mapproxy_bad_referer(self): payload = {'_id': self.hash()} - headers = {'referer': 'http://gooffy-referer.ch'} + headers = {'referer': 'http://gooffy-referer.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload, headers=headers) @@ -205,7 +205,7 @@ def test_mapproxy_bad_referer(self): def test_mapproxy_good_referer(self): payload = {'_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.mapproxy_url + '/1.0.0/ch.swisstopo.pixelkarte-farbe/default/current/3857/13/4265/2883.jpeg', params=payload, headers=headers) @@ -221,14 +221,14 @@ def test_ogcproxy_no_referer(self): payload = {'_id': self.hash()} - resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload) + resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertEqual(resp.status_code, 403) def test_ogcproxy_bad_referer(self): payload = {'_id': self.hash()} - headers = {'referer': 'http://goofy-referer.ch'} + headers = {'referer': 'http://goofy-referer.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload, headers=headers) @@ -237,7 +237,7 @@ def test_ogcproxy_bad_referer(self): def test_ogcproxy_good_referer(self): payload = {'_id': self.hash()} - headers = {'referer': 'http://unittest.geo.admin.ch'} + headers = {'referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(self.api_url + '/ogcproxy?url=http%3A%2F%2Fmapserver1.gr.ch%2Fwms%2Fadmineinteilung%3FSERVICE%3DWMS%26REQUEST%3DGetCapabilities%26VERSION%3D1.3.0', params=payload, headers=headers) @@ -248,13 +248,13 @@ class TestFilestorage(TestVarnish): def test_post_filestorage_no_referer(self): - resp = requests.post(self.api_url + '/files', VALID_KML) + resp = requests.post(self.api_url + '/files', VALID_KML, headers={'User-Agent': 'mf-geoadmin/python'}) self.assertEqual(resp.status_code, 403) def test_post_filestorage_good_referer(self): - headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch'} + headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.post(self.api_url + '/files', VALID_KML, headers=headers) self.assertEqual(resp.status_code, 200) @@ -263,28 +263,28 @@ def test_post_filestorage_good_referer(self): def test_post_filestorage_wrong_referer(self): - headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://foo.bar'} + headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://foo.bar', 'User-Agent': 'mf-geaodmin/python'} resp = requests.post(self.api_url + '/files', VALID_KML, headers=headers) self.assertEqual(resp.status_code, 403) def test_post_filestorage_wrong_content_type(self): - headers = {'Content-Type': 'application/xml', 'Referer': 'http://unittest.geo.admin.ch'} + headers = {'Content-Type': 'application/xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.post(self.api_url + '/files', VALID_KML, headers=headers) self.assertEqual(resp.status_code, 415) def test_post_filestorage_not_well_formed(self): - headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch'} + headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} resp = requests.post(self.api_url + '/files', NOT_WELL_FORMED_KML, headers=headers) self.assertEqual(resp.status_code, 415) def test_post_filestorage_too_big(self): - headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch'} + headers = {'Content-Type': 'application/vnd.google-earth.kml+xml', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} current_dir = os.path.dirname(os.path.abspath(__file__)) with open(os.path.join(current_dir, '../integration', 'big.kml')) as f: data = f.read() diff --git a/chsdi/tests/e2e/test_wms.py b/chsdi/tests/e2e/test_wms.py index 089da74129..1985b30dd8 100644 --- a/chsdi/tests/e2e/test_wms.py +++ b/chsdi/tests/e2e/test_wms.py @@ -23,7 +23,7 @@ def get_wms_layers(): - resp = requests.get(api_url + '/rest/services/all/MapServer/layersConfig') + resp = requests.get(api_url + '/rest/services/all/MapServer/layersConfig', headers={'User-Agent': 'mf-geoadmin/python'}) layers = resp.json() return [layers[k] for k in layers.keys() if layers[k]['type'] == 'wms'] @@ -58,7 +58,7 @@ def check_status_code(url, scheme): s = requests.Session() a = requests.adapters.HTTPAdapter(max_retries=max_retry) s.mount('%s://' % scheme, a) - resp = s.get(url) + resp = s.get(url, headers={'User-Agent': 'mf-geoadmin/python'}) assert resp.status_code in [200, 204, 304], resp.status_code assert resp.headers['content-type'] in ['image/png', 'image/jpeg'], resp.headers['content-type'] if geodata_staging == 'prod': diff --git a/chsdi/tests/e2e/test_wmts.py b/chsdi/tests/e2e/test_wmts.py index ad4dc24581..74c3b46506 100644 --- a/chsdi/tests/e2e/test_wmts.py +++ b/chsdi/tests/e2e/test_wmts.py @@ -14,7 +14,7 @@ def test_getcap_21781(): - resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.xml?lang=de') + resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.xml?lang=de', headers={'User-Agent': 'mf-geoadmin/python'}) words = ['Alpenkonvention', 'Schweiz', 'Verkehrswege ', 'Flachmoor'] for w in words: @@ -22,7 +22,7 @@ def test_getcap_21781(): def test_getcap_de(): - resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=de') + resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=de', headers={'User-Agent': 'mf-geoadmin/python'}) words = ['Alpenkonvention', 'Schweiz', 'Verkehrswege ', 'Flachmoor'] for w in words: @@ -30,7 +30,7 @@ def test_getcap_de(): def test_getcap_fr(): - resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=fr') + resp = requests.get(api_url + '/1.0.0/WMTSCapabilities.EPSG.4326.xml?lang=fr', headers={'User-Agent': 'mf-geoadmin/python'}) words = ['Convention', 'Suisse', 'voies', u'marécage'] for w in words: diff --git a/chsdi/tests/e2e/test_wmtscapabilities_auth.py b/chsdi/tests/e2e/test_wmtscapabilities_auth.py index 56ab21e522..f5c3bf27b4 100644 --- a/chsdi/tests/e2e/test_wmtscapabilities_auth.py +++ b/chsdi/tests/e2e/test_wmtscapabilities_auth.py @@ -23,10 +23,10 @@ def tearDown(self): def check_status_code(self, url, referer, code): headers = None if referer: - headers = {'Referer': referer} + headers = {'Referer': referer, 'User-Agent': 'mf-geoadmin/python'} resp = requests.get(url, params={'_id': self.mp.hash()}, headers=headers) else: - resp = requests.get(url, params={'_id': self.mp.hash()}) + resp = requests.get(url, params={'_id': self.mp.hash()}, headers={'User-Agent': 'mf-geoadmin/python'}) assert (resp.status_code == code), 'Called Url: ' + url + ' [referer: ' + str(referer) + '] with return code: ' + str(resp.status_code) diff --git a/chsdi/tests/e2e/test_wmtsgettile.py b/chsdi/tests/e2e/test_wmtsgettile.py index 15912af572..5c3d7026c9 100644 --- a/chsdi/tests/e2e/test_wmtsgettile.py +++ b/chsdi/tests/e2e/test_wmtsgettile.py @@ -35,13 +35,13 @@ def rotateUrl(url): HEADER_RESULTS = [{ # NOTE Varnish transforms all non 200 status code into 204 (even 404)! 'Results': [200, 204, 304], - 'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://unittest.geo.admin.ch'} + 'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://unittest.geo.admin.ch', 'User-Agent': 'mf-geoadmin/python'} }, { 'Results': [403], - 'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': None} + 'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': None, 'User-Agent': 'mf-geaodmin/python'} }, { 'Results': [403], - 'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://foonogood.ch'} + 'Header': {'User-Agent': 'WMTS Unit Tester v0.0.1', 'Referer': 'http://foonogood.ch', 'User-Agent': 'mf-geoadmin/python'} } ] diff --git a/chsdi/views/admin.py b/chsdi/views/admin.py index 87cf51f04f..12cac229ab 100644 --- a/chsdi/views/admin.py +++ b/chsdi/views/admin.py @@ -33,7 +33,7 @@ def kml_load(api_url='//api3.geo.admin.ch', bucket_name='public.geo.admin.ch'): results = table.query_2(bucket__eq=bucket_name, timestamp__beginswith=date, index='bucketTimestampIndex', limit=LIMIT * 4, reverse=True) for f in results: try: - resp = requests.head("http:" + api_url + "/files/" + f['fileId']) + resp = requests.head("http:" + api_url + "/files/" + f['fileId'], headers={'User-Agent': 'mf-geoadmin/python'}) if int(resp.status_code) == 200: fileids.append((f['fileId'], f['adminId'], f['timestamp'])) except RequestException: