From d0cd0e692475bae48d2197e1e44dd8227fa082d9 Mon Sep 17 00:00:00 2001 From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk> Date: Mon, 2 Nov 2015 09:17:03 +0000 Subject: [PATCH 1/4] Fix order of lots SaaS and PaaS were the wrong way round --- dmscripts/generate_framework_agreement_data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py index 07dc87664df..51dcde6c919 100644 --- a/dmscripts/generate_framework_agreement_data.py +++ b/dmscripts/generate_framework_agreement_data.py @@ -29,8 +29,8 @@ def __init__(self, declaration, lots): self.contact_email = declaration[19] self.lot1 = "Lot 1: Infrastructure as a Service (IaaS)" if int(lots[3]) > 0 else "" - self.lot2 = "Lot 2: Software as a Service (SaaS)" if int(lots[7]) > 0 else "" - self.lot3 = "Lot 3: Platform as a Service (PaaS)" if int(lots[5]) > 0 else "" + self.lot2 = "Lot 2: Platform as a Service (PaaS)" if int(lots[5]) > 0 else "" + self.lot3 = "Lot 3: Software as a Service (SaaS)" if int(lots[7]) > 0 else "" self.lot4 = "Lot 4: Specialist Cloud Services (SCS)" if int(lots[9]) > 0 else "" def __str__(self): From 083275b062381fa101100de4871b9e478ecddae1 Mon Sep 17 00:00:00 2001 From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk> Date: Mon, 2 Nov 2015 13:41:43 +0000 Subject: [PATCH 2/4] Also create output for failed suppliers Late-breaking requirement from CCS is to also have a file for failed suppliers for generating letters. --- .../generate_framework_agreement_data.py | 123 +++++++++++------- 1 file changed, 73 insertions(+), 50 deletions(-) diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py index 51dcde6c919..2ad84972526 100644 --- a/dmscripts/generate_framework_agreement_data.py +++ b/dmscripts/generate_framework_agreement_data.py @@ -39,6 +39,13 @@ def __str__(self): self.company_number, self.registered_office_address, self.contact_name, self.contact_email) +class FailedSupplier: + + def __init__(self, declaration): + self.supplier_id = declaration[0] + self.registered_company_name = declaration[20] + + def read_csv(filepath): all_rows = [] with open(filepath, 'r') as csvfile: @@ -60,10 +67,7 @@ def make_filename_key(supplier_name, supplier_id): def supplier_is_on_framework(client, supplier_id): try: framework_interest = client.get_supplier_framework_info(supplier_id, 'g-cloud-7') - if framework_interest['frameworkInterest']['onFramework']: - return True - else: - return False + return framework_interest['frameworkInterest']['onFramework'] except HTTPError as e: print("ERROR checking if supplier {} is on framework: {}".format(supplier_id, str(e))) return False @@ -73,57 +77,76 @@ def build_framework_agreements(client, declarations, lots, output_dir): if not os.path.exists(output_dir): os.makedirs(output_dir) with open('{}/g7-framework-data.tsv'.format(output_dir), 'w') as csvfile: - # This defines the order of the fields - fields can be in any order in - # the dictionary for each row and will be mapped to the order defined here. - fieldnames = [ - 'Key', - 'Supplier ID', - 'Registered Company Name', - 'Country of Registration', - 'Registered Company Number', - 'Registered Address', - 'Framework Contact Name', - 'Framework Contact Email address', - 'Lot1', - 'Lot2', - 'Lot3', - 'Lot4', - 'Lot1Letter', - 'Lot2Letter', - 'Lot3Letter', - 'Lot4Letter', - ] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab') - writer.writeheader() - - for declaration in declarations: - supplier_id = declaration[0] - if supplier_is_on_framework(client, supplier_id): - lot_count = lot_counts_for_supplier_id(lots, supplier_id) - if lot_count: - supplier = Supplier(declaration, lot_count) + with open('{}/g7-fail-data.tsv'.format(output_dir), 'w') as failfile: + # This defines the order of the fields - fields can be in any order in + # the dictionary for each row and will be mapped to the order defined here. + fieldnames = [ + 'Key', + 'Supplier ID', + 'Registered Company Name', + 'Country of Registration', + 'Registered Company Number', + 'Registered Address', + 'Framework Contact Name', + 'Framework Contact Email address', + 'Lot1', + 'Lot2', + 'Lot3', + 'Lot4', + 'Lot1Letter', + 'Lot2Letter', + 'Lot3Letter', + 'Lot4Letter', + ] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab') + writer.writeheader() + + fail_fieldnames = [ + 'Key', + 'Supplier ID', + 'Registered Company Name' + ] + fail_writer = csv.DictWriter(failfile, fieldnames=fail_fieldnames, dialect='excel-tab') + fail_writer.writeheader() + + for declaration in declarations: + supplier_id = declaration[0] + on_framework = supplier_is_on_framework(client, supplier_id) + if on_framework is True: + lot_count = lot_counts_for_supplier_id(lots, supplier_id) + if lot_count: + supplier = Supplier(declaration, lot_count) + row = { + 'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id), + 'Supplier ID': supplier.supplier_id, + 'Registered Company Name': supplier.registered_company_name, + 'Country of Registration': supplier.country_of_registration, + 'Registered Company Number': supplier.company_number, + 'Registered Address': supplier.registered_office_address, + 'Framework Contact Name': supplier.contact_name, + 'Framework Contact Email address': supplier.contact_email, + 'Lot1': supplier.lot1, + 'Lot2': supplier.lot2, + 'Lot3': supplier.lot3, + 'Lot4': supplier.lot4, + 'Lot1Letter': "Pass" if supplier.lot1 else "No bid", + 'Lot2Letter': "Pass" if supplier.lot2 else "No bid", + 'Lot3Letter': "Pass" if supplier.lot3 else "No bid", + 'Lot4Letter': "Pass" if supplier.lot4 else "No bid", + } + writer.writerow(row) + elif on_framework is False: + print("Failed supplier: {}".format(supplier_id)) + supplier = FailedSupplier(declaration) row = { 'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id), 'Supplier ID': supplier.supplier_id, 'Registered Company Name': supplier.registered_company_name, - 'Country of Registration': supplier.country_of_registration, - 'Registered Company Number': supplier.company_number, - 'Registered Address': supplier.registered_office_address, - 'Framework Contact Name': supplier.contact_name, - 'Framework Contact Email address': supplier.contact_email, - 'Lot1': supplier.lot1, - 'Lot2': supplier.lot2, - 'Lot3': supplier.lot3, - 'Lot4': supplier.lot4, - 'Lot1Letter': "Pass" if supplier.lot1 else "No bid", - 'Lot2Letter': "Pass" if supplier.lot2 else "No bid", - 'Lot3Letter': "Pass" if supplier.lot3 else "No bid", - 'Lot4Letter': "Pass" if supplier.lot4 else "No bid", } - writer.writerow(row) - else: - print("Skipping supplier not on framework: {}".format(supplier_id)) - continue + fail_writer.writerow(row) + else: + print("Supplier did not apply: {}".format(supplier_id)) + continue def lot_counts_for_supplier_id(lot_counts, supplier_id): From 434a59e22be0a52c660776546cd420eb630aa224 Mon Sep 17 00:00:00 2001 From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk> Date: Mon, 2 Nov 2015 13:42:55 +0000 Subject: [PATCH 3/4] Put supplier name first in file key --- dmscripts/generate_framework_agreement_data.py | 4 ++-- tests/test_generate_framework_agreement_data.py | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py index 2ad84972526..f1a69b1317b 100644 --- a/dmscripts/generate_framework_agreement_data.py +++ b/dmscripts/generate_framework_agreement_data.py @@ -10,7 +10,7 @@ # List of bad characters taken from: http://www.mtu.edu/umc/services/web/cms/characters-avoid/ BAD_FILENAME_CHARACTERS = ['#', '%', '&', '{', '}', '\\', '<', '>', '*', '?', '/', - '$', '!', "'", '"', ':', '@', '+', '`', '|', '='] + '$', '!', "'", '"', ':', '@', '+', '`', '|', '=', ',', '.'] class Supplier: @@ -61,7 +61,7 @@ def make_filename_key(supplier_name, supplier_id): sanitised_supplier_name = sanitised_supplier_name.replace(bad_char, '') while '__' in sanitised_supplier_name: sanitised_supplier_name = sanitised_supplier_name.replace('__', '_') - return "{}-{}".format(supplier_id, sanitised_supplier_name) + return "{}-{}".format(sanitised_supplier_name, supplier_id) def supplier_is_on_framework(client, supplier_id): diff --git a/tests/test_generate_framework_agreement_data.py b/tests/test_generate_framework_agreement_data.py index 69f52671575..f3a1e8b2244 100644 --- a/tests/test_generate_framework_agreement_data.py +++ b/tests/test_generate_framework_agreement_data.py @@ -26,11 +26,11 @@ def test_read_csv(): def test_make_filename_key(): - assert make_filename_key('Kev\'s Butties', 1234) == '1234-Kevs_Butties' - assert make_filename_key(' Supplier A ', 1234) == '1234-Supplier_A' - assert make_filename_key('Kev & Sons. | Ltd', 1234) == '1234-Kev_and_Sons._Ltd' - assert make_filename_key('\ / : * ? \' " < > |', 1234) == '1234-_' - assert make_filename_key('kev@the*agency', 1234) == '1234-kevtheagency' + assert make_filename_key('Kev\'s Butties', 1234) == 'Kevs_Butties-1234' + assert make_filename_key(' Supplier A ', 1234) == 'Supplier_A-1234' + assert make_filename_key('Kev & Sons. | Ltd', 1234) == 'Kev_and_Sons_Ltd-1234' + assert make_filename_key('\ / : * ? \' " < > |', 1234) == '_-1234' + assert make_filename_key('kev@the*agency', 1234) == 'kevtheagency-1234' def test_supplier_is_on_framework(mock_data_client): @@ -41,7 +41,7 @@ def test_supplier_is_on_framework(mock_data_client): assert supplier_is_on_framework(mock_data_client, 123) is False mock_data_client.get_supplier_framework_info.return_value = {'frameworkInterest': {'onFramework': None}} - assert supplier_is_on_framework(mock_data_client, 123) is False + assert supplier_is_on_framework(mock_data_client, 123) is None mock_data_client.get_supplier_framework_info.side_effect = HTTPError() assert supplier_is_on_framework(mock_data_client, 123) is False From 320db1c2062b71d661308e0f365c93b88a34324c Mon Sep 17 00:00:00 2001 From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk> Date: Tue, 3 Nov 2015 10:03:33 +0000 Subject: [PATCH 4/4] Use utils method to sanitise supplier name --- dmscripts/generate_framework_agreement_data.py | 12 ++---------- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py index f1a69b1317b..f16b42376ea 100644 --- a/dmscripts/generate_framework_agreement_data.py +++ b/dmscripts/generate_framework_agreement_data.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- import os from dmutils.apiclient.errors import HTTPError +from dmutils.documents import sanitise_supplier_name import sys if sys.version_info > (3, 0): @@ -8,10 +9,6 @@ else: import unicodecsv as csv -# List of bad characters taken from: http://www.mtu.edu/umc/services/web/cms/characters-avoid/ -BAD_FILENAME_CHARACTERS = ['#', '%', '&', '{', '}', '\\', '<', '>', '*', '?', '/', - '$', '!', "'", '"', ':', '@', '+', '`', '|', '=', ',', '.'] - class Supplier: @@ -56,12 +53,7 @@ def read_csv(filepath): def make_filename_key(supplier_name, supplier_id): - sanitised_supplier_name = supplier_name.strip().replace(' ', '_').replace('&', 'and') - for bad_char in BAD_FILENAME_CHARACTERS: - sanitised_supplier_name = sanitised_supplier_name.replace(bad_char, '') - while '__' in sanitised_supplier_name: - sanitised_supplier_name = sanitised_supplier_name.replace('__', '_') - return "{}-{}".format(sanitised_supplier_name, supplier_id) + return "{}-{}".format(sanitise_supplier_name(supplier_name), supplier_id) def supplier_is_on_framework(client, supplier_id): diff --git a/requirements.txt b/requirements.txt index 5e061da83f6..3b6aea67a2a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ git+https://github.com/madzak/python-json-logger.git@v0.1.3#egg=python-json-logger==v0.1.3 -git+https://github.com/alphagov/digitalmarketplace-utils.git@11.1.2#egg=digitalmarketplace-utils==11.1.2 +git+https://github.com/alphagov/digitalmarketplace-utils.git@13.0.0#egg=digitalmarketplace-utils==13.0.0 unicodecsv==0.14.1