Skip to content

Commit

Permalink
Merge pull request #8 from alphagov/fix-lots-in-export
Browse files Browse the repository at this point in the history 
Fix lots in export
  • Loading branch information
@TheDoubleK
TheDoubleK committed Nov 3, 2015
2 parents ea6ced3 + 320db1c commit dac6690
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 69 deletions.
139 changes: 77 additions & 62 deletions dmscripts/generate_framework_agreement_data.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,14 @@
# -*- coding: utf-8 -*-
import os
from dmutils.apiclient.errors import HTTPError
from dmutils.documents import sanitise_supplier_name

import sys
if sys.version_info > (3, 0):
import csv
else:
import unicodecsv as csv

# List of bad characters taken from: http://www.mtu.edu/umc/services/web/cms/characters-avoid/
BAD_FILENAME_CHARACTERS = ['#', '%', '&', '{', '}', '\\', '<', '>', '*', '?', '/',
'$', '!', "'", '"', ':', '@', '+', '`', '|', '=']


class Supplier:

Expand All @@ -29,8 +26,8 @@ def __init__(self, declaration, lots):
self.contact_email = declaration[19]

self.lot1 = "Lot 1: Infrastructure as a Service (IaaS)" if int(lots[3]) > 0 else ""
self.lot2 = "Lot 2: Software as a Service (SaaS)" if int(lots[7]) > 0 else ""
self.lot3 = "Lot 3: Platform as a Service (PaaS)" if int(lots[5]) > 0 else ""
self.lot2 = "Lot 2: Platform as a Service (PaaS)" if int(lots[5]) > 0 else ""
self.lot3 = "Lot 3: Software as a Service (SaaS)" if int(lots[7]) > 0 else ""
self.lot4 = "Lot 4: Specialist Cloud Services (SCS)" if int(lots[9]) > 0 else ""

def __str__(self):
Expand All @@ -39,6 +36,13 @@ def __str__(self):
self.company_number, self.registered_office_address, self.contact_name, self.contact_email)


class FailedSupplier:

def __init__(self, declaration):
self.supplier_id = declaration[0]
self.registered_company_name = declaration[20]


def read_csv(filepath):
all_rows = []
with open(filepath, 'r') as csvfile:
Expand All @@ -49,21 +53,13 @@ def read_csv(filepath):


def make_filename_key(supplier_name, supplier_id):
sanitised_supplier_name = supplier_name.strip().replace(' ', '_').replace('&', 'and')
for bad_char in BAD_FILENAME_CHARACTERS:
sanitised_supplier_name = sanitised_supplier_name.replace(bad_char, '')
while '__' in sanitised_supplier_name:
sanitised_supplier_name = sanitised_supplier_name.replace('__', '_')
return "{}-{}".format(supplier_id, sanitised_supplier_name)
return "{}-{}".format(sanitise_supplier_name(supplier_name), supplier_id)


def supplier_is_on_framework(client, supplier_id):
try:
framework_interest = client.get_supplier_framework_info(supplier_id, 'g-cloud-7')
if framework_interest['frameworkInterest']['onFramework']:
return True
else:
return False
return framework_interest['frameworkInterest']['onFramework']
except HTTPError as e:
print("ERROR checking if supplier {} is on framework: {}".format(supplier_id, str(e)))
return False
Expand All @@ -73,57 +69,76 @@ def build_framework_agreements(client, declarations, lots, output_dir):
if not os.path.exists(output_dir):
os.makedirs(output_dir)
with open('{}/g7-framework-data.tsv'.format(output_dir), 'w') as csvfile:
# This defines the order of the fields - fields can be in any order in
# the dictionary for each row and will be mapped to the order defined here.
fieldnames = [
'Key',
'Supplier ID',
'Registered Company Name',
'Country of Registration',
'Registered Company Number',
'Registered Address',
'Framework Contact Name',
'Framework Contact Email address',
'Lot1',
'Lot2',
'Lot3',
'Lot4',
'Lot1Letter',
'Lot2Letter',
'Lot3Letter',
'Lot4Letter',
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab')
writer.writeheader()

for declaration in declarations:
supplier_id = declaration[0]
if supplier_is_on_framework(client, supplier_id):
lot_count = lot_counts_for_supplier_id(lots, supplier_id)
if lot_count:
supplier = Supplier(declaration, lot_count)
with open('{}/g7-fail-data.tsv'.format(output_dir), 'w') as failfile:
# This defines the order of the fields - fields can be in any order in
# the dictionary for each row and will be mapped to the order defined here.
fieldnames = [
'Key',
'Supplier ID',
'Registered Company Name',
'Country of Registration',
'Registered Company Number',
'Registered Address',
'Framework Contact Name',
'Framework Contact Email address',
'Lot1',
'Lot2',
'Lot3',
'Lot4',
'Lot1Letter',
'Lot2Letter',
'Lot3Letter',
'Lot4Letter',
]
writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab')
writer.writeheader()

fail_fieldnames = [
'Key',
'Supplier ID',
'Registered Company Name'
]
fail_writer = csv.DictWriter(failfile, fieldnames=fail_fieldnames, dialect='excel-tab')
fail_writer.writeheader()

for declaration in declarations:
supplier_id = declaration[0]
on_framework = supplier_is_on_framework(client, supplier_id)
if on_framework is True:
lot_count = lot_counts_for_supplier_id(lots, supplier_id)
if lot_count:
supplier = Supplier(declaration, lot_count)
row = {
'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id),
'Supplier ID': supplier.supplier_id,
'Registered Company Name': supplier.registered_company_name,
'Country of Registration': supplier.country_of_registration,
'Registered Company Number': supplier.company_number,
'Registered Address': supplier.registered_office_address,
'Framework Contact Name': supplier.contact_name,
'Framework Contact Email address': supplier.contact_email,
'Lot1': supplier.lot1,
'Lot2': supplier.lot2,
'Lot3': supplier.lot3,
'Lot4': supplier.lot4,
'Lot1Letter': "Pass" if supplier.lot1 else "No bid",
'Lot2Letter': "Pass" if supplier.lot2 else "No bid",
'Lot3Letter': "Pass" if supplier.lot3 else "No bid",
'Lot4Letter': "Pass" if supplier.lot4 else "No bid",
}
writer.writerow(row)
elif on_framework is False:
print("Failed supplier: {}".format(supplier_id))
supplier = FailedSupplier(declaration)
row = {
'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id),
'Supplier ID': supplier.supplier_id,
'Registered Company Name': supplier.registered_company_name,
'Country of Registration': supplier.country_of_registration,
'Registered Company Number': supplier.company_number,
'Registered Address': supplier.registered_office_address,
'Framework Contact Name': supplier.contact_name,
'Framework Contact Email address': supplier.contact_email,
'Lot1': supplier.lot1,
'Lot2': supplier.lot2,
'Lot3': supplier.lot3,
'Lot4': supplier.lot4,
'Lot1Letter': "Pass" if supplier.lot1 else "No bid",
'Lot2Letter': "Pass" if supplier.lot2 else "No bid",
'Lot3Letter': "Pass" if supplier.lot3 else "No bid",
'Lot4Letter': "Pass" if supplier.lot4 else "No bid",
}
writer.writerow(row)
else:
print("Skipping supplier not on framework: {}".format(supplier_id))
continue
fail_writer.writerow(row)
else:
print("Supplier did not apply: {}".format(supplier_id))
continue


def lot_counts_for_supplier_id(lot_counts, supplier_id):
Expand Down
2 changes: 1 addition & 1 deletion requirements.txt
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
git+https://github.com/madzak/[email protected]#egg=python-json-logger==v0.1.3
git+https://github.com/alphagov/digitalmarketplace-utils.git@11.1.2#egg=digitalmarketplace-utils==11.1.2
git+https://github.com/alphagov/digitalmarketplace-utils.git@13.0.0#egg=digitalmarketplace-utils==13.0.0

unicodecsv==0.14.1
12 changes: 6 additions & 6 deletions tests/test_generate_framework_agreement_data.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,11 +26,11 @@ def test_read_csv():


def test_make_filename_key():
assert make_filename_key('Kev\'s Butties', 1234) == '1234-Kevs_Butties'
assert make_filename_key(' Supplier A ', 1234) == '1234-Supplier_A'
assert make_filename_key('Kev & Sons. | Ltd', 1234) == '1234-Kev_and_Sons._Ltd'
assert make_filename_key('\ / : * ? \' " < > |', 1234) == '1234-_'
assert make_filename_key('kev@the*agency', 1234) == '1234-kevtheagency'
assert make_filename_key('Kev\'s Butties', 1234) == 'Kevs_Butties-1234'
assert make_filename_key(' Supplier A ', 1234) == 'Supplier_A-1234'
assert make_filename_key('Kev & Sons. | Ltd', 1234) == 'Kev_and_Sons_Ltd-1234'
assert make_filename_key('\ / : * ? \' " < > |', 1234) == '_-1234'
assert make_filename_key('kev@the*agency', 1234) == 'kevtheagency-1234'


def test_supplier_is_on_framework(mock_data_client):
Expand All @@ -41,7 +41,7 @@ def test_supplier_is_on_framework(mock_data_client):
assert supplier_is_on_framework(mock_data_client, 123) is False

mock_data_client.get_supplier_framework_info.return_value = {'frameworkInterest': {'onFramework': None}}
assert supplier_is_on_framework(mock_data_client, 123) is False
assert supplier_is_on_framework(mock_data_client, 123) is None

mock_data_client.get_supplier_framework_info.side_effect = HTTPError()
assert supplier_is_on_framework(mock_data_client, 123) is False

0 comments on commit dac6690

Please sign in to comment.