Fix lots in export

dmscripts/generate_framework_agreement_data.py

@@ -1,17 +1,14 @@
 # -*- coding: utf-8 -*-
 import os
 from dmutils.apiclient.errors import HTTPError
+from dmutils.documents import sanitise_supplier_name
 
 import sys
 if sys.version_info > (3, 0):
     import csv
 else:
     import unicodecsv as csv
 
-#  List of bad characters taken from: http://www.mtu.edu/umc/services/web/cms/characters-avoid/
-BAD_FILENAME_CHARACTERS = ['#', '%', '&', '{', '}', '\\', '<', '>', '*', '?', '/',
-                           '$', '!', "'", '"', ':', '@', '+', '`', '|', '=']
-
 
 class Supplier:
 
@@ -29,8 +26,8 @@ def __init__(self, declaration, lots):
         self.contact_email = declaration[19]
 
         self.lot1 = "Lot 1: Infrastructure as a Service (IaaS)" if int(lots[3]) > 0 else ""
-        self.lot2 = "Lot 2: Software as a Service (SaaS)" if int(lots[7]) > 0 else ""
-        self.lot3 = "Lot 3: Platform as a Service (PaaS)" if int(lots[5]) > 0 else ""
+        self.lot2 = "Lot 2: Platform as a Service (PaaS)" if int(lots[5]) > 0 else ""
+        self.lot3 = "Lot 3: Software as a Service (SaaS)" if int(lots[7]) > 0 else ""
         self.lot4 = "Lot 4: Specialist Cloud Services (SCS)" if int(lots[9]) > 0 else ""
 
     def __str__(self):
@@ -39,6 +36,13 @@ def __str__(self):
             self.company_number, self.registered_office_address, self.contact_name, self.contact_email)
 
 
+class FailedSupplier:
+
+    def __init__(self, declaration):
+        self.supplier_id = declaration[0]
+        self.registered_company_name = declaration[20]
+
+
 def read_csv(filepath):
     all_rows = []
     with open(filepath, 'r') as csvfile:
@@ -49,21 +53,13 @@ def read_csv(filepath):
 
 
 def make_filename_key(supplier_name, supplier_id):
-    sanitised_supplier_name = supplier_name.strip().replace(' ', '_').replace('&', 'and')
-    for bad_char in BAD_FILENAME_CHARACTERS:
-        sanitised_supplier_name = sanitised_supplier_name.replace(bad_char, '')
-    while '__' in sanitised_supplier_name:
-        sanitised_supplier_name = sanitised_supplier_name.replace('__', '_')
-    return "{}-{}".format(supplier_id, sanitised_supplier_name)
+    return "{}-{}".format(sanitise_supplier_name(supplier_name), supplier_id)
 
 
 def supplier_is_on_framework(client, supplier_id):
     try:
         framework_interest = client.get_supplier_framework_info(supplier_id, 'g-cloud-7')
-        if framework_interest['frameworkInterest']['onFramework']:
-            return True
-        else:
-            return False
+        return framework_interest['frameworkInterest']['onFramework']
     except HTTPError as e:
         print("ERROR checking if supplier {} is on framework: {}".format(supplier_id, str(e)))
         return False
@@ -73,57 +69,76 @@ def build_framework_agreements(client, declarations, lots, output_dir):
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
     with open('{}/g7-framework-data.tsv'.format(output_dir), 'w') as csvfile:
-        # This defines the order of the fields - fields can be in any order in
-        # the dictionary for each row and will be mapped to the order defined here.
-        fieldnames = [
-            'Key',
-            'Supplier ID',
-            'Registered Company Name',
-            'Country of Registration',
-            'Registered Company Number',
-            'Registered Address',
-            'Framework Contact Name',
-            'Framework Contact Email address',
-            'Lot1',
-            'Lot2',
-            'Lot3',
-            'Lot4',
-            'Lot1Letter',
-            'Lot2Letter',
-            'Lot3Letter',
-            'Lot4Letter',
-        ]
-        writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab')
-        writer.writeheader()
-
-        for declaration in declarations:
-            supplier_id = declaration[0]
-            if supplier_is_on_framework(client, supplier_id):
-                lot_count = lot_counts_for_supplier_id(lots, supplier_id)
-                if lot_count:
-                    supplier = Supplier(declaration, lot_count)
+        with open('{}/g7-fail-data.tsv'.format(output_dir), 'w') as failfile:
+            # This defines the order of the fields - fields can be in any order in
+            # the dictionary for each row and will be mapped to the order defined here.
+            fieldnames = [
+                'Key',
+                'Supplier ID',
+                'Registered Company Name',
+                'Country of Registration',
+                'Registered Company Number',
+                'Registered Address',
+                'Framework Contact Name',
+                'Framework Contact Email address',
+                'Lot1',
+                'Lot2',
+                'Lot3',
+                'Lot4',
+                'Lot1Letter',
+                'Lot2Letter',
+                'Lot3Letter',
+                'Lot4Letter',
+            ]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab')
+            writer.writeheader()
+
+            fail_fieldnames = [
+                'Key',
+                'Supplier ID',
+                'Registered Company Name'
+            ]
+            fail_writer = csv.DictWriter(failfile, fieldnames=fail_fieldnames, dialect='excel-tab')
+            fail_writer.writeheader()
+
+            for declaration in declarations:
+                supplier_id = declaration[0]
+                on_framework = supplier_is_on_framework(client, supplier_id)
+                if on_framework is True:
+                    lot_count = lot_counts_for_supplier_id(lots, supplier_id)
+                    if lot_count:
+                        supplier = Supplier(declaration, lot_count)
+                        row = {
+                            'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id),
+                            'Supplier ID': supplier.supplier_id,
+                            'Registered Company Name': supplier.registered_company_name,
+                            'Country of Registration': supplier.country_of_registration,
+                            'Registered Company Number': supplier.company_number,
+                            'Registered Address': supplier.registered_office_address,
+                            'Framework Contact Name': supplier.contact_name,
+                            'Framework Contact Email address': supplier.contact_email,
+                            'Lot1': supplier.lot1,
+                            'Lot2': supplier.lot2,
+                            'Lot3': supplier.lot3,
+                            'Lot4': supplier.lot4,
+                            'Lot1Letter': "Pass" if supplier.lot1 else "No bid",
+                            'Lot2Letter': "Pass" if supplier.lot2 else "No bid",
+                            'Lot3Letter': "Pass" if supplier.lot3 else "No bid",
+                            'Lot4Letter': "Pass" if supplier.lot4 else "No bid",
+                        }
+                        writer.writerow(row)
+                elif on_framework is False:
+                    print("Failed supplier: {}".format(supplier_id))
+                    supplier = FailedSupplier(declaration)
                     row = {
                         'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id),
                         'Supplier ID': supplier.supplier_id,
                         'Registered Company Name': supplier.registered_company_name,
-                        'Country of Registration': supplier.country_of_registration,
-                        'Registered Company Number': supplier.company_number,
-                        'Registered Address': supplier.registered_office_address,
-                        'Framework Contact Name': supplier.contact_name,
-                        'Framework Contact Email address': supplier.contact_email,
-                        'Lot1': supplier.lot1,
-                        'Lot2': supplier.lot2,
-                        'Lot3': supplier.lot3,
-                        'Lot4': supplier.lot4,
-                        'Lot1Letter': "Pass" if supplier.lot1 else "No bid",
-                        'Lot2Letter': "Pass" if supplier.lot2 else "No bid",
-                        'Lot3Letter': "Pass" if supplier.lot3 else "No bid",
-                        'Lot4Letter': "Pass" if supplier.lot4 else "No bid",
                     }
-                    writer.writerow(row)
-            else:
-                print("Skipping supplier not on framework: {}".format(supplier_id))
-                continue
+                    fail_writer.writerow(row)
+                else:
+                    print("Supplier did not apply: {}".format(supplier_id))
+                    continue
 
 
 def lot_counts_for_supplier_id(lot_counts, supplier_id):

requirements.txt

@@ -1,4 +1,4 @@
 git+https://github.com/madzak/[email protected]#egg=python-json-logger==v0.1.3
-git+https://github.com/alphagov/digitalmarketplace-utils.git@11.1.2#egg=digitalmarketplace-utils==11.1.2
+git+https://github.com/alphagov/digitalmarketplace-utils.git@13.0.0#egg=digitalmarketplace-utils==13.0.0
 
 unicodecsv==0.14.1

tests/test_generate_framework_agreement_data.py

@@ -26,11 +26,11 @@ def test_read_csv():
 
 
 def test_make_filename_key():
-    assert make_filename_key('Kev\'s Butties', 1234) == '1234-Kevs_Butties'
-    assert make_filename_key('   Supplier A   ', 1234) == '1234-Supplier_A'
-    assert make_filename_key('Kev & Sons. | Ltd', 1234) == '1234-Kev_and_Sons._Ltd'
-    assert make_filename_key('\ / : * ? \' " < > |', 1234) == '1234-_'
-    assert make_filename_key('kev@the*agency', 1234) == '1234-kevtheagency'
+    assert make_filename_key('Kev\'s Butties', 1234) == 'Kevs_Butties-1234'
+    assert make_filename_key('   Supplier A   ', 1234) == 'Supplier_A-1234'
+    assert make_filename_key('Kev & Sons. | Ltd', 1234) == 'Kev_and_Sons_Ltd-1234'
+    assert make_filename_key('\ / : * ? \' " < > |', 1234) == '_-1234'
+    assert make_filename_key('kev@the*agency', 1234) == 'kevtheagency-1234'
 
 
 def test_supplier_is_on_framework(mock_data_client):
@@ -41,7 +41,7 @@ def test_supplier_is_on_framework(mock_data_client):
     assert supplier_is_on_framework(mock_data_client, 123) is False
 
     mock_data_client.get_supplier_framework_info.return_value = {'frameworkInterest': {'onFramework': None}}
-    assert supplier_is_on_framework(mock_data_client, 123) is False
+    assert supplier_is_on_framework(mock_data_client, 123) is None
 
     mock_data_client.get_supplier_framework_info.side_effect = HTTPError()
     assert supplier_is_on_framework(mock_data_client, 123) is False