From d0cd0e692475bae48d2197e1e44dd8227fa082d9 Mon Sep 17 00:00:00 2001
From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk>
Date: Mon, 2 Nov 2015 09:17:03 +0000
Subject: [PATCH 1/4] Fix order of lots

SaaS and PaaS were the wrong way round
---
 dmscripts/generate_framework_agreement_data.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py
index 07dc87664df..51dcde6c919 100644
--- a/dmscripts/generate_framework_agreement_data.py
+++ b/dmscripts/generate_framework_agreement_data.py
@@ -29,8 +29,8 @@ def __init__(self, declaration, lots):
         self.contact_email = declaration[19]
 
         self.lot1 = "Lot 1: Infrastructure as a Service (IaaS)" if int(lots[3]) > 0 else ""
-        self.lot2 = "Lot 2: Software as a Service (SaaS)" if int(lots[7]) > 0 else ""
-        self.lot3 = "Lot 3: Platform as a Service (PaaS)" if int(lots[5]) > 0 else ""
+        self.lot2 = "Lot 2: Platform as a Service (PaaS)" if int(lots[5]) > 0 else ""
+        self.lot3 = "Lot 3: Software as a Service (SaaS)" if int(lots[7]) > 0 else ""
         self.lot4 = "Lot 4: Specialist Cloud Services (SCS)" if int(lots[9]) > 0 else ""
 
     def __str__(self):

From 083275b062381fa101100de4871b9e478ecddae1 Mon Sep 17 00:00:00 2001
From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk>
Date: Mon, 2 Nov 2015 13:41:43 +0000
Subject: [PATCH 2/4] Also create output for failed suppliers

Late-breaking requirement from CCS is to also have a file for failed suppliers for generating letters.
---
 .../generate_framework_agreement_data.py      | 123 +++++++++++-------
 1 file changed, 73 insertions(+), 50 deletions(-)

diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py
index 51dcde6c919..2ad84972526 100644
--- a/dmscripts/generate_framework_agreement_data.py
+++ b/dmscripts/generate_framework_agreement_data.py
@@ -39,6 +39,13 @@ def __str__(self):
             self.company_number, self.registered_office_address, self.contact_name, self.contact_email)
 
 
+class FailedSupplier:
+
+    def __init__(self, declaration):
+        self.supplier_id = declaration[0]
+        self.registered_company_name = declaration[20]
+
+
 def read_csv(filepath):
     all_rows = []
     with open(filepath, 'r') as csvfile:
@@ -60,10 +67,7 @@ def make_filename_key(supplier_name, supplier_id):
 def supplier_is_on_framework(client, supplier_id):
     try:
         framework_interest = client.get_supplier_framework_info(supplier_id, 'g-cloud-7')
-        if framework_interest['frameworkInterest']['onFramework']:
-            return True
-        else:
-            return False
+        return framework_interest['frameworkInterest']['onFramework']
     except HTTPError as e:
         print("ERROR checking if supplier {} is on framework: {}".format(supplier_id, str(e)))
         return False
@@ -73,57 +77,76 @@ def build_framework_agreements(client, declarations, lots, output_dir):
     if not os.path.exists(output_dir):
         os.makedirs(output_dir)
     with open('{}/g7-framework-data.tsv'.format(output_dir), 'w') as csvfile:
-        # This defines the order of the fields - fields can be in any order in
-        # the dictionary for each row and will be mapped to the order defined here.
-        fieldnames = [
-            'Key',
-            'Supplier ID',
-            'Registered Company Name',
-            'Country of Registration',
-            'Registered Company Number',
-            'Registered Address',
-            'Framework Contact Name',
-            'Framework Contact Email address',
-            'Lot1',
-            'Lot2',
-            'Lot3',
-            'Lot4',
-            'Lot1Letter',
-            'Lot2Letter',
-            'Lot3Letter',
-            'Lot4Letter',
-        ]
-        writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab')
-        writer.writeheader()
-
-        for declaration in declarations:
-            supplier_id = declaration[0]
-            if supplier_is_on_framework(client, supplier_id):
-                lot_count = lot_counts_for_supplier_id(lots, supplier_id)
-                if lot_count:
-                    supplier = Supplier(declaration, lot_count)
+        with open('{}/g7-fail-data.tsv'.format(output_dir), 'w') as failfile:
+            # This defines the order of the fields - fields can be in any order in
+            # the dictionary for each row and will be mapped to the order defined here.
+            fieldnames = [
+                'Key',
+                'Supplier ID',
+                'Registered Company Name',
+                'Country of Registration',
+                'Registered Company Number',
+                'Registered Address',
+                'Framework Contact Name',
+                'Framework Contact Email address',
+                'Lot1',
+                'Lot2',
+                'Lot3',
+                'Lot4',
+                'Lot1Letter',
+                'Lot2Letter',
+                'Lot3Letter',
+                'Lot4Letter',
+            ]
+            writer = csv.DictWriter(csvfile, fieldnames=fieldnames, dialect='excel-tab')
+            writer.writeheader()
+
+            fail_fieldnames = [
+                'Key',
+                'Supplier ID',
+                'Registered Company Name'
+            ]
+            fail_writer = csv.DictWriter(failfile, fieldnames=fail_fieldnames, dialect='excel-tab')
+            fail_writer.writeheader()
+
+            for declaration in declarations:
+                supplier_id = declaration[0]
+                on_framework = supplier_is_on_framework(client, supplier_id)
+                if on_framework is True:
+                    lot_count = lot_counts_for_supplier_id(lots, supplier_id)
+                    if lot_count:
+                        supplier = Supplier(declaration, lot_count)
+                        row = {
+                            'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id),
+                            'Supplier ID': supplier.supplier_id,
+                            'Registered Company Name': supplier.registered_company_name,
+                            'Country of Registration': supplier.country_of_registration,
+                            'Registered Company Number': supplier.company_number,
+                            'Registered Address': supplier.registered_office_address,
+                            'Framework Contact Name': supplier.contact_name,
+                            'Framework Contact Email address': supplier.contact_email,
+                            'Lot1': supplier.lot1,
+                            'Lot2': supplier.lot2,
+                            'Lot3': supplier.lot3,
+                            'Lot4': supplier.lot4,
+                            'Lot1Letter': "Pass" if supplier.lot1 else "No bid",
+                            'Lot2Letter': "Pass" if supplier.lot2 else "No bid",
+                            'Lot3Letter': "Pass" if supplier.lot3 else "No bid",
+                            'Lot4Letter': "Pass" if supplier.lot4 else "No bid",
+                        }
+                        writer.writerow(row)
+                elif on_framework is False:
+                    print("Failed supplier: {}".format(supplier_id))
+                    supplier = FailedSupplier(declaration)
                     row = {
                         'Key': make_filename_key(supplier.registered_company_name, supplier.supplier_id),
                         'Supplier ID': supplier.supplier_id,
                         'Registered Company Name': supplier.registered_company_name,
-                        'Country of Registration': supplier.country_of_registration,
-                        'Registered Company Number': supplier.company_number,
-                        'Registered Address': supplier.registered_office_address,
-                        'Framework Contact Name': supplier.contact_name,
-                        'Framework Contact Email address': supplier.contact_email,
-                        'Lot1': supplier.lot1,
-                        'Lot2': supplier.lot2,
-                        'Lot3': supplier.lot3,
-                        'Lot4': supplier.lot4,
-                        'Lot1Letter': "Pass" if supplier.lot1 else "No bid",
-                        'Lot2Letter': "Pass" if supplier.lot2 else "No bid",
-                        'Lot3Letter': "Pass" if supplier.lot3 else "No bid",
-                        'Lot4Letter': "Pass" if supplier.lot4 else "No bid",
                     }
-                    writer.writerow(row)
-            else:
-                print("Skipping supplier not on framework: {}".format(supplier_id))
-                continue
+                    fail_writer.writerow(row)
+                else:
+                    print("Supplier did not apply: {}".format(supplier_id))
+                    continue
 
 
 def lot_counts_for_supplier_id(lot_counts, supplier_id):

From 434a59e22be0a52c660776546cd420eb630aa224 Mon Sep 17 00:00:00 2001
From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk>
Date: Mon, 2 Nov 2015 13:42:55 +0000
Subject: [PATCH 3/4] Put supplier name first in file key

---
 dmscripts/generate_framework_agreement_data.py  |  4 ++--
 tests/test_generate_framework_agreement_data.py | 12 ++++++------
 2 files changed, 8 insertions(+), 8 deletions(-)

diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py
index 2ad84972526..f1a69b1317b 100644
--- a/dmscripts/generate_framework_agreement_data.py
+++ b/dmscripts/generate_framework_agreement_data.py
@@ -10,7 +10,7 @@
 
 #  List of bad characters taken from: http://www.mtu.edu/umc/services/web/cms/characters-avoid/
 BAD_FILENAME_CHARACTERS = ['#', '%', '&', '{', '}', '\\', '<', '>', '*', '?', '/',
-                           '$', '!', "'", '"', ':', '@', '+', '`', '|', '=']
+                           '$', '!', "'", '"', ':', '@', '+', '`', '|', '=', ',', '.']
 
 
 class Supplier:
@@ -61,7 +61,7 @@ def make_filename_key(supplier_name, supplier_id):
         sanitised_supplier_name = sanitised_supplier_name.replace(bad_char, '')
     while '__' in sanitised_supplier_name:
         sanitised_supplier_name = sanitised_supplier_name.replace('__', '_')
-    return "{}-{}".format(supplier_id, sanitised_supplier_name)
+    return "{}-{}".format(sanitised_supplier_name, supplier_id)
 
 
 def supplier_is_on_framework(client, supplier_id):
diff --git a/tests/test_generate_framework_agreement_data.py b/tests/test_generate_framework_agreement_data.py
index 69f52671575..f3a1e8b2244 100644
--- a/tests/test_generate_framework_agreement_data.py
+++ b/tests/test_generate_framework_agreement_data.py
@@ -26,11 +26,11 @@ def test_read_csv():
 
 
 def test_make_filename_key():
-    assert make_filename_key('Kev\'s Butties', 1234) == '1234-Kevs_Butties'
-    assert make_filename_key('   Supplier A   ', 1234) == '1234-Supplier_A'
-    assert make_filename_key('Kev & Sons. | Ltd', 1234) == '1234-Kev_and_Sons._Ltd'
-    assert make_filename_key('\ / : * ? \' " < > |', 1234) == '1234-_'
-    assert make_filename_key('kev@the*agency', 1234) == '1234-kevtheagency'
+    assert make_filename_key('Kev\'s Butties', 1234) == 'Kevs_Butties-1234'
+    assert make_filename_key('   Supplier A   ', 1234) == 'Supplier_A-1234'
+    assert make_filename_key('Kev & Sons. | Ltd', 1234) == 'Kev_and_Sons_Ltd-1234'
+    assert make_filename_key('\ / : * ? \' " < > |', 1234) == '_-1234'
+    assert make_filename_key('kev@the*agency', 1234) == 'kevtheagency-1234'
 
 
 def test_supplier_is_on_framework(mock_data_client):
@@ -41,7 +41,7 @@ def test_supplier_is_on_framework(mock_data_client):
     assert supplier_is_on_framework(mock_data_client, 123) is False
 
     mock_data_client.get_supplier_framework_info.return_value = {'frameworkInterest': {'onFramework': None}}
-    assert supplier_is_on_framework(mock_data_client, 123) is False
+    assert supplier_is_on_framework(mock_data_client, 123) is None
 
     mock_data_client.get_supplier_framework_info.side_effect = HTTPError()
     assert supplier_is_on_framework(mock_data_client, 123) is False

From 320db1c2062b71d661308e0f365c93b88a34324c Mon Sep 17 00:00:00 2001
From: Kev Keenoy <kevin.keenoy@digital.cabinet-office.gov.uk>
Date: Tue, 3 Nov 2015 10:03:33 +0000
Subject: [PATCH 4/4] Use utils method to sanitise supplier name

---
 dmscripts/generate_framework_agreement_data.py | 12 ++----------
 requirements.txt                               |  2 +-
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/dmscripts/generate_framework_agreement_data.py b/dmscripts/generate_framework_agreement_data.py
index f1a69b1317b..f16b42376ea 100644
--- a/dmscripts/generate_framework_agreement_data.py
+++ b/dmscripts/generate_framework_agreement_data.py
@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import os
 from dmutils.apiclient.errors import HTTPError
+from dmutils.documents import sanitise_supplier_name
 
 import sys
 if sys.version_info > (3, 0):
@@ -8,10 +9,6 @@
 else:
     import unicodecsv as csv
 
-#  List of bad characters taken from: http://www.mtu.edu/umc/services/web/cms/characters-avoid/
-BAD_FILENAME_CHARACTERS = ['#', '%', '&', '{', '}', '\\', '<', '>', '*', '?', '/',
-                           '$', '!', "'", '"', ':', '@', '+', '`', '|', '=', ',', '.']
-
 
 class Supplier:
 
@@ -56,12 +53,7 @@ def read_csv(filepath):
 
 
 def make_filename_key(supplier_name, supplier_id):
-    sanitised_supplier_name = supplier_name.strip().replace(' ', '_').replace('&', 'and')
-    for bad_char in BAD_FILENAME_CHARACTERS:
-        sanitised_supplier_name = sanitised_supplier_name.replace(bad_char, '')
-    while '__' in sanitised_supplier_name:
-        sanitised_supplier_name = sanitised_supplier_name.replace('__', '_')
-    return "{}-{}".format(sanitised_supplier_name, supplier_id)
+    return "{}-{}".format(sanitise_supplier_name(supplier_name), supplier_id)
 
 
 def supplier_is_on_framework(client, supplier_id):
diff --git a/requirements.txt b/requirements.txt
index 5e061da83f6..3b6aea67a2a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,4 +1,4 @@
 git+https://github.com/madzak/python-json-logger.git@v0.1.3#egg=python-json-logger==v0.1.3
-git+https://github.com/alphagov/digitalmarketplace-utils.git@11.1.2#egg=digitalmarketplace-utils==11.1.2
+git+https://github.com/alphagov/digitalmarketplace-utils.git@13.0.0#egg=digitalmarketplace-utils==13.0.0
 
 unicodecsv==0.14.1