Skip to content

Commit

Permalink
Merge pull request #578 from ideal-postcodes/spd_fix
Browse files Browse the repository at this point in the history 
SPD fix
  • Loading branch information
@cblanc
cblanc authored Aug 23, 2020
2 parents 1eab750 + 954469c commit 86de005
Show file tree
Hide file tree
Showing 3 changed files with 52 additions and 24 deletions.
39 changes: 29 additions & 10 deletions app/models/scottish_postcode.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ const foreignColumns = [
const toJoinString = () => {
return relationships
.map(
r =>
(r) =>
`LEFT OUTER JOIN ${r.table} ON ${TABLE_NAME}.${r.key}=${r.table}.${r.foreignKey}`
)
.join(" ");
Expand All @@ -65,17 +65,35 @@ const findQuery = `
`;

const SPD_COL_MAPPINGS = Object.freeze([
{ column: "postcode", method: row => row.extract("Postcode") },
{ column: "postcode", method: (row) => row.extract("Postcode") },
{
column: "pc_compact",
method: row => row.extract("Postcode").replace(/\s/g, ""),
method: (row) => row.extract("Postcode").replace(/\s/g, ""),
},
{
column: "scottish_constituency_id",
method: row => row.extract("ScottishParliamentaryConstituency2014Code"),
method: (row) => row.extract("ScottishParliamentaryConstituency2014Code"),
},
]);

const EXCEPTION_REGEX = /A$/;

/**
* Validates and potentially mutates a CSV row for ingest
*
* Unfortunately SPD appends extra characters to some postcodes. This method returns null when these cases are met, unless the postcode ends in `A`.
*
* Postcodes suffixed with `A` are made valid and returned to the stream for ingest
*/
const clean = (row) => {
const postcode = row[0];
if (Postcode.isValid(postcode)) return row;
// Reject if invalid postcide has a non-A suffix
if (postcode.match(EXCEPTION_REGEX) === null) return null;
row[0] = postcode.replace(EXCEPTION_REGEX, "");
return row;
};

class ScottishPostcode extends Base {
constructor() {
super(TABLE_NAME, schema, indexes);
Expand Down Expand Up @@ -117,13 +135,14 @@ class ScottishPostcode extends Base {
this._csvSeed(
{
filepath,
transform: row => {
row.extract = code => extractor(row, code);
transform: (row) => {
clean(row);
row.extract = (code) => extractor(row, code);
if (row.extract("Postcode") === "Postcode") return null; // Skip if header
if (row.extract("DateOfDeletion").length !== 0) return null; // Skip row if terminated
return SPD_COL_MAPPINGS.map(elem => elem.method(row));
return SPD_COL_MAPPINGS.map((elem) => elem.method(row));
},
columns: SPD_COL_MAPPINGS.map(elem => elem.column).join(","),
columns: SPD_COL_MAPPINGS.map((elem) => elem.column).join(","),
},
callback
);
Expand All @@ -136,13 +155,13 @@ class ScottishPostcode extends Base {
[
this._createRelation.bind(this),
this.clear.bind(this),
cb =>
(cb) =>
this.seedPostcodes.call(
this,
{ filepath: largeUserFile, extractor: largeUserExtractor },
cb
),
cb =>
(cb) =>
this.seedPostcodes.call(
this,
{ filepath: smallUserFile, extractor: smallUserExtractor },
Expand Down
35 changes: 21 additions & 14 deletions test/scottish_postcode.unit.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,32 +19,32 @@ describe("Scottish Postcode Model", () => {
const testPostcodeLarge = "ML11 0GH";
const testPostcodeSmall = "G82 1JW";

before(function(done) {
before(function (done) {
this.timeout(0);
series([clearPostcodeDb, seedPostcodeDb], done);
});

after(clearPostcodeDb);

describe("#setupTable", () => {
before(function(done) {
before(function (done) {
this.timeout(0);
ScottishPostcode._destroyRelation(error => {
ScottishPostcode._destroyRelation((error) => {
if (error) return done(error);
ScottishPostcode._setupTable(seedFilePath, done);
});
});

after(function(done) {
after(function (done) {
this.timeout(0);
ScottishPostcode._destroyRelation(error => {
ScottishPostcode._destroyRelation((error) => {
if (error) return done(error);
ScottishPostcode._setupTable(seedFilePath, done);
});
});

describe("#_createRelation", () => {
it(`creates a relation that matches ${ScottishPostcode.relation} schema`, done => {
it(`creates a relation that matches ${ScottishPostcode.relation} schema`, (done) => {
const query = `
SELECT
column_name, data_type, character_maximum_length, collation_name
Expand All @@ -54,7 +54,7 @@ describe("Scottish Postcode Model", () => {
ScottishPostcode._query(query, (error, result) => {
if (error) return done(error);
const impliedSchema = {};
result.rows.forEach(columnInfo => {
result.rows.forEach((columnInfo) => {
let columnName, dataType;
[columnName, dataType] = inferSchemaData(columnInfo);
impliedSchema[columnName] = dataType;
Expand All @@ -66,14 +66,21 @@ describe("Scottish Postcode Model", () => {
});

describe("#seedData", () => {
it("loads correct data from data directory", done => {
it("loads correct data from data directory", (done) => {
const query = `SELECT count(*) FROM ${ScottishPostcode.relation}`;
ScottishPostcode._query(query, (error, result) => {
if (error) return done(error);
assert.isTrue(result.rows[0].count > 0);
done();
});
});
it("loads postcodes suffixed with additional character", (done) => {
ScottishPostcode.find("PA31 8UA", (error, result) => {
if (error) return done(error);
assert.isNotNull(result);
done();
});
});
});
});

Expand All @@ -100,7 +107,7 @@ describe("Scottish Postcode Model", () => {
});

describe("#find", () => {
it("should return postcode with the right attributes for large user", done => {
it("should return postcode with the right attributes for large user", (done) => {
ScottishPostcode.find(testPostcodeLarge, (error, result) => {
if (error) return done(error);
assert.deepEqual(result, {
Expand All @@ -114,7 +121,7 @@ describe("Scottish Postcode Model", () => {
});
});

it("should return postcode with the right attributes for small users", done => {
it("should return postcode with the right attributes for small users", (done) => {
ScottishPostcode.find(testPostcodeSmall, (error, result) => {
if (error) return done(error);
assert.deepEqual(result, {
Expand All @@ -128,23 +135,23 @@ describe("Scottish Postcode Model", () => {
});
});

it("should return null for null/undefined postcode search", done => {
it("should return null for null/undefined postcode search", (done) => {
ScottishPostcode.find(null, (error, result) => {
if (error) return done(error);
assert.isNull(result);
done();
});
});

it("returns null if invalid postcode", done => {
it("returns null if invalid postcode", (done) => {
ScottishPostcode.find("1", (error, result) => {
if (error) return done(error);
assert.isNull(result);
done();
});
});

it("should be insensitive to space", done => {
it("should be insensitive to space", (done) => {
ScottishPostcode.find(
testPostcodeLarge.replace(/\s/, ""),
(error, result) => {
Expand All @@ -155,7 +162,7 @@ describe("Scottish Postcode Model", () => {
);
});

it("should return null if postcode does not exist", done => {
it("should return null if postcode does not exist", (done) => {
ScottishPostcode.find("ID11QD", (error, result) => {
if (error) return done(error);
assert.isNull(result);
Expand Down
2 changes: 2 additions & 0 deletions test/seed/SmallUser.csv
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,5 @@
"PA20 9BA","PA20","PA20 9",1/8/1973 00:00:00,,"208889","664416",55.83501986,-5.05330374,"N","S12000035","S14000005","S17000011","S16000083","S13002523","S08000022","S08000008","08","S37000004","S00094213","S00004763","6332AC07C","S01007350","S01000733","S02001382","S02000133",40,49,45,63,45,56,"1458","S30000036","UKM6","UKM63","S19001816","164001","506","S20001449","164","S35000726","S09000003","019","32","32",,"S12000035",,,,"S22000058","4","5","Y",1,
"EH41 4JZ","EH41","EH41 4",11/7/2011 00:00:00,,"350023","664926",55.87489737,-2.80032623,"N","S12000010","S14000020","S17000015","S16000102","S13002912","S08000024","S08000010","05","S37000010","S00102365","S00012406","6228AT12B","S01008248","S01001551","S02001547","S02000288",,,,,,,"4461","S30000005","UKM7","UKM73",,,,,,"S35000870","S09000002","000","28","28",,"S12000010",,,"S11000003","S22000059","5","6","Y",1,
"KY13 8GG","KY13","KY13 8",30/6/2016 00:00:00,,"311666","702764",56.20924110,-3.42566487,"N","S12000048","S14000050","S17000013","S16000138","S13003132","S08000030","S08000013","03","S37000033","S00126336","S00034434","6453AK07A","S01011838","S01004979","S02002220","S02000950",,,,,,,"6721","S30000043","UKM7","UKM77","S19001635","308001","339","S20001308","308","S35000490","S09000005","000","53","53",,"S12000048",,,"S11000005","S22000074","3","3","Y",1,
"PA31 8UAB","PA31","PA31 8",7/7/2004 00:00:00,,"179709","708103",56.214350030,-5.554182280,"Y","S12000035","S14000005","S17000011","S16000083","S13002518","S08000022","S08000008","08","S37000004","S00094054","S00004613","6332AR16B","S01007313","S01000803","S02001375","S02000145",0,0,,,,,"3967","S30000036","UKM6","UKM63",,,,,,"S35000177","S09000003","157","32","32",,"S12000035",,,,"S22000086","6","8","N",,
"PA31 8UAA","PA31","PA31 8",7/7/2004 00:00:00,,"179562","707515",56.209009330,-5.556059030,"Y","S12000035","S14000005","S17000011","S16000083","S13002518","S08000022","S08000008","08","S37000004","S00094054","S00004613","6332AR16B","S01007313","S01000803","S02001375","S02000145",9,18,,,,,"3967","S30000036","UKM6","UKM63",,,,,,"S35000177","S09000003","000","32","32",,"S12000035",,,,"S22000086","6","8","N",,

0 comments on commit 86de005

Please sign in to comment.