diff --git a/scripts/jobs/housing/housing_gx_dq_metadata.py b/scripts/jobs/housing/housing_gx_dq_metadata.py index 029beb691..75a381deb 100644 --- a/scripts/jobs/housing/housing_gx_dq_metadata.py +++ b/scripts/jobs/housing/housing_gx_dq_metadata.py @@ -73,7 +73,7 @@ def main(): dataset=True, database=target_database, table=target_table, - mode="overwrite", + mode="overwrite_partitions", partition_cols=partition_keys, dtype=dtype_dict ) diff --git a/scripts/jobs/housing/housing_person_reshape_gx_suite.py b/scripts/jobs/housing/housing_person_reshape_gx_suite.py index 8ec82ffad..e4447a451 100644 --- a/scripts/jobs/housing/housing_person_reshape_gx_suite.py +++ b/scripts/jobs/housing/housing_person_reshape_gx_suite.py @@ -23,24 +23,6 @@ class ExpectSurnameColumnValueLength(gxe.ExpectColumnValueLengthsToBeBetween): description: str = "Expect surname to be at least 1 character length" -class ExpectUPRNColumnValueLengthsBetween(gxe.ExpectColumnValueLengthsToBeBetween): - column: str = "uprn" - min_value: int = 11 - max_value: int = 12 - description: str = "Expect UPRN to be between 11 and 12 characters length inclusive" - - -class ExpectUPRNColumnValuesToMatchRegex(gxe.ExpectColumnValuesToMatchRegex): - column: str = "uprn" - regex: str = r"^[1-9]\d{10,11}" - description: str = "Expect UPRN to match regex ^[1-9]\d{10,11} (starting with digit 1-9, followed by 10 or 11 digits" - - -class ExpectUPRNNotToBeNull(gxe.ExpectColumnValuesToNotBeNull): - column: str = "uprn" - description: str = "Expect UPRN column to be complete with no missing values" - - class ExpectPersonTypeValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet): column: str = 'person_type' value_set: list = ['Tenant', 'HouseholdMember', 'Leaseholder', 'Freeholder', 'Occupant', 'HousingOfficer', @@ -81,11 +63,6 @@ class ExpectPersonIDAndPaymentReferenceColumnValuesToBeUniqueWithinRecord( description: str = "Expect Person ID and Payment Reference to be unique within dataset" -class ExpectUPRNColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull): - column: str = 'uprn' - description: str = "Expect UPRN be complete with no missing values" - - class ExpectDateOfBirthColumnValuesToNotBeNull(gxe.ExpectColumnValuesToNotBeNull): column: str = 'dateofbirth_parsed' description: str = "Expect dateofbirth_parsed be complete with no missing values" @@ -104,9 +81,6 @@ class ExpectDateOfBirthToBeBetween(gxe.ExpectColumnValuesToBeBetween): suite = gx.ExpectationSuite(name='person_reshape_suite') suite.add_expectation(ExpectFirstNameColumnValueLength()) suite.add_expectation(ExpectSurnameColumnValueLength()) -suite.add_expectation(ExpectUPRNColumnValueLengthsBetween()) -suite.add_expectation(ExpectUPRNColumnValuesToMatchRegex()) -suite.add_expectation(ExpectUPRNNotToBeNull()) suite.add_expectation(ExpectPersonTypeValuesToBeInSet()) suite.add_expectation(ExpectPreferredTitleValuesToBeInSet()) suite.add_expectation(ExpectPersonIDColumnValuesToBeUnique()) @@ -114,7 +88,6 @@ class ExpectDateOfBirthToBeBetween(gxe.ExpectColumnValuesToBeBetween): suite.add_expectation(ExpectPersonIDAndPropertyReferenceColumnValuesToBeUniqueWithinRecord()) suite.add_expectation(ExpectPropertyRefColumnValuesToNotBeNull()) suite.add_expectation(ExpectPersonIDAndPaymentReferenceColumnValuesToBeUniqueWithinRecord()) -suite.add_expectation(ExpectUPRNColumnValuesToNotBeNull()) suite.add_expectation(ExpectDateOfBirthColumnValuesToNotBeNull()) suite.add_expectation(ExpectDateOfBirthToBeBetween()) diff --git a/scripts/jobs/housing/housing_tenure_reshape_gx_suite.py b/scripts/jobs/housing/housing_tenure_reshape_gx_suite.py index 8d4fe7368..355609318 100644 --- a/scripts/jobs/housing/housing_tenure_reshape_gx_suite.py +++ b/scripts/jobs/housing/housing_tenure_reshape_gx_suite.py @@ -17,19 +17,6 @@ class ExpectMemberFullNameColumnValueLengthsBetween(gxe.ExpectColumnValueLengths description: str = "Expect Member Fullname to be at least 1 character length" -class ExpectUPRNColumnValueLengthsBetween(gxe.ExpectColumnValueLengthsToBeBetween): - column: str = "uprn" - min_value: int = 11 - max_value: int = 12 - description: str = "Expect UPRN to be between 11 and 12 characters length inclusive" - - -class ExpectUPRNColumnValuesToMatchRegex(gxe.ExpectColumnValuesToMatchRegex): - column: str = "uprn" - regex: str = r"^[1-9]\d{10,11}" - description: str = "Expect UPRN to match regex ^[1-9]\d{10,11} (starting with digit 1-9, followed by 10 or 11 digits" - - class ExpectDescriptionValuesToBeInSet(gxe.ExpectColumnValuesToBeInSet): column: str = 'description' value_set: list = ['Asylum Seeker', 'Commercial Let', 'Temp Decant', 'Freehold', 'Freehold (Serv)', 'Introductory', @@ -76,11 +63,6 @@ class ExpectTenancyIDColumnNotToBeNull(gxe.ExpectColumnValuesToNotBeNull): description: str = "Expect Tenancy ID column to be complete with no missing values" -class ExpectUPRNNotToBeNull(gxe.ExpectColumnValuesToNotBeNull): - column: str = "uprn" - description: str = "Expect UPRN column to be complete with no missing values" - - class ExpectStartOfTenureDateColumnNotToBeNull(gxe.ExpectColumnValuesToNotBeNull): column: str = "startoftenuredate" description: str = "Expect Start of Tenure Date column to be complete with no missing values" @@ -97,14 +79,11 @@ class ExpectTenureCodeColumnNotToBeNull(gxe.ExpectColumnValuesToNotBeNull): suite = gx.ExpectationSuite(name='tenure_reshape_suite') suite.add_expectation(ExpectTenancyIDAndPaymentReferenceColumnValuesToBeUniqueWithinRecord()) suite.add_expectation(ExpectMemberFullNameColumnValueLengthsBetween()) -suite.add_expectation(ExpectUPRNColumnValueLengthsBetween()) -suite.add_expectation(ExpectUPRNColumnValuesToMatchRegex()) suite.add_expectation(ExpectDescriptionValuesToBeInSet()) suite.add_expectation(ExpectAssetTypeValuesToBeInSet()) suite.add_expectation(ExpectTenancyIDAndPropertyReferenceColumnValuesToBeUniqueWithinRecord()) suite.add_expectation(ExpectPaymentReferenceColumnNotToBeNull()) suite.add_expectation(ExpectTenancyIDColumnNotToBeNull()) -suite.add_expectation(ExpectUPRNNotToBeNull()) suite.add_expectation(ExpectStartOfTenureDateColumnNotToBeNull()) suite.add_expectation(ExpectTenureCodeColumnNotToBeNull())