diff --git a/abacus_factory.yml b/abacus_factory.yml deleted file mode 100644 index 13198227..00000000 --- a/abacus_factory.yml +++ /dev/null @@ -1,398 +0,0 @@ -- macro: OppDefaults - fields: - amount: 4 - StageName: Prospecting - CloseDate: 2019-11-26 - npe01__Do_Not_Automatically_Create_Payment__c: True - -- macro: npePaymentDefaults - fields: - npe01__Payment_Amount__c: 3 - npe01__Paid__c: True - npe01__Payment_Method__c: Cheque - npe01__Check_Reference_Number__c: 12345 - npe01__Payment_Date__c: 2019-11-26 -- macro: npeScheduledPaymentDefaults - fields: - npe01__Payment_Amount__c: 3 - npe01__Payment_Method__c: Cheque - npe01__Check_Reference_Number__c: 12345 - npe01__Scheduled_Date__c: 2019-12-26 - -- object: npsp__General_Accounting_Unit__c - nickname: GAU1 - just_once: True - fields: - name: GAU 1 -- object: npsp__General_Accounting_Unit__c - nickname: GAU2 - just_once: True - fields: - name: GAU 2 -# - object: General_Accounting_Unit__c -# nickname: GAU3 -# fields: -# name: GAU 3 -- object: npsp__General_Accounting_Unit__c - nickname: Default GAU - just_once: True - fields: - name: Default GAU - -- object: Account - nickname: Account1 - fields: - name: Account 1 -- object: Account - nickname: Account2 - fields: - name: Account 2 -- object: Account - nickname: Account3 - fields: - name: Account 3 -- object: Account - nickname: Account4 - fields: - name: Account 4 -- object: Account - nickname: Account5 - fields: - name: Account 5 -- object: Account - nickname: Account6 - fields: - name: Account 6 - -- object: __Repeatable Grouper - count: 3 - friends: - - object: Opportunity - include: OppDefaults - nickname: Opportunity_1 - fields: - name: ABA-1.1a Soft Act. Open Oppty. Insert Partially Paid Pmt. Default GAU. GroupByAlloc - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account1 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_2 - fields: - name: ABA-1.1b Soft Act. Open Oppty. Insert Partially Paid Pmt. Default GAU. GroupByDate - Revenue_Grouping_Scheme__c: 2 - AccountId: - reference: Account2 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_3 - fields: - name: ABA-1.2a Soft Act. Open Oppty. Insert Partially Paid Pmt. 50% GAU1. GroupByAlloc - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account3 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_4 - fields: - name: ABA-1.2b Soft Act. Open Oppty. Insert Partially Paid Pmt. 50% GAU1. GroupByDate - Revenue_Grouping_Scheme__c: 2 - AccountId: - reference: Account4 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_5 - fields: - name: ABA-1.3a Soft Act. Open Oppty. 1 Paid Pmt 1 Sched Pmt. $50 GAU1. GroupByAlloc - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account5 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_6 - fields: - name: ABA-1.3b Soft Act. Open Oppty. 1 Paid Pmt 1 Sched Pmt. $50 GAU1. GroupByDate - Revenue_Grouping_Scheme__c: 2 - AccountId: - reference: Account6 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_7 - fields: - name: ABA-1.4a Soft Act. Open Oppty. 1 Written-Off Pmt. Default GAU. GroupByAlloc - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account1 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_8 - fields: - name: ABA-1.5a Soft Act. Open Oppty. 1 Paid Pmt 1 Refund. Default GAU. GroupByAlloc - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account2 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_9 - fields: - name: ABA-2.1a Hard Act. Pledged Oppty. Insert Sched Pmt. Default GAU. GroupByAlloc - StageName: Pledged - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account3 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_10 - fields: - name: ABA-2.1b Hard Act. Pledged Oppty. Insert Sched Pmt. Default GAU. GroupByDate - StageName: Pledged - Revenue_Grouping_Scheme__c: 2 - AccountId: - reference: Account4 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_11 - fields: - name: ABA-2.2a Hard Act. Closed Won Oppty. Insert Sched Pmt. Default GAU. GroupByAlloc - StageName: Closed Won - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account5 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_12 - fields: - name: ABA-2.3a Hard Act. Pledged Oppty. 2 Sched Pmts. 50% GAU1 40% GAU2. GroupByAlloc - StageName: Pledged - Revenue_Grouping_Scheme__c: 1 - AccountId: - reference: Account6 - - object: Opportunity - include: OppDefaults - nickname: Opportunity_13 - fields: - name: ABA-2.3b Hard Act. Pledged Oppty. 2 Sched Pmts. 50% GAU1 40% GAU2. GroupByDate - StageName: Pledged - Revenue_Grouping_Scheme__c: 2 - AccountId: - reference: Account1 - - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_1 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_2 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_3 - npsp__General_Accounting_Unit__c: - reference: GAU1 - npsp__Percent__c: 50 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_4 - npsp__General_Accounting_Unit__c: - reference: GAU1 - npsp__Percent__c: 50 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_5 - npsp__General_Accounting_Unit__c: - reference: GAU1 - npsp__Amount__c: 50 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_6 - npsp__General_Accounting_Unit__c: - reference: GAU1 - npsp__Amount__c: 50 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_7 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_8 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_9 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_10 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_11 - npsp__General_Accounting_Unit__c: - reference: Default GAU - npsp__Amount__c: 100 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_12 - npsp__General_Accounting_Unit__c: - reference: GAU1 - npsp__Percent__c: 50 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_12 - npsp__General_Accounting_Unit__c: - reference: GAU2 - npsp__Percent__c: 40 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_13 - npsp__General_Accounting_Unit__c: - reference: GAU1 - npsp__Percent__c: 50 - - object: npsp__Allocation__c - fields: - npsp__Opportunity__c: - reference: Opportunity_13 - npsp__General_Accounting_Unit__c: - reference: GAU2 - npsp__Percent__c: 40 - - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_1 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_2 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_3 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_4 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_5 - npe01__Payment_Amount__c: 50 - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_5 - npe01__Payment_Amount__c: 25 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_6 - npe01__Payment_Amount__c: 50 - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_6 - npe01__Payment_Amount__c: 25 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_7 - npe01__Payment_Amount__c: 50 - npe01__Paid__c: False - npe01__Written_Off__c: True - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_8 - npe01__Payment_Amount__c: 50 - - object: npe01__OppPayment__c - include: npePaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_8 - npe01__Payment_Amount__c: -25 - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_9 - npe01__Payment_Amount__c: 100 - npe01__Payment_Method__c: Cash - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_10 - npe01__Payment_Amount__c: 100 - npe01__Payment_Method__c: Cash - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_11 - npe01__Payment_Amount__c: 100 - npe01__Payment_Method__c: Cash - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_12 - npe01__Payment_Amount__c: 25 - npe01__Payment_Method__c: Cash - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_12 - npe01__Payment_Amount__c: 75 - npe01__Payment_Method__c: Cash - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_13 - npe01__Payment_Amount__c: 25 - npe01__Payment_Method__c: Cash - - object: npe01__OppPayment__c - include: npeScheduledPaymentDefaults - fields: - npe01__Opportunity__c: - reference: Opportunity_13 - npe01__Payment_Amount__c: 75 - npe01__Payment_Method__c: Cash diff --git a/snowfakery/data_generator_runtime_dom.py b/snowfakery/data_generator_runtime_dom.py index 10198488..0db3aae4 100644 --- a/snowfakery/data_generator_runtime_dom.py +++ b/snowfakery/data_generator_runtime_dom.py @@ -18,7 +18,7 @@ # objects that represent the hierarchy of a data generator. # roughly similar to the YAML structure but with domain-specific objects -Scalar = Union[str, Number, date, datetime] +Scalar = Union[str, Number, date, datetime, None] FieldValue = Union[None, Scalar, ObjectRow, tuple] Definition = Union["ObjectTemplate", "SimpleValue", "StructuredValue"] diff --git a/snowfakery/parse_factory_yaml.py b/snowfakery/parse_factory_yaml.py index 2bcd52e8..adae1ac8 100644 --- a/snowfakery/parse_factory_yaml.py +++ b/snowfakery/parse_factory_yaml.py @@ -53,7 +53,13 @@ def __init__(self, name): self._templates = [] def register(self, template: ObjectTemplate) -> None: - self.fields.update({field.name: field for field in template.fields}) + self.fields.update( + { + field.name: field + for field in template.fields + if not field.name.startswith("__") + } + ) self.friends.update({friend.tablename: friend for friend in template.friends}) self._templates.append(template) @@ -175,11 +181,7 @@ def parse_structured_value( def parse_field_value( name: str, field, context: ParseContext, allow_structured_values=True ) -> Union[SimpleValue, StructuredValue, ObjectTemplate]: - if field is None: - raise DataGenSyntaxError( - f"Field should not be empty: {name}", **context.line_num(field) - ) - if isinstance(field, (str, Number, date)): + if isinstance(field, (str, Number, date, type(None))): return SimpleValue(field, **(context.line_num(field) or context.line_num())) elif isinstance(field, dict) and field.get("object"): with context.change_current_parent_object(field): @@ -199,10 +201,6 @@ def parse_field_value( def parse_field(name: str, definition, context: ParseContext) -> FieldFactory: assert name, name - if definition is None: - raise DataGenSyntaxError( - f"Field should have a definition: {name}", **context.line_num(name) - ) return FieldFactory( name, parse_field_value(name, definition, context), diff --git a/snowfakery/utils/template_utils.py b/snowfakery/utils/template_utils.py index 6206b6f0..1f19474d 100644 --- a/snowfakery/utils/template_utils.py +++ b/snowfakery/utils/template_utils.py @@ -27,7 +27,7 @@ def __init__(self, func): self.func = func def __str__(self): - return self.func() + return str(self.func()) def __call__(self, *args, **kwargs): return self.func(*args, **kwargs) diff --git a/tests/test_errors.py b/tests/test_errors.py index efdad509..4a224c97 100644 --- a/tests/test_errors.py +++ b/tests/test_errors.py @@ -76,19 +76,6 @@ def test_extra_keys(self): generate(StringIO(yaml)) assert 4 > e.exception.line_num >= 2 - def test_missing_field_value(self): - yaml = """ - - object: Person - count: 5 - fields: - gender: Male - name: - """ - with self.assertRaises(DataGenError) as e: - generate(StringIO(yaml)) - assert "name" in str(e.exception) - assert "Field" in str(e.exception) - def test_missing_param(self): yaml = """ - object: Person @@ -100,4 +87,4 @@ def test_missing_param(self): """ with self.assertRaises(DataGenError) as e: generate(StringIO(yaml)) - assert "Field" in str(e.exception) + assert "Problem rendering field" in str(e.exception) diff --git a/tests/test_output_streams.py b/tests/test_output_streams.py index 0aa1ffb1..44df422c 100644 --- a/tests/test_output_streams.py +++ b/tests/test_output_streams.py @@ -64,6 +64,16 @@ def test_bool(self): values = self.do_output(yaml)["foo"][0] assert str(values["is_true"]) == "1" + def test_null(self): + yaml = """ + - object: foo + fields: + is_null: + """ + values = self.do_output(yaml)["foo"][0] + print(values) + assert values["is_null"] is None + def test_flushes(self): yaml = """ - object: foo @@ -139,9 +149,19 @@ def do_output(self, yaml): } return tables + def test_null(self): + yaml = """ + - object: foo + fields: + is_null: + """ + values = self.do_output(yaml)["foo"][0] + assert values["is_null"] is None + class JSONTables: def __init__(self, json_data, table_names): + self.raw = json_data self.data = json.loads(json_data) self.tables = {table_name: [] for table_name in table_names} for row in self.data: @@ -196,9 +216,28 @@ def test_from_cli(self): x = StringIO() with redirect_stdout(x): generate_cli.callback( - yaml_file=sample_yaml, output_format="json", output_files=["-"] + yaml_file=sample_yaml, output_format="json", ) - # TODO: more validation! + data = json.loads(x.getvalue()) + assert data == [ + { + "_table": "Account", + "id": 1, + "name": "Default Company Name", + "ShippingCountry": "Canada", + } + ] + + def test_null(self): + yaml = """ + - object: foo + fields: + is_null: + """ + output = self.do_output(yaml) + assert "null" in output.raw + values = output["foo"][0] + assert values["is_null"] is None class TestCSVOutputStream(unittest.TestCase, OutputCommonTests): @@ -245,3 +284,15 @@ def test_csv_output(self): "foo.csv", "bar.csv", } + + def test_null(self): + yaml = """ + - object: foo + fields: + is_null: + """ + values = self.do_output(yaml)["foo"][0] + print(values) + assert ( + values["is_null"] == "" + ) # CSV is no way of distingushing null from empty str