From 4c91c2e4ba1131722d2e4e0a2eba64dd3f8ef32b Mon Sep 17 00:00:00 2001 From: Jonathan Buttner <56361221+jonathan-buttner@users.noreply.github.com> Date: Fri, 14 Feb 2020 10:39:40 -0500 Subject: [PATCH] Adding --out flag (#748) Allows for the artifacts to be output to a different directory. This is useful for users who leverage the ECS tooling to manage their Elasticsearch templates. --- CHANGELOG.next.md | 1 + Makefile | 2 +- scripts/generator.py | 28 +++++++++++++++++------- scripts/generators/asciidoc_fields.py | 11 +++++----- scripts/generators/beats.py | 12 +++++----- scripts/generators/csv_generator.py | 8 +++++-- scripts/generators/ecs_helpers.py | 8 +++++++ scripts/generators/es_template.py | 12 +++++----- scripts/generators/intermediate_files.py | 8 ++++--- scripts/schema_reader.py | 11 ++++++++++ 10 files changed, 70 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.next.md b/CHANGELOG.next.md index ffd18264f7..c1bd4f5c52 100644 --- a/CHANGELOG.next.md +++ b/CHANGELOG.next.md @@ -40,6 +40,7 @@ Thanks, you're awesome :-) --> * schema_reader.py now reliably supports chaining reusable fieldsets together. #722 * Allow the artifact generator to consider and output only a subset of fields. #737 * Add support for reusing fields in places other than the top level of the destination fieldset. #739 +* Add support for specifying the directory to write the generated files. #748 #### Deprecated diff --git a/Makefile b/Makefile index 7fce7deb60..9b934b1ba4 100644 --- a/Makefile +++ b/Makefile @@ -101,7 +101,7 @@ setup: ve # Run the ECS tests .PHONY: test -test: +test: ve $(PYTHON) -m unittest discover --start-directory scripts/tests # Create a virtualenv to run Python. diff --git a/scripts/generator.py b/scripts/generator.py index 6afeb5fc9d..b232197d38 100644 --- a/scripts/generator.py +++ b/scripts/generator.py @@ -23,11 +23,11 @@ def main(): # Maybe load user specified directory of schemas if args.include: - include_glob = os.path.join(args.include, '*.yml') + include_glob = schema_reader.get_glob_files(args.include, schema_reader.YAML_EXT) print('Loading user defined schemas: {0}'.format(include_glob)) - intermediate_custom = schema_reader.load_schemas(sorted(glob.glob(include_glob))) + intermediate_custom = schema_reader.load_schemas(include_glob) schema_reader.merge_schema_fields(intermediate_fields, intermediate_custom) if args.subset: @@ -40,24 +40,36 @@ def main(): intermediate_fields = ecs_helpers.fields_subset(subset, intermediate_fields) (nested, flat) = schema_reader.generate_nested_flat(intermediate_fields) - intermediate_files.generate(nested, flat) + + # default location to save files + out_dir = 'generated' + docs_dir = 'docs' + if args.out: + out_dir = os.path.join(args.out, out_dir) + docs_dir = os.path.join(args.out, docs_dir) + + ecs_helpers.make_dirs(out_dir) + ecs_helpers.make_dirs(docs_dir) + + intermediate_files.generate(nested, flat, out_dir) if args.intermediate_only: exit() - csv_generator.generate(flat, ecs_version) - es_template.generate(flat, ecs_version) - beats.generate(nested, ecs_version) - asciidoc_fields.generate(nested, flat, ecs_version) + csv_generator.generate(flat, ecs_version, out_dir) + es_template.generate(flat, ecs_version, out_dir) + beats.generate(nested, ecs_version, out_dir) + asciidoc_fields.generate(nested, flat, ecs_version, docs_dir) def argument_parser(): parser = argparse.ArgumentParser() parser.add_argument('--intermediate-only', action='store_true', help='generate intermediary files only') - parser.add_argument('--include', action='store', + parser.add_argument('--include', nargs='+', help='include user specified directory of custom field definitions') parser.add_argument('--subset', nargs='+', help='render a subset of the schema') + parser.add_argument('--out', action='store', help='directory to store the generated files') return parser.parse_args() diff --git a/scripts/generators/asciidoc_fields.py b/scripts/generators/asciidoc_fields.py index 869b8fbc25..6faa689d7f 100644 --- a/scripts/generators/asciidoc_fields.py +++ b/scripts/generators/asciidoc_fields.py @@ -1,12 +1,11 @@ -import sys - +from os.path import join from generators import ecs_helpers -def generate(ecs_nested, ecs_flat, ecs_version): - save_asciidoc('docs/fields.asciidoc', page_field_index(ecs_nested, ecs_version)) - save_asciidoc('docs/field-details.asciidoc', page_field_details(ecs_nested)) - save_asciidoc('docs/field-values.asciidoc', page_field_values(ecs_flat)) +def generate(ecs_nested, ecs_flat, ecs_version, out_dir): + save_asciidoc(join(out_dir, 'fields.asciidoc'), page_field_index(ecs_nested, ecs_version)) + save_asciidoc(join(out_dir, 'field-details.asciidoc'), page_field_details(ecs_nested)) + save_asciidoc(join(out_dir, 'field-values.asciidoc'), page_field_values(ecs_flat)) # Helpers diff --git a/scripts/generators/beats.py b/scripts/generators/beats.py index d1a836599c..fdad322e9d 100644 --- a/scripts/generators/beats.py +++ b/scripts/generators/beats.py @@ -1,10 +1,9 @@ -import yaml - +from os.path import join from collections import OrderedDict from generators import ecs_helpers -def generate(ecs_nested, ecs_version): +def generate(ecs_nested, ecs_version, out_dir): # Load temporary whitelist for default_fields workaround. df_whitelist = ecs_helpers.yaml_load('scripts/generators/beats_default_fields_whitelist.yml') @@ -28,7 +27,7 @@ def generate(ecs_nested, ecs_version): beats_file['description'] = 'ECS Fields.' beats_file['fields'] = beats_fields - write_beats_yaml(beats_file, ecs_version) + write_beats_yaml(beats_file, ecs_version, out_dir) def fieldset_field_array(source_fields, df_whitelist): @@ -65,9 +64,10 @@ def fieldset_field_array(source_fields, df_whitelist): # Helpers -def write_beats_yaml(beats_file, ecs_version): +def write_beats_yaml(beats_file, ecs_version, out_dir): + ecs_helpers.make_dirs(join(out_dir, 'beats')) warning = file_header().format(version=ecs_version) - ecs_helpers.yaml_dump('generated/beats/fields.ecs.yml', [beats_file], preamble=warning) + ecs_helpers.yaml_dump(join(out_dir, 'beats/fields.ecs.yml'), [beats_file], preamble=warning) # Templates diff --git a/scripts/generators/csv_generator.py b/scripts/generators/csv_generator.py index ddd79b5d88..e1f6bad89e 100644 --- a/scripts/generators/csv_generator.py +++ b/scripts/generators/csv_generator.py @@ -1,10 +1,14 @@ import csv import sys +from os.path import join +from generator import ecs_helpers -def generate(ecs_flat, version): + +def generate(ecs_flat, version, out_dir): + ecs_helpers.make_dirs(join(out_dir, 'csv')) sorted_fields = base_first(ecs_flat) - save_csv('generated/csv/fields.csv', sorted_fields, version) + save_csv(join(out_dir, 'csv/fields.csv'), sorted_fields, version) def base_first(ecs_flat): diff --git a/scripts/generators/ecs_helpers.py b/scripts/generators/ecs_helpers.py index 42d02aebc4..cce9792e24 100644 --- a/scripts/generators/ecs_helpers.py +++ b/scripts/generators/ecs_helpers.py @@ -1,4 +1,5 @@ import yaml +import os from collections import OrderedDict from copy import deepcopy @@ -96,6 +97,13 @@ def dict_rename_keys(dict, renames): # File helpers +def make_dirs(path): + try: + os.makedirs(path, exist_ok=True) + except OSError as e: + print('Unable to create output directory: {}'.format(e)) + raise e + def yaml_dump(filename, data, preamble=None): with open(filename, 'w') as outfile: diff --git a/scripts/generators/es_template.py b/scripts/generators/es_template.py index 0ac6a296c8..6a04461008 100644 --- a/scripts/generators/es_template.py +++ b/scripts/generators/es_template.py @@ -1,10 +1,11 @@ import json import sys +from os.path import join from generators import ecs_helpers -def generate(ecs_flat, ecs_version): +def generate(ecs_flat, ecs_version, out_dir): field_mappings = {} for flat_name in sorted(ecs_flat): field = ecs_flat[flat_name] @@ -14,8 +15,8 @@ def generate(ecs_flat, ecs_version): mappings_section = mapping_settings(ecs_version) mappings_section['properties'] = field_mappings - generate_template_version(6, mappings_section) - generate_template_version(7, mappings_section) + generate_template_version(6, mappings_section, out_dir) + generate_template_version(7, mappings_section, out_dir) # Field mappings @@ -65,14 +66,15 @@ def entry_for(field): # Generated files -def generate_template_version(elasticsearch_version, mappings_section): +def generate_template_version(elasticsearch_version, mappings_section, out_dir): + ecs_helpers.make_dirs(join(out_dir, 'elasticsearch', str(elasticsearch_version))) template = template_settings() if elasticsearch_version == 6: template['mappings'] = {'_doc': mappings_section} else: template['mappings'] = mappings_section - filename = "generated/elasticsearch/{}/template.json".format(elasticsearch_version) + filename = join(out_dir, "elasticsearch/{}/template.json".format(elasticsearch_version)) save_json(filename, template) diff --git a/scripts/generators/intermediate_files.py b/scripts/generators/intermediate_files.py index 1b50e02d08..cc9234e5e9 100644 --- a/scripts/generators/intermediate_files.py +++ b/scripts/generators/intermediate_files.py @@ -1,6 +1,8 @@ from generators import ecs_helpers +from os.path import join -def generate(ecs_nested, ecs_flat): - ecs_helpers.yaml_dump('generated/ecs/ecs_flat.yml', ecs_flat) - ecs_helpers.yaml_dump('generated/ecs/ecs_nested.yml', ecs_nested) +def generate(ecs_nested, ecs_flat, out_dir): + ecs_helpers.make_dirs(join(out_dir, 'ecs')) + ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_flat.yml'), ecs_flat) + ecs_helpers.yaml_dump(join(out_dir, 'ecs/ecs_nested.yml'), ecs_nested) diff --git a/scripts/schema_reader.py b/scripts/schema_reader.py index 5e8a7c8e6e..806efcc54f 100644 --- a/scripts/schema_reader.py +++ b/scripts/schema_reader.py @@ -5,6 +5,17 @@ # File loading stuff +YAML_EXT = ('*.yml', '*.yaml') + + +def get_glob_files(paths, file_types): + all_files = [] + for path in paths: + for t in file_types: + all_files.extend(glob.glob(os.path.join(path, t))) + + return sorted(all_files) + def ecs_files(): """Return the schema file list to load"""