From c9c46bb4d3e30dad670ebbf5147a7e0121e3ea5f Mon Sep 17 00:00:00 2001 From: Sachin Sahu Date: Fri, 3 Nov 2023 10:50:36 +0000 Subject: [PATCH] Handle unknown categories Add custom heading mapping Fix code format Signed-off-by: Sachin Sahu --- .../release_notes-2.8.0.md | 420 ++++++++++++++++++ .../release_notes-2.9.0.md | 288 +++++++----- .../release_notes_urls-2.8.0.txt | 18 + src/run_releasenotes_check.py | 135 +++--- 4 files changed, 684 insertions(+), 177 deletions(-) create mode 100644 src/release_notes_automation/release_notes-2.8.0.md create mode 100644 src/release_notes_automation/release_notes_urls-2.8.0.txt diff --git a/src/release_notes_automation/release_notes-2.8.0.md b/src/release_notes_automation/release_notes-2.8.0.md new file mode 100644 index 0000000000..a9d5bfea05 --- /dev/null +++ b/src/release_notes_automation/release_notes-2.8.0.md @@ -0,0 +1,420 @@ +

OpenSearch and OpenSearch Dashboards 2.8.0 Release Notes

+

FEATURES

+ +

Opensearch Alerting

+ + + +

Opensearch Index Management

+ + + +

Opensearch Security

+ + + +

Opensearch Sql

+ + + +

Opensearch Common Utils

+ + + +

Opensearch Security Analytics

+ + + +

ENHANCEMENTS

+ +

Opensearch Ml Common

+ + + +

Opensearch Security

+ + + +

Opensearch Sql

+ + + +

Opensearch Cross Cluster Replication

+ + + +

Opensearch Knn

+ + + +

BUG FIXES

+ +

Opensearch Alerting

+ + + +

Opensearch Observability

+ + + +

Opensearch Ml Common

+ + + +

Opensearch Index Management

+ + + +

Opensearch Performance Analyzer

+ + + +

Opensearch Security

+ + + +

Opensearch Sql

+ + + +

Opensearch Cross Cluster Replication

+ + + +

Opensearch Reporting

+ + + +

Opensearch Security Analytics

+ + + +

Opensearch Notifications

+ + + +

INFRASTRUCTURE

+ +

Opensearch Observability

+ + + +

Opensearch Neural Search

+ + + +

Opensearch Performance Analyzer

+ + + +

Opensearch Anomaly Detection

+ + + +

Opensearch Geospatial

+ + + +

Opensearch Common Utils

+ + + +

Opensearch Knn

+ + + +

Opensearch Notifications

+ + + +

DOCUMENTATION

+ +

Opensearch Alerting

+ + + +

Opensearch Ml Common

+ +

Documentation

+ +

Opensearch Index Management

+ + + +

Opensearch Sql

+ + + +

Opensearch Common Utils

+ + + +

Opensearch Security Analytics

+ + + +

Opensearch Notifications

+ + + +

MAINTENANCE

+ +

Opensearch Alerting

+ + + +

Opensearch Observability

+ + + +

Opensearch Ml Common

+ + + +

Opensearch Index Management

+ + + +

Opensearch.job Scheduler

+ + + +

Opensearch Performance Analyzer

+ + + +

Opensearch Security

+ + + +

Opensearch Sql

+ + + +

Opensearch Reporting

+ + + +

Opensearch Geospatial

+ + + +

Opensearch Common Utils

+ + + +

Opensearch Asynchronous Search

+ + + +

Opensearch Security Analytics

+ + + +

Opensearch Notifications

+ + + +

REFACTORING

+ +

Opensearch Ml Common

+ + + +

EXPERIMENTAL

+ +

Opensearch Ml Common

+ + + +

ADDED [NEW CATEGORY]

+

Opensearch.job Scheduler

+ + diff --git a/src/release_notes_automation/release_notes-2.9.0.md b/src/release_notes_automation/release_notes-2.9.0.md index 36cd9d9036..7a603a3d45 100644 --- a/src/release_notes_automation/release_notes-2.9.0.md +++ b/src/release_notes_automation/release_notes-2.9.0.md @@ -1,27 +1,46 @@ -Release Notes 2.9.0 - -

BREAKING

+

OpenSearch and OpenSearch Dashboards 2.9.0 Release Notes

FEATURES

-

Opensearch Sql

-

Opensearch Knn

+ -

Opensearch Security Analytics

+

Opensearch Common Utils

+ + +

Opensearch Sql

+ + + +

Opensearch Alerting

+ +

Opensearch Ml Common

+ +

Opensearch Security Analytics

+ + +

ENHANCEMENTS

+

Opensearch Performance Analyzer

+ -

Opensearch Security

- -

Opensearch Sql

+ -

Opensearch Ml Common

+

Opensearch Anomaly Detection

+ -

Opensearch Anomaly Detection

+

Opensearch Security

+ + +

Opensearch Ml Common

+ +

BUG FIXES

+ +

Opensearch Neural Search

+ +

Bug Fixes

+

Fix update document with knnn_vector size not matching issue (#208)

+

Opensearch Performance Analyzer

+ +

Opensearch Common Utils

+ + + +

Opensearch Reporting

+ + +

Opensearch Sql

+

Opensearch Alerting

+ -

Opensearch Common Utils

- - -

Opensearch Security Analytics

- -

Opensearch Ml Common

+ -

Opensearch Neural Search

-

Bug Fixes

-

Fix update document with knnn_vector size not matching issue (#208)

+

Opensearch Security Analytics

-

Opensearch Reporting

INFRASTRUCTURE

+ +

Opensearch Notifications

+ + +

Opensearch Performance Analyzer

+

Opensearch Sql

+ +

Opensearch Anomaly Detection

+ + +

Opensearch Ml Common

+ +

DOCUMENTATION

+

Opensearch Notifications

+ -

Opensearch Anomaly Detection

+

Opensearch Common Utils

+ -

DOCUMENTATION

Opensearch Sql

+

Opensearch Alerting

+ -

Opensearch Common Utils

- +

Opensearch Anomaly Detection

-

Opensearch Security Analytics

Opensearch Ml Common

+ -

Opensearch Notifications

+

Opensearch Security Analytics

+ -

Opensearch Anomaly Detection

+

MAINTENANCE

+ +

Opensearch Neural Search

+ +

Maintenance

+

Increment version to 2.9.0-SNAPSHOT (#191)

+ +

Opensearch Notifications

+ -

MAINTENANCE

Opensearch Performance Analyzer

+ +

Opensearch Geospatial

+ +

Maintenance

+

Increment version to 2.9.0-SNAPSHOT (#329)

+ +

Opensearch Common Utils

+ + + +

Opensearch Reporting

+ + + +

Opensearch Asynchronous Search

+ + + +

Opensearch Alerting

+ + +

Opensearch Security

+ -

Opensearch Alerting

- +

Opensearch Ml Common

-

Opensearch Common Utils

Opensearch Security Analytics

+ -

Opensearch Asynchronous Search

- +

REFACTORING

-

Opensearch Ml Common

+

Opensearch Observability

+ +

Refactoring

Opensearch Geospatial

-

Maintenance

-

Increment version to 2.9.0-SNAPSHOT (#329)

-

Opensearch Neural Search

-

Maintenance

-

Increment version to 2.9.0-SNAPSHOT (#191)

+

Refactoring

+

Change package for Strings.hasText (#314)

-

Opensearch Notifications

- +

Opensearch Common Utils

-

Opensearch Reporting

-

REFACTORING

Opensearch Sql

+

Opensearch Alerting

- -

Opensearch Common Utils

Opensearch Security Analytics

- -

Opensearch Geospatial

-

Refactoring

-

Change package for Strings.hasText (#314)

- -

Opensearch Observability

-

Refactoring

-

EXPERIMENTAL

diff --git a/src/release_notes_automation/release_notes_urls-2.8.0.txt b/src/release_notes_automation/release_notes_urls-2.8.0.txt new file mode 100644 index 0000000000..6c8820a2fc --- /dev/null +++ b/src/release_notes_automation/release_notes_urls-2.8.0.txt @@ -0,0 +1,18 @@ +https://raw.githubusercontent.com/opensearch-project/alerting/2.8.0.0/release-notes/opensearch-alerting.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/anomaly-detection/2.8.0.0/release-notes/opensearch-anomaly-detection.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/asynchronous-search/2.8.0.0/release-notes/opensearch-asynchronous-search.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/common-utils/2.8.0.0/release-notes/opensearch-common-utils.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/cross-cluster-replication/2.8.0.0/release-notes/opensearch-cross-cluster-replication.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/geospatial/2.8.0.0/release-notes/opensearch-geospatial.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/index-management/2.8.0.0/release-notes/opensearch-index-management.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/job-scheduler/2.8.0.0/release-notes/opensearch.job-scheduler.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/k-NN/2.8.0.0/release-notes/opensearch-knn.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/ml-commons/2.8.0.0/release-notes/opensearch-ml-common.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/neural-search/2.8.0.0/release-notes/opensearch-neural-search.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/notifications/2.8.0.0/release-notes/opensearch-notifications.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/observability/2.8.0.0/release-notes/opensearch-observability.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/reporting/2.8.0.0/release-notes/opensearch-reporting.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/performance-analyzer/2.8.0.0/release-notes/opensearch-performance-analyzer.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/security/2.8.0.0/release-notes/opensearch-security.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/security-analytics/2.8.0.0/release-notes/opensearch-security-analytics.release-notes-2.8.0.0.md +https://raw.githubusercontent.com/opensearch-project/sql/2.8.0.0/release-notes/opensearch-sql.release-notes-2.8.0.0.md \ No newline at end of file diff --git a/src/run_releasenotes_check.py b/src/run_releasenotes_check.py index 57505d2ac8..5e232d4b86 100644 --- a/src/run_releasenotes_check.py +++ b/src/run_releasenotes_check.py @@ -24,25 +24,23 @@ def main() -> int: console.configure(level=args.logging_level) manifest_file = InputManifest.from_file(args.manifest) BUILD_VERSION = manifest_file.build.version - # print(f"BUILD_VERSION: {BUILD_VERSION}") - BASE_FILE_PATH = 'release_notes_automation' - table_filename = f'{BASE_FILE_PATH}/release_notes_table-{BUILD_VERSION}.md' - urls_filename = f'{BASE_FILE_PATH}/release_notes_urls-{BUILD_VERSION}.txt' + BASE_FILE_PATH = "release_notes_automation" + table_filename = f"{BASE_FILE_PATH}/release_notes_table-{BUILD_VERSION}.md" + urls_filename = f"{BASE_FILE_PATH}/release_notes_urls-{BUILD_VERSION}.txt" - def format_component_name_from_url(url): - start_index = url.find('release-notes/') + def format_component_name_from_url(url) -> str: + start_index = url.find("release-notes/") if start_index == -1: raise ValueError("'release-notes/' not found in the URL") - end_index = url.find('.release-notes', start_index) + end_index = url.find(".release-notes", start_index) if end_index == -1: raise ValueError("'.release-notes' not found after 'release-notes/'") - component_name = url[start_index + len('release-notes/'):end_index] - formatted_name = ' '.join(word.capitalize() for word in component_name.split('-')) + component_name = url[start_index + len("release-notes/") : end_index] + formatted_name = " ".join(word.capitalize() for word in component_name.split("-")) return formatted_name def create_urls_file_if_not_exists() -> None: - # print("enter create_urls_file function") urls_filepath = os.path.join(os.path.dirname(__file__), urls_filename) if os.path.exists(urls_filepath): print("URLs file already exists. Skipping creation.") @@ -54,37 +52,35 @@ def create_urls_file_if_not_exists() -> None: table_filepath = os.path.join(os.path.dirname(__file__), table_filename) os.makedirs(os.path.dirname(table_filepath), exist_ok=True) - with open(table_filepath, 'w') as table_file: + with open(table_filepath, "w") as table_file: table.dump(table_file) if args.output is not None: print(f"Moving {table_filepath} to {args.output}") shutil.move(table_filepath, args.output) else: - with open(table_filepath, 'r') as table_file: + with open(table_filepath, "r") as table_file: print(table_file.read()) urls = [row[-1].strip() for row in table.value_matrix if row[-1]] os.makedirs(os.path.dirname(urls_filepath), exist_ok=True) urls_filepath = os.path.join(os.path.dirname(__file__), urls_filename) - with open(urls_filepath, 'w') as urls_file: - urls_file.writelines('\n'.join(urls)) + with open(urls_filepath, "w") as urls_file: + urls_file.writelines("\n".join(urls)) if args.action == "check": - # print("check") create_urls_file_if_not_exists() elif args.action == "compile": - # print("compile") create_urls_file_if_not_exists() RELEASENOTES_CATEGORIES = "BREAKING,FEATURES,ENHANCEMENTS,BUG FIXES,INFRASTRUCTURE,DOCUMENTATION,MAINTENANCE,REFACTORING,EXPERIMENTAL" - RELEASE_NOTE_MD = f'{BASE_FILE_PATH}/release_notes-{BUILD_VERSION}.md' + RELEASE_NOTE_MD = f"{BASE_FILE_PATH}/release_notes-{BUILD_VERSION}.md" # Clean up URLs in the file urls_filepath = os.path.join(os.path.dirname(__file__), urls_filename) - with open(urls_filepath, 'r') as file: + with open(urls_filepath, "r") as file: urls = [line.strip() for line in file if line.strip()] unique_urls = list(set(urls)) @@ -92,86 +88,97 @@ def create_urls_file_if_not_exists() -> None: # store plugin data plugin_data = defaultdict(lambda: defaultdict(list)) - # # TODO: store unknown categories - # unknown_categories = defaultdict(list) - + # handle custom headings + heading_mapping = { + "Feature": "Features", + "Feat": "Features", + "Experimental Features": "Experimental", + "Refactor": "Refactoring", + "Enhancement": "Enhancements", + "Bug Fix": "Bug Fixes", + } + unique_headings = set() for url in unique_urls: if not url.startswith("#"): response = requests.get(url) - # print(f"Processing URL: {url}") if response.status_code == 200: content = response.text plugin_name = format_component_name_from_url(url) - # print(f"Plugin Name: {plugin_name}") # obtain headings (###) from the content - headings = [match.strip() for match in re.findall(r'###.+', content)] + headings = [match.strip() for match in re.findall(r"###.+", content)] if not headings: continue - # print(f"Headings: {headings}") # Store content under each heading in respective plugin for i in range(len(headings)): heading = headings[i].strip() - if heading.startswith("### "): heading = heading[4:] - # print(f"Heading 1: {heading}") + heading = heading.title() + + if heading in heading_mapping: + heading = heading_mapping[heading] + unique_headings.add(heading) + content_start = content.find(headings[i]) if content_start != -1: if i == len(headings) - 1: content_to_end = content[content_start:] else: - content_to_end = content[content_start:content.find(headings[i + 1])] + content_to_end = content[content_start : content.find(headings[i + 1])] # remove heading from obtained content to avoid duplication - parts = content_to_end.split('*', 1) + parts = content_to_end.split("*", 1) if len(parts) == 2: - content_to_end = '*' + parts[1] + content_to_end = "*" + parts[1] plugin_data[plugin_name][heading].append(content_to_end) - # print("=====================================================") # print(plugin_data[plugin_name][heading]) - # print("=====================================================") - # print("Compilation complete.") + print("Compilation complete.") + # print("Unique Headings:") + # for heading in sorted(unique_headings): + # print(heading) # Markdown renderer markdown = mistune.create_markdown() RELEASE_NOTE_MD_path = os.path.join(os.path.dirname(__file__), RELEASE_NOTE_MD) os.makedirs(os.path.dirname(RELEASE_NOTE_MD_path), exist_ok=True) - # print(f"RELEASE_NOTE_MD_path: {RELEASE_NOTE_MD_path}") # Filter content for each category - with open(RELEASE_NOTE_MD_path, 'w') as outfile: + with open(RELEASE_NOTE_MD_path, "w") as outfile: outfile.write(markdown(f"# OpenSearch and OpenSearch Dashboards {BUILD_VERSION} Release Notes\n\n")) - for category in RELEASENOTES_CATEGORIES.split(','): - outfile.write(markdown(f'\n## {category}\n\n')) - for plugin, categories in plugin_data.items(): - # print(f"Plugin: {plugin}") - # print(f"Categories: {categories}") - # print("=====================================================") - if category.lower() in [cat.lower() for cat in categories.keys()]: - # print(f"Category: {category}") - outfile.write(markdown(f'\n### {plugin}\n\n')) + for category in RELEASENOTES_CATEGORIES.split(","): + # Discard category content if no data is available + temp_content = [] + temp_content.append(markdown(f"\n## {category}\n\n")) + for plugin, categories in plugin_data.items(): + if category.lower() in [cat.lower() for cat in categories.keys()]: + temp_content.append(markdown(f"\n### {plugin}\n\n")) for cat, content_list in categories.items(): - # print(f"cat: {cat}") - # print(f"content_list: {content_list}") if cat.lower() == category.lower(): for content in content_list: - # print("=====================================================") - # print(f"Content: {content}") - outfile.write(markdown(content)) - outfile.write('\n') - # for content in categories[category]: - # print("=====================================================") - # print(f"categories[category]: {categories[category]}") - # print(f"Content: {content}") - # outfile.write(markdown(content)) - # outfile.write('\n') - - # print("=====================================================") + if content.strip(): + temp_content.append(markdown(content)) + + if len(temp_content) > 1: + outfile.write("\n".join(temp_content)) + outfile.write("\n") + else: + print(f"\n## {category} was empty\n\n") + + # Handle unknown categories + for plugin, categories in plugin_data.items(): + for cat, content_list in categories.items(): + if cat.lower() not in RELEASENOTES_CATEGORIES.lower(): + outfile.write(markdown(f"\n## {cat.upper()} [NEW CATEGORY]\n\n")) + outfile.write(markdown(f"\n### {plugin}\n\n")) + for content in content_list: + outfile.write(markdown(content)) + outfile.write("\n") + if args.output is not None: print(f"Moving {RELEASE_NOTE_MD} to {args.output}") shutil.move(RELEASE_NOTE_MD_path, args.output) @@ -179,18 +186,6 @@ def create_urls_file_if_not_exists() -> None: print(f"Release notes compiled to {RELEASE_NOTE_MD_path}") return 0 -# sub categories obtained: -# ### Infrastructure -# ### Enhancements -# ### Feature -# ### Maintenance -# ### Bug Fixes -# ### Documentation -# ### Features -# ### Added -# ### Experimental Features -# ### Refactoring - if __name__ == "__main__": main()