From c37d2f3bda94fcb4a237019ec2f8a8a98954c73f Mon Sep 17 00:00:00 2001 From: Danil Alexeev Date: Fri, 11 Oct 2024 17:35:28 +0300 Subject: [PATCH] POT Generator: Add support for `TRANSLATORS:` and `NO_TRANSLATE` comments --- doc/classes/EditorTranslationParserPlugin.xml | 8 ++ editor/editor_translation_parser.cpp | 17 +++- editor/editor_translation_parser.h | 2 + editor/pot_generator.cpp | 28 ++++-- editor/pot_generator.h | 3 +- .../gdscript_translation_parser_plugin.cpp | 90 +++++++++++++++++-- .../gdscript_translation_parser_plugin.h | 15 +++- modules/gdscript/gdscript_parser.cpp | 4 + modules/gdscript/gdscript_parser.h | 2 + 9 files changed, 151 insertions(+), 18 deletions(-) diff --git a/doc/classes/EditorTranslationParserPlugin.xml b/doc/classes/EditorTranslationParserPlugin.xml index a47a41594ef7..b40a4976a36b 100644 --- a/doc/classes/EditorTranslationParserPlugin.xml +++ b/doc/classes/EditorTranslationParserPlugin.xml @@ -100,6 +100,14 @@ + + + + + + If overridden, called after [method _parse_file] to get comments for the parsed entries. This method should fill the arrays with the same number of elements and in the same order as [method _parse_file]. + + diff --git a/editor/editor_translation_parser.cpp b/editor/editor_translation_parser.cpp index 8a77ce4a82ff..d12bbc1af2f9 100644 --- a/editor/editor_translation_parser.cpp +++ b/editor/editor_translation_parser.cpp @@ -31,7 +31,6 @@ #include "editor_translation_parser.h" #include "core/error/error_macros.h" -#include "core/io/file_access.h" #include "core/object/script_language.h" #include "core/templates/hash_set.h" @@ -65,6 +64,21 @@ Error EditorTranslationParserPlugin::parse_file(const String &p_path, Vector *r_ids_comment, Vector *r_ids_ctx_plural_comment) { + TypedArray ids_comment; + TypedArray ids_ctx_plural_comment; + + if (GDVIRTUAL_CALL(_get_comments, ids_comment, ids_ctx_plural_comment)) { + for (int i = 0; i < ids_comment.size(); i++) { + r_ids_comment->append(ids_comment[i]); + } + + for (int i = 0; i < ids_ctx_plural_comment.size(); i++) { + r_ids_ctx_plural_comment->append(ids_ctx_plural_comment[i]); + } + } +} + void EditorTranslationParserPlugin::get_recognized_extensions(List *r_extensions) const { Vector extensions; if (GDVIRTUAL_CALL(_get_recognized_extensions, extensions)) { @@ -78,6 +92,7 @@ void EditorTranslationParserPlugin::get_recognized_extensions(List *r_ex void EditorTranslationParserPlugin::_bind_methods() { GDVIRTUAL_BIND(_parse_file, "path", "msgids", "msgids_context_plural"); + GDVIRTUAL_BIND(_get_comments, "msgids_comment", "msgids_context_plural_comment"); GDVIRTUAL_BIND(_get_recognized_extensions); } diff --git a/editor/editor_translation_parser.h b/editor/editor_translation_parser.h index 78dc726c52e0..20c8ed79391b 100644 --- a/editor/editor_translation_parser.h +++ b/editor/editor_translation_parser.h @@ -43,10 +43,12 @@ class EditorTranslationParserPlugin : public RefCounted { static void _bind_methods(); GDVIRTUAL3(_parse_file, String, TypedArray, TypedArray) + GDVIRTUAL2(_get_comments, TypedArray, TypedArray) GDVIRTUAL0RC(Vector, _get_recognized_extensions) public: virtual Error parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural); + virtual void get_comments(Vector *r_ids_comment, Vector *r_ids_ctx_plural_comment); virtual void get_recognized_extensions(List *r_extensions) const; }; diff --git a/editor/pot_generator.cpp b/editor/pot_generator.cpp index 76b6593f1d58..6aea12c04b8f 100644 --- a/editor/pot_generator.cpp +++ b/editor/pot_generator.cpp @@ -69,11 +69,16 @@ void POTGenerator::generate_pot(const String &p_file) { for (int i = 0; i < files.size(); i++) { Vector msgids; Vector> msgids_context_plural; + + Vector msgids_comment; + Vector msgids_context_plural_comment; + const String &file_path = files[i]; String file_extension = file_path.get_extension(); if (EditorTranslationParser::get_singleton()->can_parse(file_extension)) { EditorTranslationParser::get_singleton()->get_parser(file_extension)->parse_file(file_path, &msgids, &msgids_context_plural); + EditorTranslationParser::get_singleton()->get_parser(file_extension)->get_comments(&msgids_comment, &msgids_context_plural_comment); } else { ERR_PRINT("Unrecognized file extension " + file_extension + " in generate_pot()"); return; @@ -81,16 +86,18 @@ void POTGenerator::generate_pot(const String &p_file) { for (int j = 0; j < msgids_context_plural.size(); j++) { const Vector &entry = msgids_context_plural[j]; - _add_new_msgid(entry[0], entry[1], entry[2], file_path); + const String &comment = (j < msgids_context_plural_comment.size()) ? msgids_context_plural_comment[j] : String(); + _add_new_msgid(entry[0], entry[1], entry[2], file_path, comment); } for (int j = 0; j < msgids.size(); j++) { - _add_new_msgid(msgids[j], "", "", file_path); + const String &comment = (j < msgids_comment.size()) ? msgids_comment[j] : String(); + _add_new_msgid(msgids[j], "", "", file_path, comment); } } if (GLOBAL_GET("internationalization/locale/translation_add_builtin_strings_to_pot")) { for (const Vector &extractable_msgids : get_extractable_message_list()) { - _add_new_msgid(extractable_msgids[0], extractable_msgids[1], extractable_msgids[2], ""); + _add_new_msgid(extractable_msgids[0], extractable_msgids[1], extractable_msgids[2], "", ""); } } @@ -136,10 +143,18 @@ void POTGenerator::_write_to_pot(const String &p_file) { String context = v_msgid_data[i].ctx; String plural = v_msgid_data[i].plural; const HashSet &locations = v_msgid_data[i].locations; + const HashSet &comments = v_msgid_data[i].comments; // Put the blank line at the start, to avoid a double at the end when closing the file. file->store_line(""); + // Write comments. + for (const String &E : comments) { + if (!E.is_empty()) { + file->store_line("#. " + E.replace("\n", "\n#. ")); + } + } + // Write file locations. for (const String &E : locations) { if (!E.is_empty()) { @@ -199,7 +214,7 @@ void POTGenerator::_write_msgid(Ref r_file, const String &p_id, bool } } -void POTGenerator::_add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location) { +void POTGenerator::_add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location, const String &p_comment) { // Insert new location if msgid under same context exists already. if (all_translation_strings.has(p_msgid)) { Vector &v_mdata = all_translation_strings[p_msgid]; @@ -209,17 +224,18 @@ void POTGenerator::_add_new_msgid(const String &p_msgid, const String &p_context WARN_PRINT("Redefinition of plural message (msgid_plural), under the same message (msgid) and context (msgctxt)"); } v_mdata.write[i].locations.insert(p_location); + v_mdata.write[i].comments.insert(p_comment); return; } } } - // Add a new entry of msgid, context, plural and location - context and plural might be empty if the inserted msgid doesn't associated - // context or plurals. + // Add a new entry. MsgidData mdata; mdata.ctx = p_context; mdata.plural = p_plural; mdata.locations.insert(p_location); + mdata.comments.insert(p_comment); all_translation_strings[p_msgid].push_back(mdata); } diff --git a/editor/pot_generator.h b/editor/pot_generator.h index 8bcb2e5cac36..54f4aa06522b 100644 --- a/editor/pot_generator.h +++ b/editor/pot_generator.h @@ -44,13 +44,14 @@ class POTGenerator { String ctx; String plural; HashSet locations; + HashSet comments; }; // Store msgid as key and the additional data around the msgid - if it's under a context, has plurals and its file locations. HashMap> all_translation_strings; void _write_to_pot(const String &p_file); void _write_msgid(Ref r_file, const String &p_id, bool p_plural); - void _add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location); + void _add_new_msgid(const String &p_msgid, const String &p_context, const String &p_plural, const String &p_location, const String &p_comment); #ifdef DEBUG_POT void _print_all_translation_strings(); diff --git a/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp b/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp index b31ae878cef6..f7b65e5cba6d 100644 --- a/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp +++ b/modules/gdscript/editor/gdscript_translation_parser_plugin.cpp @@ -51,6 +51,10 @@ Error GDScriptEditorTranslationParserPlugin::parse_file(const String &p_path, Ve ids = r_ids; ids_ctx_plural = r_ids_ctx_plural; + + ids_comment.clear(); + ids_ctx_plural_comment.clear(); + Ref gdscript = loaded_res; String source_code = gdscript->get_source_code(); @@ -62,18 +66,86 @@ Error GDScriptEditorTranslationParserPlugin::parse_file(const String &p_path, Ve err = analyzer.analyze(); ERR_FAIL_COND_V_MSG(err, err, "Failed to analyze GDScript with GDScriptAnalyzer."); + comment_data = &parser.comment_data; + // Traverse through the parsed tree from GDScriptParser. GDScriptParser::ClassNode *c = parser.get_tree(); _traverse_class(c); + comment_data = nullptr; + return OK; } +void GDScriptEditorTranslationParserPlugin::get_comments(Vector *r_ids_comment, Vector *r_ids_ctx_plural_comment) { + r_ids_comment->append_array(ids_comment); + r_ids_ctx_plural_comment->append_array(ids_ctx_plural_comment); +} + bool GDScriptEditorTranslationParserPlugin::_is_constant_string(const GDScriptParser::ExpressionNode *p_expression) { ERR_FAIL_NULL_V(p_expression, false); return p_expression->is_constant && p_expression->reduced_value.is_string(); } +String GDScriptEditorTranslationParserPlugin::_parse_comment(int p_line, bool &r_skip) const { + // Parse inline comment. + if (comment_data->has(p_line)) { + const String stripped_comment = comment_data->get(p_line).comment.trim_prefix("#").strip_edges(); + + if (stripped_comment.begins_with("TRANSLATORS:")) { + return stripped_comment; + } + if (stripped_comment == "NO_TRANSLATE" || stripped_comment.begins_with("NO_TRANSLATE:")) { + r_skip = true; + return String(); + } + } + + // Parse multiline comment. + String multiline_comment; + for (int line = p_line - 1; comment_data->has(line); line--) { + const String stripped_comment = comment_data->get(line).comment.trim_prefix("#").strip_edges(); + + if (multiline_comment.is_empty()) { + multiline_comment = stripped_comment; + } else { + multiline_comment = stripped_comment + "\n" + multiline_comment; + } + + if (stripped_comment.begins_with("TRANSLATORS:")) { + return multiline_comment; + } + if (stripped_comment == "NO_TRANSLATE" || stripped_comment.begins_with("NO_TRANSLATE:")) { + r_skip = true; + return String(); + } + } + + return String(); +} + +void GDScriptEditorTranslationParserPlugin::_add_id(const String &p_id, int p_line) { + bool skip = false; + const String comment = _parse_comment(p_line, skip); + if (skip) { + return; + } + + ids->push_back(p_id); + ids_comment.push_back(comment); +} + +void GDScriptEditorTranslationParserPlugin::_add_id_ctx_plural(const Vector &p_id_ctx_plural, int p_line) { + bool skip = false; + const String comment = _parse_comment(p_line, skip); + if (skip) { + return; + } + + ids_ctx_plural->push_back(p_id_ctx_plural); + ids_ctx_plural_comment.push_back(comment); +} + void GDScriptEditorTranslationParserPlugin::_traverse_class(const GDScriptParser::ClassNode *p_class) { for (int i = 0; i < p_class->members.size(); i++) { const GDScriptParser::ClassNode::Member &m = p_class->members[i]; @@ -253,7 +325,7 @@ void GDScriptEditorTranslationParserPlugin::_assess_assignment(const GDScriptPar if (assignee_name != StringName() && assignment_patterns.has(assignee_name) && _is_constant_string(p_assignment->assigned_value)) { // If the assignment is towards one of the extract patterns (text, tooltip_text etc.), and the value is a constant string, we collect the string. - ids->push_back(p_assignment->assigned_value->reduced_value); + _add_id(p_assignment->assigned_value->reduced_value, p_assignment->assigned_value->start_line); } else if (assignee_name == fd_filters) { // Extract from `get_node("FileDialog").filters = `. _extract_fd_filter_array(p_assignment->assigned_value); @@ -287,7 +359,7 @@ void GDScriptEditorTranslationParserPlugin::_assess_call(const GDScriptParser::C } } if (extract_id_ctx_plural) { - ids_ctx_plural->push_back(id_ctx_plural); + _add_id_ctx_plural(id_ctx_plural, p_call->start_line); } } else if (function_name == trn_func || function_name == atrn_func) { // Extract from `tr_n(id, plural, n, ctx)` or `atr_n(id, plural, n, ctx)`. @@ -307,20 +379,20 @@ void GDScriptEditorTranslationParserPlugin::_assess_call(const GDScriptParser::C } } if (extract_id_ctx_plural) { - ids_ctx_plural->push_back(id_ctx_plural); + _add_id_ctx_plural(id_ctx_plural, p_call->start_line); } } else if (first_arg_patterns.has(function_name)) { if (!p_call->arguments.is_empty() && _is_constant_string(p_call->arguments[0])) { - ids->push_back(p_call->arguments[0]->reduced_value); + _add_id(p_call->arguments[0]->reduced_value, p_call->arguments[0]->start_line); } } else if (second_arg_patterns.has(function_name)) { if (p_call->arguments.size() > 1 && _is_constant_string(p_call->arguments[1])) { - ids->push_back(p_call->arguments[1]->reduced_value); + _add_id(p_call->arguments[1]->reduced_value, p_call->arguments[1]->start_line); } } else if (function_name == fd_add_filter) { // Extract the 'JPE Images' in this example - get_node("FileDialog").add_filter("*.jpg; JPE Images"). if (!p_call->arguments.is_empty()) { - _extract_fd_filter_string(p_call->arguments[0]); + _extract_fd_filter_string(p_call->arguments[0], p_call->arguments[0]->start_line); } } else if (function_name == fd_set_filter) { // Extract from `get_node("FileDialog").set_filters()`. @@ -330,12 +402,12 @@ void GDScriptEditorTranslationParserPlugin::_assess_call(const GDScriptParser::C } } -void GDScriptEditorTranslationParserPlugin::_extract_fd_filter_string(const GDScriptParser::ExpressionNode *p_expression) { +void GDScriptEditorTranslationParserPlugin::_extract_fd_filter_string(const GDScriptParser::ExpressionNode *p_expression, int p_line) { // Extract the name in "extension ; name". if (_is_constant_string(p_expression)) { PackedStringArray arr = p_expression->reduced_value.operator String().split(";", true); ERR_FAIL_COND_MSG(arr.size() != 2, "Argument for setting FileDialog has bad format."); - ids->push_back(arr[1].strip_edges()); + _add_id(arr[1].strip_edges(), p_line); } } @@ -355,7 +427,7 @@ void GDScriptEditorTranslationParserPlugin::_extract_fd_filter_array(const GDScr if (array_node) { for (int i = 0; i < array_node->elements.size(); i++) { - _extract_fd_filter_string(array_node->elements[i]); + _extract_fd_filter_string(array_node->elements[i], array_node->elements[i]->start_line); } } } diff --git a/modules/gdscript/editor/gdscript_translation_parser_plugin.h b/modules/gdscript/editor/gdscript_translation_parser_plugin.h index 61ff81ed661b..73e8f5311063 100644 --- a/modules/gdscript/editor/gdscript_translation_parser_plugin.h +++ b/modules/gdscript/editor/gdscript_translation_parser_plugin.h @@ -32,16 +32,23 @@ #define GDSCRIPT_TRANSLATION_PARSER_PLUGIN_H #include "../gdscript_parser.h" +#include "../gdscript_tokenizer.h" +#include "core/templates/hash_map.h" #include "core/templates/hash_set.h" #include "editor/editor_translation_parser.h" class GDScriptEditorTranslationParserPlugin : public EditorTranslationParserPlugin { GDCLASS(GDScriptEditorTranslationParserPlugin, EditorTranslationParserPlugin); + const HashMap *comment_data = nullptr; + Vector *ids = nullptr; Vector> *ids_ctx_plural = nullptr; + Vector ids_comment; + Vector ids_ctx_plural_comment; + // List of patterns used for extracting translation strings. StringName tr_func = "tr"; StringName trn_func = "tr_n"; @@ -57,6 +64,11 @@ class GDScriptEditorTranslationParserPlugin : public EditorTranslationParserPlug static bool _is_constant_string(const GDScriptParser::ExpressionNode *p_expression); + String _parse_comment(int p_line, bool &r_skip) const; + + void _add_id(const String &p_id, int p_line); + void _add_id_ctx_plural(const Vector &p_id_ctx_plural, int p_line); + void _traverse_class(const GDScriptParser::ClassNode *p_class); void _traverse_function(const GDScriptParser::FunctionNode *p_func); void _traverse_block(const GDScriptParser::SuiteNode *p_suite); @@ -65,11 +77,12 @@ class GDScriptEditorTranslationParserPlugin : public EditorTranslationParserPlug void _assess_assignment(const GDScriptParser::AssignmentNode *p_assignment); void _assess_call(const GDScriptParser::CallNode *p_call); - void _extract_fd_filter_string(const GDScriptParser::ExpressionNode *p_expression); + void _extract_fd_filter_string(const GDScriptParser::ExpressionNode *p_expression, int p_line); void _extract_fd_filter_array(const GDScriptParser::ExpressionNode *p_expression); public: virtual Error parse_file(const String &p_path, Vector *r_ids, Vector> *r_ids_ctx_plural) override; + virtual void get_comments(Vector *r_ids_comment, Vector *r_ids_ctx_plural_comment) override; virtual void get_recognized_extensions(List *r_extensions) const override; GDScriptEditorTranslationParserPlugin(); diff --git a/modules/gdscript/gdscript_parser.cpp b/modules/gdscript/gdscript_parser.cpp index 111a39d73029..6a40ac5f30f4 100644 --- a/modules/gdscript/gdscript_parser.cpp +++ b/modules/gdscript/gdscript_parser.cpp @@ -409,6 +409,10 @@ Error GDScriptParser::parse(const String &p_source_code, const String &p_script_ parse_program(); pop_multiline(); +#ifdef TOOLS_ENABLED + comment_data = tokenizer->get_comments(); +#endif + memdelete(text_tokenizer); tokenizer = nullptr; diff --git a/modules/gdscript/gdscript_parser.h b/modules/gdscript/gdscript_parser.h index 7f64ae902b03..8e54a55b0403 100644 --- a/modules/gdscript/gdscript_parser.h +++ b/modules/gdscript/gdscript_parser.h @@ -1597,6 +1597,8 @@ class GDScriptParser { #ifdef TOOLS_ENABLED static HashMap theme_color_names; + + HashMap comment_data; #endif // TOOLS_ENABLED GDScriptParser();