diff --git a/core/io/resource.cpp b/core/io/resource.cpp index 5f8a4b85a4ec..a1f33c00e73e 100644 --- a/core/io/resource.cpp +++ b/core/io/resource.cpp @@ -528,6 +528,10 @@ String Resource::get_id_for_path(const String &p_path) const { #endif } +void Resource::_start_load(const StringName &p_res_format_type, int p_res_format_version) {} + +void Resource::_finish_load(const StringName &p_res_format_type, int p_res_format_version) {} + void Resource::_bind_methods() { ClassDB::bind_method(D_METHOD("set_path", "path"), &Resource::_set_path); ClassDB::bind_method(D_METHOD("take_over_path", "path"), &Resource::_take_over_path); diff --git a/core/io/resource.h b/core/io/resource.h index 8966c0233cac..1e7ec7e37553 100644 --- a/core/io/resource.h +++ b/core/io/resource.h @@ -151,6 +151,9 @@ class Resource : public RefCounted { void set_id_for_path(const String &p_path, const String &p_id); String get_id_for_path(const String &p_path) const; + virtual void _start_load(const StringName &p_res_format_type, int p_res_format_version); + virtual void _finish_load(const StringName &p_res_format_type, int p_res_format_version); + Resource(); ~Resource(); }; diff --git a/core/io/resource_format_binary.cpp b/core/io/resource_format_binary.cpp index b4826c356e01..a251de44b5e2 100644 --- a/core/io/resource_format_binary.cpp +++ b/core/io/resource_format_binary.cpp @@ -811,6 +811,8 @@ Error ResourceLoaderBinary::load() { internal_index_cache[path] = res; } + res->_start_load("binary", ver_format); + int pc = f->get_32(); //set properties @@ -889,6 +891,8 @@ Error ResourceLoaderBinary::load() { res->set_edited(false); #endif + res->_finish_load("binary", ver_format); + if (progress) { *progress = (i + 1) / float(internal_resources.size()); } diff --git a/editor/import/3d/resource_importer_scene.cpp b/editor/import/3d/resource_importer_scene.cpp index cb348f713c12..46b9ee6c2c9f 100644 --- a/editor/import/3d/resource_importer_scene.cpp +++ b/editor/import/3d/resource_importer_scene.cpp @@ -3270,11 +3270,43 @@ void EditorSceneFormatImporterESCN::get_extensions(List *r_extensions) c r_extensions->push_back("escn"); } +int get_text_format_version(String p_path) { + Error error; + Ref f = FileAccess::open(p_path, FileAccess::READ, &error); + ERR_FAIL_COND_V_MSG(error != OK || f.is_null(), -1, "Cannot open file '" + p_path + "'."); + String line = f->get_line().strip_edges(); + // skip empty lines and comments + while (line.is_empty() || line.begins_with(";")) { + line = f->get_line().strip_edges(); + if (f->eof_reached()) { + break; + } + } + int format_index = line.find("format"); + ERR_FAIL_COND_V_MSG(format_index == -1, -1, "No format specifier in file '" + p_path + "'."); + String format_str = line.substr(format_index).get_slicec('=', 1).strip_edges(); + ERR_FAIL_COND_V_MSG(!format_str.substr(0, 1).is_numeric(), -1, "Invalid format in file '" + p_path + "'."); + int format = format_str.to_int(); + return format; +} + Node *EditorSceneFormatImporterESCN::import_scene(const String &p_path, uint32_t p_flags, const HashMap &p_options, List *r_missing_deps, Error *r_err) { Error error; + int format = get_text_format_version(p_path); + ERR_FAIL_COND_V(format == -1, nullptr); Ref ps = ResourceFormatLoaderText::singleton->load(p_path, p_path, &error); ERR_FAIL_COND_V_MSG(!ps.is_valid(), nullptr, "Cannot load scene as text resource from path '" + p_path + "'."); Node *scene = ps->instantiate(); + ERR_FAIL_COND_V(!scene, nullptr); + if (format == 2) { + TypedArray skel_nodes = scene->find_children("*", "AnimationPlayer"); + for (int32_t node_i = 0; node_i < skel_nodes.size(); node_i++) { + // Force re-compute animation tracks. + AnimationPlayer *player = cast_to(skel_nodes[node_i]); + ERR_CONTINUE(!player); + player->advance(0); + } + } TypedArray nodes = scene->find_children("*", "MeshInstance3D"); for (int32_t node_i = 0; node_i < nodes.size(); node_i++) { MeshInstance3D *mesh_3d = cast_to(nodes[node_i]); @@ -3283,9 +3315,22 @@ Node *EditorSceneFormatImporterESCN::import_scene(const String &p_path, uint32_t // Ignore the aabb, it will be recomputed. ImporterMeshInstance3D *importer_mesh_3d = memnew(ImporterMeshInstance3D); importer_mesh_3d->set_name(mesh_3d->get_name()); - importer_mesh_3d->set_transform(mesh_3d->get_relative_transform(mesh_3d->get_parent())); - importer_mesh_3d->set_skin(mesh_3d->get_skin()); + Node *parent = mesh_3d->get_parent(); + Transform3D rel_transform = mesh_3d->get_relative_transform(parent); + if (rel_transform == Transform3D() && parent && parent != mesh_3d) { + // If we're here, we probably got a "data.parent is null" error + // Node3D.data.parent hasn't been set yet but Node.data.parent has, so we need to get the transform manually + Node3D *parent_3d = mesh_3d->get_parent_node_3d(); + if (parent == parent_3d) { + rel_transform = mesh_3d->get_transform(); + } else if (parent_3d) { + rel_transform = parent_3d->get_relative_transform(parent) * mesh_3d->get_transform(); + } // Otherwise, parent isn't a Node3D. + } + importer_mesh_3d->set_transform(rel_transform); + Ref skin = mesh_3d->get_skin(); importer_mesh_3d->set_skeleton_path(mesh_3d->get_skeleton_path()); + importer_mesh_3d->set_skin(skin); Ref array_mesh_3d_mesh = mesh_3d->get_mesh(); if (array_mesh_3d_mesh.is_valid()) { // For the MeshInstance3D nodes, we need to convert the ArrayMesh to an ImporterMesh specially. @@ -3326,7 +3371,5 @@ Node *EditorSceneFormatImporterESCN::import_scene(const String &p_path, uint32_t } } - ERR_FAIL_NULL_V(scene, nullptr); - return scene; } diff --git a/editor/plugins/text_shader_editor.cpp b/editor/plugins/text_shader_editor.cpp index 0ff7aaa3fe46..ac7bbc3480a5 100644 --- a/editor/plugins/text_shader_editor.cpp +++ b/editor/plugins/text_shader_editor.cpp @@ -42,6 +42,10 @@ #include "servers/rendering/shader_preprocessor.h" #include "servers/rendering/shader_types.h" +#ifndef DISABLE_DEPRECATED +#include "servers/rendering/shader_converter.h" +#endif + /*** SHADER SYNTAX HIGHLIGHTER ****/ Dictionary GDShaderSyntaxHighlighter::_get_line_syntax_highlighting_impl(int p_line) { @@ -699,6 +703,28 @@ void TextShaderEditor::_menu_option(int p_option) { case EDIT_COMPLETE: { code_editor->get_text_editor()->request_code_completion(); } break; +#ifndef DISABLE_DEPRECATED + case EDIT_CONVERT: { + if (shader.is_null()) { + return; + } + String code = code_editor->get_text_editor()->get_text(); + if (code.is_empty()) { + return; + } + ShaderDeprecatedConverter converter; + if (!converter.is_code_deprecated(code)) { + if (converter.get_error_text() != String()) { + shader_convert_error_dialog->set_text(vformat(RTR("Line %d: %s"), converter.get_error_line(), converter.get_error_text())); + shader_convert_error_dialog->popup_centered(); + ERR_PRINT("Shader conversion failed: " + converter.get_error_text()); + } + confirm_convert_shader->popup_centered(); + return; + } + _convert_shader(); + } break; +#endif case SEARCH_FIND: { code_editor->get_find_replace_bar()->popup_search(); } break; @@ -755,6 +781,43 @@ void TextShaderEditor::_notification(int p_what) { } break; } } +#ifndef DISABLE_DEPRECATED +void TextShaderEditor::_convert_shader() { + if (shader.is_null()) { + return; + } + String code = code_editor->get_text_editor()->get_text(); + if (code.is_empty()) { + return; + } + ShaderDeprecatedConverter converter; + if (!converter.convert_code(code)) { + String err_text = converter.get_error_text(); + if (err_text.is_empty()) { + err_text = TTR("Unknown error occurred while converting the shader."); + } else if (converter.get_error_line() > 0) { + err_text = vformat("%s (line %d)", err_text, converter.get_error_line()); + } + + shader_convert_error_dialog->set_text(err_text); + shader_convert_error_dialog->popup_centered(); + ERR_PRINT("Shader conversion failed: " + err_text); + return; + } + String new_code = converter.emit_code(); + +#ifdef DEBUG_ENABLED + print_line(converter.get_report()); +#endif + if (new_code == code) { + return; + } + // Ensure undoable. + code_editor->get_text_editor()->set_text(new_code); + code_editor->get_text_editor()->tag_saved_version(); + code_editor->_validate_script(); +} +#endif void TextShaderEditor::_editor_settings_changed() { if (!EditorThemeManager::is_generated_theme_outdated() && @@ -1170,8 +1233,12 @@ TextShaderEditor::TextShaderEditor() { edit_menu->get_popup()->add_shortcut(ED_GET_SHORTCUT("script_text_editor/toggle_word_wrap"), EDIT_TOGGLE_WORD_WRAP); edit_menu->get_popup()->add_separator(); edit_menu->get_popup()->add_shortcut(ED_GET_SHORTCUT("ui_text_completion_query"), EDIT_COMPLETE); - edit_menu->get_popup()->connect(SceneStringName(id_pressed), callable_mp(this, &TextShaderEditor::_menu_option)); +#ifndef DISABLE_DEPRECATED + edit_menu->get_popup()->add_separator(); + edit_menu->get_popup()->add_item(TTR("Convert 3.x Shader"), EDIT_CONVERT); +#endif + edit_menu->get_popup()->connect(SceneStringName(id_pressed), callable_mp(this, &TextShaderEditor::_menu_option)); search_menu = memnew(MenuButton); search_menu->set_shortcut_context(this); search_menu->set_text(TTR("Search")); @@ -1254,7 +1321,20 @@ TextShaderEditor::TextShaderEditor() { disk_changed->connect("custom_action", callable_mp(this, &TextShaderEditor::save_external_data)); add_child(disk_changed); - +#ifndef DISABLE_DEPRECATED + shader_convert_error_dialog = memnew(AcceptDialog); + shader_convert_error_dialog->set_title(TTR("Error converting shader")); + shader_convert_error_dialog->set_hide_on_ok(true); + add_child(shader_convert_error_dialog); + + confirm_convert_shader = memnew(ConfirmationDialog); + confirm_convert_shader->set_title(TTR("Confirm Convert 3.x Shader")); + confirm_convert_shader->set_text(TTR("This shader does not appear to be a 3.x shader.\nAre you sure you want to convert it?")); + confirm_convert_shader->get_ok_button()->set_text(TTR("Convert")); + confirm_convert_shader->get_cancel_button()->set_text(TTR("Cancel")); + confirm_convert_shader->connect("confirmed", callable_mp(this, &TextShaderEditor::_convert_shader)); + add_child(confirm_convert_shader); +#endif _editor_settings_changed(); code_editor->show_toggle_scripts_button(); // TODO: Disabled for now, because it doesn't work properly. } diff --git a/editor/plugins/text_shader_editor.h b/editor/plugins/text_shader_editor.h index 55efb4e30ed8..320e66c1fd32 100644 --- a/editor/plugins/text_shader_editor.h +++ b/editor/plugins/text_shader_editor.h @@ -125,6 +125,7 @@ class TextShaderEditor : public ShaderEditor { EDIT_TOGGLE_WORD_WRAP, EDIT_TOGGLE_COMMENT, EDIT_COMPLETE, + EDIT_CONVERT, SEARCH_FIND, SEARCH_FIND_NEXT, SEARCH_FIND_PREV, @@ -150,6 +151,11 @@ class TextShaderEditor : public ShaderEditor { ConfirmationDialog *disk_changed = nullptr; ShaderTextEditor *code_editor = nullptr; +#ifndef DISABLE_DEPRECATED + AcceptDialog *shader_convert_error_dialog = nullptr; + ConfirmationDialog *confirm_convert_shader = nullptr; +#endif + bool compilation_success = true; void _menu_option(int p_option); @@ -157,6 +163,9 @@ class TextShaderEditor : public ShaderEditor { mutable Ref shader; mutable Ref shader_inc; +#ifndef DISABLE_DEPRECATED + void _convert_shader(); +#endif void _editor_settings_changed(); void _apply_editor_settings(); void _project_settings_changed(); diff --git a/scene/3d/mesh_instance_3d.cpp b/scene/3d/mesh_instance_3d.cpp index e8166802f8f2..c96625ecb8f9 100644 --- a/scene/3d/mesh_instance_3d.cpp +++ b/scene/3d/mesh_instance_3d.cpp @@ -50,8 +50,19 @@ bool MeshInstance3D::_set(const StringName &p_name, const Variant &p_value) { return true; } - if (p_name.operator String().begins_with("surface_material_override/")) { - int idx = p_name.operator String().get_slicec('/', 1).to_int(); + String name = p_name; + +#ifndef DISABLE_DEPRECATED + if (name.begins_with("material/")) { + WARN_DEPRECATED_MSG("This mesh uses an old deprecated parameter name. Consider re-saving this scene in order for it to continue working in future versions." + + (is_inside_tree() ? vformat(" Path: \"%s\"", get_path()) : String())); + } + if (name.begins_with("surface_material_override/") || name.begins_with("material/")) +#else + if (name.begins_with("surface_material_override/")) +#endif + { + int idx = name.get_slicec('/', 1).to_int(); if (idx >= surface_override_materials.size() || idx < 0) { return false; diff --git a/scene/animation/animation_mixer.cpp b/scene/animation/animation_mixer.cpp index eb8bc8c38245..282bf43e83f8 100644 --- a/scene/animation/animation_mixer.cpp +++ b/scene/animation/animation_mixer.cpp @@ -242,6 +242,97 @@ TypedArray AnimationMixer::_get_animation_library_list() const { return ret; } +#if !defined(_3D_DISABLED) || !defined(DISABLE_DEPRECATED) +bool AnimationMixer::_recalc_animation(Ref &anim) { + HashMap> new_track_values_map; + Node *parent = get_node_or_null(root_node); + if (!parent) { + return false; + } + + for (int i = 0; i < anim->get_track_count(); i++) { + int track_type = anim->track_get_type(i); + if (track_type == Animation::TYPE_POSITION_3D || track_type == Animation::TYPE_ROTATION_3D || track_type == Animation::TYPE_SCALE_3D) { + NodePath path = anim->track_get_path(i); + Node *node = parent->get_node(path); + ERR_FAIL_COND_V(!node, false); + Skeleton3D *skel = Object::cast_to(node); + ERR_FAIL_COND_V(!skel, false); + + StringName bone = path.get_subname(0); + int bone_idx = skel->find_bone(bone); + if (bone_idx == -1) { + continue; + } + Transform3D rest = skel->get_bone_rest(bone_idx); + new_track_values_map[i] = Vector(); + const int32_t POSITION_TRACK_SIZE = 5; + const int32_t ROTATION_TRACK_SIZE = 6; + const int32_t SCALE_TRACK_SIZE = 5; + int32_t track_size = POSITION_TRACK_SIZE; + if (track_type == Animation::TYPE_ROTATION_3D) { + track_size = ROTATION_TRACK_SIZE; + } + new_track_values_map[i].resize(track_size * anim->track_get_key_count(i)); + real_t *r = new_track_values_map[i].ptrw(); + for (int j = 0; j < anim->track_get_key_count(i); j++) { + real_t time = anim->track_get_key_time(i, j); + real_t transition = anim->track_get_key_transition(i, j); + if (track_type == Animation::TYPE_POSITION_3D) { + Vector3 a_pos = anim->track_get_key_value(i, j); + Transform3D t = Transform3D(); + t.set_origin(a_pos); + Vector3 new_a_pos = (rest * t).origin; + + real_t *ofs = &r[j * POSITION_TRACK_SIZE]; + ofs[0] = time; + ofs[1] = transition; + ofs[2] = new_a_pos.x; + ofs[3] = new_a_pos.y; + ofs[4] = new_a_pos.z; + } else if (track_type == Animation::TYPE_ROTATION_3D) { + Quaternion q = anim->track_get_key_value(i, j); + Transform3D t = Transform3D(); + t.basis.rotate(q); + Quaternion new_q = (rest * t).basis.get_rotation_quaternion(); + real_t *ofs = &r[j * ROTATION_TRACK_SIZE]; + ofs[0] = time; + ofs[1] = transition; + ofs[2] = new_q.x; + ofs[3] = new_q.y; + ofs[4] = new_q.z; + ofs[5] = new_q.w; + } else if (track_type == Animation::TYPE_SCALE_3D) { + Vector3 v = anim->track_get_key_value(i, j); + Transform3D t = Transform3D(); + t.scale(v); + Vector3 new_v = (rest * t).basis.get_scale(); + + real_t *ofs = &r[j * SCALE_TRACK_SIZE]; + ofs[0] = time; + ofs[1] = transition; + ofs[2] = new_v.x; + ofs[3] = new_v.y; + ofs[4] = new_v.z; + } + } + } + } + if (new_track_values_map.is_empty()) { + return false; + } + for (int i = 0; i < anim->get_track_count(); i++) { + if (!new_track_values_map.has(i)) { + continue; + } + anim->set("tracks/" + itos(i) + "/keys", new_track_values_map[i]); + anim->set("tracks/" + itos(i) + "/relative_to_rest", false); + } + anim->emit_changed(); + return true; +} +#endif // !defined(_3D_DISABLED) || !defined(DISABLE_DEPRECATED) + void AnimationMixer::get_animation_library_list(List *p_libraries) const { for (const AnimationLibraryData &lib : animation_libraries) { p_libraries->push_back(lib.name); @@ -986,6 +1077,18 @@ void AnimationMixer::_blend_init() { root_motion_scale_accumulator = Vector3(1, 1, 1); if (!cache_valid) { +#if !defined(_3D_DISABLED) || !defined(DISABLE_DEPRECATED) + List sname; + get_animation_list(&sname); + + for (const StringName &E : sname) { + Ref anim = get_animation(E); + + if (anim->has_tracks_relative_to_rest()) { + _recalc_animation(anim); + } + } +#endif if (!_update_caches()) { return; } diff --git a/scene/animation/animation_mixer.h b/scene/animation/animation_mixer.h index 27c9a00a9c5b..550a045f782e 100644 --- a/scene/animation/animation_mixer.h +++ b/scene/animation/animation_mixer.h @@ -319,6 +319,10 @@ class AnimationMixer : public Node { void _init_root_motion_cache(); bool _update_caches(); +#if !defined(_3D_DISABLED) || !defined(DISABLE_DEPRECATED) + bool _recalc_animation(Ref &p_anim); +#endif + /* ---- Audio ---- */ AudioServer::PlaybackType playback_type; diff --git a/scene/resources/animation.cpp b/scene/resources/animation.cpp index 57a4e35f7a7f..2c1601f37cd6 100644 --- a/scene/resources/animation.cpp +++ b/scene/resources/animation.cpp @@ -34,6 +34,10 @@ #include "core/io/marshalls.h" #include "core/math/geometry_3d.h" +#define _LOAD_META_PROPERTY "_load" +#define _TRANSFORM_TRACK_LIST_META_PROPERTY "_transform_track_list" +#define _TRANSFORM_TRACK_DATA_META_PROPERTY(m_track_idx) "_transform_track_data__" + String::num(m_track_idx) + bool Animation::_set(const StringName &p_name, const Variant &p_value) { String prop_name = p_name; @@ -106,6 +110,23 @@ bool Animation::_set(const StringName &p_name, const Variant &p_value) { } else if (type == "animation") { add_track(TYPE_ANIMATION); } else { +#ifndef DISABLE_DEPRECATED + // for compatibility with 3.x animations + if (get_meta(_LOAD_META_PROPERTY, false)) { // Only do compatibility conversions if we are loading a resource. + if (type == "transform") { + WARN_DEPRECATED_MSG("Animation uses old 'transform' track types, which is deprecated (and loads slower). Consider re-importing or re-saving the resource."); + PackedInt32Array track_list = get_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY, PackedInt32Array()); + track_list.push_back(track); + set_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY, track_list); + Dictionary track_data; + track_data["type"] = "transform"; + set_meta(_TRANSFORM_TRACK_DATA_META_PROPERTY(track), track_data); + // Add an empty track that will get replaced after we are finished loading, so we don't throw off the track indices. + add_track(TYPE_ANIMATION, track); + return true; + } + } +#endif // DISABLE_DEPRECATED return false; } @@ -114,6 +135,42 @@ bool Animation::_set(const StringName &p_name, const Variant &p_value) { ERR_FAIL_INDEX_V(track, tracks.size(), false); +#ifndef DISABLE_DEPRECATED + if (what == "relative_to_rest") { + Track *t = tracks[track]; + switch (t->type) { + case TYPE_POSITION_3D: { + PositionTrack *tt = static_cast(t); + tt->relative_to_rest = p_value; + } break; + case TYPE_ROTATION_3D: { + RotationTrack *rt = static_cast(t); + rt->relative_to_rest = p_value; + } break; + case TYPE_SCALE_3D: { + ScaleTrack *st = static_cast(t); + st->relative_to_rest = p_value; + } break; + default: { + return false; + } + } + return true; + } + // If we have a transform track, we need to store the data in the metadata to be able to convert it to the new format after the load is finished. + if (get_meta(_LOAD_META_PROPERTY, false)) { // Only do this on resource loads, not on editor changes + // check the metadata to see if this track is a transform track that we are holding on to + PackedInt32Array transform_tracks = get_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY, PackedInt32Array()); + if (transform_tracks.has(track)) { + // get the track data + Dictionary track_data = get_meta(_TRANSFORM_TRACK_DATA_META_PROPERTY(track), Dictionary()); + track_data[what] = p_value; + set_meta(_TRANSFORM_TRACK_DATA_META_PROPERTY(track), track_data); + return true; + } + } +#endif // DISABLE_DEPRECATED + if (what == "path") { track_set_path(track, p_value); } else if (what == "compressed_track") { @@ -880,6 +937,11 @@ void Animation::_get_property_list(List *p_list) const { p_list->push_back(PropertyInfo(Variant::INT, "tracks/" + itos(i) + "/interp", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL)); p_list->push_back(PropertyInfo(Variant::BOOL, "tracks/" + itos(i) + "/loop_wrap", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL)); p_list->push_back(PropertyInfo(Variant::ARRAY, "tracks/" + itos(i) + "/keys", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL)); +#ifndef DISABLE_DEPRECATED + if (track_is_relative_to_rest(i)) { + p_list->push_back(PropertyInfo(Variant::ARRAY, "tracks/" + itos(i) + "/relative_to_rest", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL)); + } +#endif // DISABLE_DEPRECATED } if (track_get_type(i) == TYPE_AUDIO) { p_list->push_back(PropertyInfo(Variant::BOOL, "tracks/" + itos(i) + "/use_blend", PROPERTY_HINT_NONE, "", PROPERTY_USAGE_NO_EDITOR | PROPERTY_USAGE_INTERNAL)); @@ -1144,6 +1206,63 @@ void Animation::_clear(T &p_keys) { } //// +#ifndef DISABLE_DEPRECATED +bool Animation::has_tracks_relative_to_rest() const { + for (int i = 0; i < tracks.size(); i++) { + if (track_is_relative_to_rest(i)) { + return true; + } + } + return false; +} + +bool Animation::track_is_relative_to_rest(int p_track) const { + ERR_FAIL_INDEX_V(p_track, tracks.size(), false); + Track *t = tracks[p_track]; + + switch (t->type) { + case TYPE_POSITION_3D: { + PositionTrack *tt = static_cast(t); + return tt->relative_to_rest; + } break; + case TYPE_ROTATION_3D: { + RotationTrack *rt = static_cast(t); + return rt->relative_to_rest; + } break; + case TYPE_SCALE_3D: { + ScaleTrack *st = static_cast(t); + return st->relative_to_rest; + } break; + default: { + return false; // Animation does not really use transitions. + } break; + } +} + +void Animation::track_set_relative_to_rest(int p_track, bool p_relative_to_rest) { + ERR_FAIL_INDEX(p_track, tracks.size()); + Track *t = tracks[p_track]; + + switch (t->type) { + case TYPE_POSITION_3D: { + PositionTrack *tt = static_cast(t); + tt->relative_to_rest = p_relative_to_rest; + } break; + case TYPE_ROTATION_3D: { + RotationTrack *rt = static_cast(t); + rt->relative_to_rest = p_relative_to_rest; + } break; + case TYPE_SCALE_3D: { + ScaleTrack *st = static_cast(t); + st->relative_to_rest = p_relative_to_rest; + } break; + default: { + return; // Animation does not really use transitions. + } break; + } + emit_changed(); +} +#endif // DISABLE_DEPRECATED int Animation::position_track_insert_key(int p_track, double p_time, const Vector3 &p_position) { ERR_FAIL_INDEX_V(p_track, tracks.size(), -1); @@ -5216,6 +5335,116 @@ void Animation::compress(uint32_t p_page_size, uint32_t p_fps, float p_split_tol #endif } +void Animation::_start_load(const StringName &p_res_format_type, int p_res_format_version) { +#ifndef DISABLE_DEPRECATED + set_meta(_LOAD_META_PROPERTY, true); +#endif +} + +void Animation::_finish_load(const StringName &p_res_format_type, int p_res_format_version) { +#ifndef DISABLE_DEPRECATED // 3.x compatibility, convert transform tracks to separate tracks + if (!has_meta(_LOAD_META_PROPERTY)) { + return; + } + set_meta(_LOAD_META_PROPERTY, Variant()); + if (!has_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY)) { + return; + } + PackedInt32Array transform_track_list = get_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY, PackedInt32Array()); + set_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY, Variant()); + if (transform_track_list.is_empty()) { + return; + } + int offset = 0; + for (int track_idx : transform_track_list) { + // now that we have all the tracks, we need to split the transform tracks into separate tracks + // this is because the current animation player doesn't support transform tracks + // so we need to split them into separate position, rotation, and scale tracks + // No need to worry about compression here; this was added in 4.x and wasn't backported to 3.x. + Dictionary track_data = get_meta(_TRANSFORM_TRACK_DATA_META_PROPERTY(track_idx), Dictionary()); + if (track_data.is_empty()) { + continue; + } + // split the transform track into separate tracks + + // Old scene format only used 32-bit floats, did not have configurable real_t. + Vector keys = track_data["keys"]; + int vcount = keys.size(); + int tcount = vcount / 12; + ERR_CONTINUE_MSG((vcount % 12) != 0, "Failed to convert transform track: invalid number of key frames."); + + Vector position_keys; + Vector rotation_keys; + Vector scale_keys; + position_keys.resize(tcount * 5); // time + transition + xyz + rotation_keys.resize(tcount * 6); // time + transition + xyzw + scale_keys.resize(tcount * 5); // time + transition + xyz + // split the keys into separate tracks + for (int j = 0; j < tcount; j++) { + // it's position (Vector3, xyz), then rotation (Quaternion, xyzw), then scale (Vector3, xyz) + // each track has time and transition values, so get those + const float *ofs = &(keys.ptr()[j * 12]); + float time = ofs[0]; + float transition = ofs[1]; + + position_keys.write[j * 5 + 0] = time; + position_keys.write[j * 5 + 1] = transition; + position_keys.write[j * 5 + 2] = ofs[2]; // x + position_keys.write[j * 5 + 3] = ofs[3]; // y + position_keys.write[j * 5 + 4] = ofs[4]; // z + + rotation_keys.write[j * 6 + 0] = time; + rotation_keys.write[j * 6 + 1] = transition; + rotation_keys.write[j * 6 + 2] = ofs[5]; // x + rotation_keys.write[j * 6 + 3] = ofs[6]; // y + rotation_keys.write[j * 6 + 4] = ofs[7]; // z + rotation_keys.write[j * 6 + 5] = ofs[8]; // w + + scale_keys.write[j * 5 + 0] = time; + scale_keys.write[j * 5 + 1] = transition; + scale_keys.write[j * 5 + 2] = ofs[9]; // x + scale_keys.write[j * 5 + 3] = ofs[10]; // y + scale_keys.write[j * 5 + 4] = ofs[11]; // z + } + Array c_track_keys = track_data.keys(); + c_track_keys.erase("type"); + c_track_keys.erase("keys"); + remove_track(track_idx + offset); // remove dummy track + + add_track(TYPE_POSITION_3D, track_idx + offset); + for (int j = 0; j < c_track_keys.size(); j++) { + String key = c_track_keys[j]; + _set("tracks/" + itos(track_idx + offset) + "/" + key, track_data[key]); + } + _set("tracks/" + itos(track_idx + offset) + "/keys", position_keys); + _set("tracks/" + itos(track_idx + offset) + "/relative_to_rest", true); + offset++; + + add_track(TYPE_ROTATION_3D, track_idx + offset); + for (int j = 0; j < c_track_keys.size(); j++) { + String key = c_track_keys[j]; + _set("tracks/" + itos(track_idx + offset) + "/" + key, track_data[key]); + } + _set("tracks/" + itos(track_idx + offset) + "/keys", rotation_keys); + _set("tracks/" + itos(track_idx + offset) + "/relative_to_rest", true); + offset++; + add_track(TYPE_SCALE_3D, track_idx + offset); + for (int j = 0; j < c_track_keys.size(); j++) { + String key = c_track_keys[j]; + _set("tracks/" + itos(track_idx + offset) + "/" + key, track_data[key]); + } + _set("tracks/" + itos(track_idx + offset) + "/keys", scale_keys); + _set("tracks/" + itos(track_idx + offset) + "/relative_to_rest", true); + offset++; + offset--; // subtract 1 because we removed the dummy track + // erase the track data + set_meta(_TRANSFORM_TRACK_DATA_META_PROPERTY(track_idx), Variant()); + } + // erase the track list + set_meta(_TRANSFORM_TRACK_LIST_META_PROPERTY, Variant()); +#endif // DISABLE_DEPRECATED +} + bool Animation::_rotation_interpolate_compressed(uint32_t p_compressed_track, double p_time, Quaternion &r_ret) const { Vector3i current; Vector3i next; diff --git a/scene/resources/animation.h b/scene/resources/animation.h index 618dc9ca17cd..de3c6b15fa07 100644 --- a/scene/resources/animation.h +++ b/scene/resources/animation.h @@ -138,6 +138,9 @@ class Animation : public Resource { struct PositionTrack : public Track { Vector> positions; int32_t compressed_track = -1; +#ifndef DISABLE_DEPRECATED + bool relative_to_rest = false; +#endif PositionTrack() { type = TYPE_POSITION_3D; } }; @@ -146,6 +149,9 @@ class Animation : public Resource { struct RotationTrack : public Track { Vector> rotations; int32_t compressed_track = -1; +#ifndef DISABLE_DEPRECATED + bool relative_to_rest = false; +#endif RotationTrack() { type = TYPE_ROTATION_3D; } }; @@ -154,6 +160,9 @@ class Animation : public Resource { struct ScaleTrack : public Track { Vector> scales; int32_t compressed_track = -1; +#ifndef DISABLE_DEPRECATED + bool relative_to_rest = false; +#endif ScaleTrack() { type = TYPE_SCALE_3D; } }; @@ -450,7 +459,11 @@ class Animation : public Resource { double track_get_key_time(int p_track, int p_key_idx) const; real_t track_get_key_transition(int p_track, int p_key_idx) const; bool track_is_compressed(int p_track) const; - +#ifndef DISABLE_DEPRECATED + bool has_tracks_relative_to_rest() const; + bool track_is_relative_to_rest(int p_track) const; + void track_set_relative_to_rest(int p_track, bool p_relative_to_rest); +#endif int position_track_insert_key(int p_track, double p_time, const Vector3 &p_position); Error position_track_get_key(int p_track, int p_key, Vector3 *r_position) const; Error try_position_track_interpolate(int p_track, double p_time, Vector3 *r_interpolation, bool p_backward = false) const; @@ -542,6 +555,9 @@ class Animation : public Resource { void optimize(real_t p_allowed_velocity_err = 0.01, real_t p_allowed_angular_err = 0.01, int p_precision = 3); void compress(uint32_t p_page_size = 8192, uint32_t p_fps = 120, float p_split_tolerance = 4.0); // 4.0 seems to be the split tolerance sweet spot from many tests. + virtual void _start_load(const StringName &p_res_format_type, int p_res_format_version) override; + virtual void _finish_load(const StringName &p_res_format_type, int p_res_format_version) override; + // Helper functions for Variant. static bool is_variant_interpolatable(const Variant p_value); diff --git a/scene/resources/resource_format_text.cpp b/scene/resources/resource_format_text.cpp index d531eea3112b..c421d97c2196 100644 --- a/scene/resources/resource_format_text.cpp +++ b/scene/resources/resource_format_text.cpp @@ -190,6 +190,7 @@ Ref ResourceLoaderText::_parse_node_tag(VariantParser::ResourcePars if (packed_scene.is_null()) { packed_scene.instantiate(); } + packed_scene->_start_load("text", format_version); while (true) { if (next_tag.name == "node") { @@ -285,6 +286,7 @@ Ref ResourceLoaderText::_parse_node_tag(VariantParser::ResourcePars return Ref(); } else { error = OK; + packed_scene->_finish_load("text", format_version); return packed_scene; } } @@ -366,6 +368,7 @@ Ref ResourceLoaderText::_parse_node_tag(VariantParser::ResourcePars return Ref(); } else { error = OK; + packed_scene->_finish_load("text", format_version); return packed_scene; } } @@ -389,6 +392,7 @@ Ref ResourceLoaderText::_parse_node_tag(VariantParser::ResourcePars return Ref(); } else { error = OK; + packed_scene->_finish_load("text", format_version); return packed_scene; } } @@ -575,6 +579,7 @@ Error ResourceLoaderText::load() { int_resources[id] = res; // Always assign int resources. if (do_assign) { + res->_start_load("text", format_version); if (cache_mode != ResourceFormatLoader::CACHE_MODE_IGNORE) { res->set_path(path, cache_mode == ResourceFormatLoader::CACHE_MODE_REPLACE); } else { @@ -662,6 +667,10 @@ Error ResourceLoaderText::load() { if (!missing_resource_properties.is_empty()) { res->set_meta(META_MISSING_RESOURCES, missing_resource_properties); } + + if (do_assign) { + res->_finish_load("text", format_version); + } } while (true) { @@ -714,6 +723,8 @@ Error ResourceLoaderText::load() { } } + resource->_start_load("text", format_version); + Dictionary missing_resource_properties; while (true) { @@ -735,6 +746,7 @@ Error ResourceLoaderText::load() { } else { resource->set_path_cache(res_path); } + break; } return error; } @@ -808,6 +820,7 @@ Error ResourceLoaderText::load() { if (!missing_resource_properties.is_empty()) { resource->set_meta(META_MISSING_RESOURCES, missing_resource_properties); } + resource->_finish_load("text", format_version); error = OK; diff --git a/scene/resources/resource_format_text.h b/scene/resources/resource_format_text.h index 8397bc985fc5..48c1d779b8e5 100644 --- a/scene/resources/resource_format_text.h +++ b/scene/resources/resource_format_text.h @@ -65,9 +65,10 @@ class ResourceLoaderText { }; bool is_scene = false; - int format_version; String res_type; + int format_version = 0; + bool ignore_resource_parsing = false; HashMap ext_resources; diff --git a/scene/resources/shader.cpp b/scene/resources/shader.cpp index 24d17108d528..6179d349fe00 100644 --- a/scene/resources/shader.cpp +++ b/scene/resources/shader.cpp @@ -32,8 +32,11 @@ #include "shader.compat.inc" #include "core/io/file_access.h" +#include "scene/scene_string_names.h" +#include "servers/rendering/rendering_server_globals.h" #include "servers/rendering/shader_language.h" #include "servers/rendering/shader_preprocessor.h" +#include "servers/rendering/shader_types.h" #include "servers/rendering_server.h" #include "texture.h" @@ -46,6 +49,12 @@ #endif #endif +#ifndef DISABLE_DEPRECATED +#include "servers/rendering/shader_converter.h" +#endif + +#define _LOAD_COMPAT_META_PROPERTY "_load_compat" + Shader::Mode Shader::get_mode() const { return mode; } @@ -81,6 +90,13 @@ void Shader::set_include_path(const String &p_path) { include_path = p_path; } +#ifndef DISABLE_DEPRECATED +ShaderLanguage::DataType _get_global_shader_uniform_type(const StringName &p_name) { + RS::GlobalShaderParameterType gvt = RenderingServerGlobals::material_storage->global_shader_parameter_get_type(p_name); + return (ShaderLanguage::DataType)RS::global_shader_uniform_type_get_shader_datatype(gvt); +} +#endif + void Shader::set_code(const String &p_code) { for (const Ref &E : include_dependencies) { E->disconnect_changed(callable_mp(this, &Shader::_dependency_changed)); @@ -88,6 +104,43 @@ void Shader::set_code(const String &p_code) { code = p_code; preprocessed_code = p_code; +#ifndef DISABLE_DEPRECATED + if (get_meta(_LOAD_COMPAT_META_PROPERTY, false)) { + // check if the Shader code compiles; if not, it's probably an old shader. + + ShaderLanguage sl; + ShaderLanguage::ShaderCompileInfo info; + String mode_string = ShaderLanguage::get_shader_type(p_code); + + RS::ShaderMode new_mode; + if (mode_string == "canvas_item") { + new_mode = RS::SHADER_CANVAS_ITEM; + } else if (mode_string == "particles") { + new_mode = RS::SHADER_PARTICLES; + } else if (mode_string == "spatial") { + new_mode = RS::SHADER_SPATIAL; + } else { + new_mode = RS::SHADER_MAX; + } + if (new_mode != RS::SHADER_MAX) { + info.functions = ShaderTypes::get_singleton()->get_functions(new_mode); + info.render_modes = ShaderTypes::get_singleton()->get_modes(new_mode); + info.shader_types = ShaderTypes::get_singleton()->get_types(); + info.global_shader_uniform_type_func = _get_global_shader_uniform_type; + Error err = sl.compile(p_code, info); + if (err) { + ShaderDeprecatedConverter sdc; + if (sdc.is_code_deprecated(p_code)) { + ERR_FAIL_COND_MSG(!sdc.convert_code(p_code), vformat("Shader conversion failed (line %d): %s", sdc.get_error_line(), sdc.get_error_text())); + code = sdc.emit_code(); + preprocessed_code = code; + } else if (sdc.get_error_text() != "") { // Preprocessing failed. + WARN_PRINT(vformat("Shader conversion failed (line %d): %s", sdc.get_error_line(), sdc.get_error_text())); + } // If the code is reported as not deprecated, let it fall through to the compile step after this if block so that we get the full compile error. + } + } + } +#endif { String path = get_path(); @@ -99,7 +152,7 @@ void Shader::set_code(const String &p_code) { // 2) Server does not do interaction with Resource filetypes, this is a scene level feature. HashSet> new_include_dependencies; ShaderPreprocessor preprocessor; - Error result = preprocessor.preprocess(p_code, path, preprocessed_code, nullptr, nullptr, nullptr, &new_include_dependencies); + Error result = preprocessor.preprocess(code, path, preprocessed_code, nullptr, nullptr, nullptr, &new_include_dependencies); if (result == OK) { // This ensures previous include resources are not freed and then re-loaded during parse (which would make compiling slower) include_dependencies = new_include_dependencies; @@ -256,6 +309,20 @@ Array Shader::_get_shader_uniform_list(bool p_get_groups) { return ret; } +void Shader::_start_load(const StringName &p_res_format_type, int p_res_format_version) { +#ifndef DISABLE_DEPRECATED + if ((p_res_format_type == "binary" && p_res_format_version == 3) || (p_res_format_type == "text" && p_res_format_version == 2)) { + set_meta(_LOAD_COMPAT_META_PROPERTY, true); + } +#endif +} + +void Shader::_finish_load(const StringName &p_res_format_type, int p_res_format_version) { +#ifndef DISABLE_DEPRECATED + set_meta(_LOAD_COMPAT_META_PROPERTY, Variant()); +#endif +} + void Shader::_bind_methods() { ClassDB::bind_method(D_METHOD("get_mode"), &Shader::get_mode); diff --git a/scene/resources/shader.h b/scene/resources/shader.h index 18197419f36a..3701a53f5359 100644 --- a/scene/resources/shader.h +++ b/scene/resources/shader.h @@ -98,6 +98,9 @@ class Shader : public Resource { virtual RID get_rid() const override; + virtual void _start_load(const StringName &p_res_format_type, int p_res_format_version) override; + virtual void _finish_load(const StringName &p_res_format_type, int p_res_format_version) override; + Shader(); ~Shader(); }; diff --git a/servers/rendering/shader_converter.cpp b/servers/rendering/shader_converter.cpp new file mode 100644 index 000000000000..03e4f993fbf0 --- /dev/null +++ b/servers/rendering/shader_converter.cpp @@ -0,0 +1,2589 @@ +/**************************************************************************/ +/* shader_converter.cpp */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#include "core/error/error_macros.h" +#include "core/templates/pair.h" +#include "core/typedefs.h" +#ifndef DISABLE_DEPRECATED + +#include "shader_converter.h" + +#define SL ShaderLanguage + +const char *ShaderDeprecatedConverter::old_builtin_funcs[]{ + "abs", + "acos", + "acosh", + "all", + "any", + "asin", + "asinh", + "atan", + "atanh", + "bool", + "bvec2", + "bvec3", + "bvec4", + "ceil", + "clamp", + "cos", + "cosh", + "cross", + "dFdx", + "dFdy", + "degrees", + "determinant", + "distance", + "dot", + "equal", + "exp", + "exp2", + "faceforward", + "float", + "floatBitsToInt", + "floatBitsToUint", + "floor", + "fract", + "fwidth", + "greaterThan", + "greaterThanEqual", + "int", + "intBitsToFloat", + "inverse", + "inversesqrt", + "isinf", + "isnan", + "ivec2", + "ivec3", + "ivec4", + "length", + "lessThan", + "lessThanEqual", + "log", + "log2", + "mat2", + "mat3", + "mat4", + "matrixCompMult", + "max", + "min", + "mix", + "mod", + "modf", + "normalize", + "not", + "notEqual", + "outerProduct", + "pow", + "radians", + "reflect", + "refract", + "round", + "roundEven", + "sign", + "sin", + "sinh", + "smoothstep", + "sqrt", + "step", + "tan", + "tanh", + "texelFetch", + "texture", + "textureGrad", + "textureLod", + "textureProj", + "textureProjLod", + "textureSize", + "transpose", + "trunc", + "uint", + "uintBitsToFloat", + "uvec2", + "uvec3", + "uvec4", + "vec2", + "vec3", + "vec4", + nullptr +}; + +const ShaderDeprecatedConverter::RenamedBuiltins ShaderDeprecatedConverter::renamed_builtins[] = { + { "ALPHA_SCISSOR", "ALPHA_SCISSOR_THRESHOLD", { { RS::SHADER_SPATIAL, { "fragment" } } }, false }, + { "CAMERA_MATRIX", "INV_VIEW_MATRIX", { { RS::SHADER_SPATIAL, { "vertex", "fragment", "light" } } }, false }, + { "INV_CAMERA_MATRIX", "VIEW_MATRIX", { { RS::SHADER_SPATIAL, { "vertex", "fragment", "light" } } }, false }, + { "NORMALMAP", "NORMAL_MAP", { { RS::SHADER_CANVAS_ITEM, { "fragment" } }, { RS::SHADER_SPATIAL, { "fragment" } } }, false }, + { "NORMALMAP_DEPTH", "NORMAL_MAP_DEPTH", { { RS::SHADER_CANVAS_ITEM, { "fragment" } }, { RS::SHADER_SPATIAL, { "fragment" } } }, false }, + { "TRANSMISSION", "BACKLIGHT", { { RS::SHADER_SPATIAL, { "fragment", "light" } } }, false }, + { "WORLD_MATRIX", "MODEL_MATRIX", { { RS::SHADER_CANVAS_ITEM, { "vertex" } }, { RS::SHADER_SPATIAL, { "vertex", "fragment", "light" } } }, false }, + { "CLEARCOAT_GLOSS", "CLEARCOAT_ROUGHNESS", { { RS::SHADER_SPATIAL, { "fragment" } } }, true }, // Usages require inversion, manually handled + { "INDEX", "INDEX", { { RS::SHADER_PARTICLES, { "vertex" } } }, true }, // No rename, was previously an int (vs. uint), usages require wrapping in `int()`. + { nullptr, nullptr, {}, false }, +}; + +const ShaderDeprecatedConverter::RenamedRenderModes ShaderDeprecatedConverter::renamed_render_modes[] = { + { RS::SHADER_SPATIAL, "depth_draw_alpha_prepass", "depth_prepass_alpha" }, + { RS::SHADER_MAX, nullptr, nullptr }, +}; + +const ShaderDeprecatedConverter::RenamedHints ShaderDeprecatedConverter::renamed_hints[]{ + { "hint_albedo", SL::TokenType::TK_HINT_SOURCE_COLOR }, + { "hint_aniso", SL::TokenType::TK_HINT_ANISOTROPY_TEXTURE }, + { "hint_black", SL::TokenType::TK_HINT_DEFAULT_BLACK_TEXTURE }, + { "hint_black_albedo", SL::TokenType::TK_HINT_DEFAULT_BLACK_TEXTURE }, + { "hint_color", SL::TokenType::TK_HINT_SOURCE_COLOR }, + { "hint_transparent", SL::TokenType::TK_HINT_DEFAULT_TRANSPARENT_TEXTURE }, + { "hint_white", SL::TokenType::TK_HINT_DEFAULT_WHITE_TEXTURE }, + { nullptr, {} }, +}; + +const ShaderDeprecatedConverter::RenamedFunctions ShaderDeprecatedConverter::renamed_functions[]{ + { RS::SHADER_PARTICLES, SL::TK_TYPE_VOID, 0, "vertex", "process" }, + { RS::SHADER_MAX, SL::TK_EMPTY, 0, nullptr, nullptr }, +}; + +const ShaderDeprecatedConverter::RemovedRenderModes ShaderDeprecatedConverter::removed_render_modes[]{ + { RS::SHADER_SPATIAL, "specular_blinn", false }, + { RS::SHADER_SPATIAL, "specular_phong", false }, + { RS::SHADER_SPATIAL, "async_visible", true }, + { RS::SHADER_SPATIAL, "async_hidden", true }, + { RS::SHADER_MAX, nullptr, false }, +}; + +// These necessitate adding a uniform to the shader. +const ShaderDeprecatedConverter::RemovedBuiltins ShaderDeprecatedConverter::removed_builtins[]{ + { "SCREEN_TEXTURE", SL::TK_TYPE_SAMPLER2D, { SL::TK_HINT_SCREEN_TEXTURE, SL::TK_FILTER_LINEAR_MIPMAP }, { { RS::SHADER_SPATIAL, { "fragment" } }, { RS::SHADER_CANVAS_ITEM, { "fragment" } } } }, + { "DEPTH_TEXTURE", SL::TK_TYPE_SAMPLER2D, { SL::TK_HINT_DEPTH_TEXTURE, SL::TK_FILTER_LINEAR_MIPMAP }, { { RS::SHADER_SPATIAL, { "fragment" } } } }, + { "NORMAL_ROUGHNESS_TEXTURE", SL::TK_TYPE_SAMPLER2D, { SL::TK_HINT_NORMAL_ROUGHNESS_TEXTURE, SL::TK_FILTER_LINEAR_MIPMAP }, { { RS::SHADER_SPATIAL, { "fragment" } } } }, + { "MODULATE", SL::TK_ERROR, {}, { { RS::SHADER_CANVAS_ITEM, { "vertex", "fragment", "light" } } } }, // TODO: remove this when the MODULATE PR lands. + { nullptr, SL::TK_EMPTY, {}, {} }, +}; + +const char *ShaderDeprecatedConverter::removed_types[]{ + nullptr, +}; + +HashSet ShaderDeprecatedConverter::_new_builtin_funcs = HashSet(); + +HashSet ShaderDeprecatedConverter::_construct_new_builtin_funcs() { + List current_builtin_funcs; + ShaderLanguage::get_builtin_funcs(¤t_builtin_funcs); + HashSet old_funcs; + for (int i = 0; old_builtin_funcs[i] != nullptr; i++) { + old_funcs.insert(old_builtin_funcs[i]); + } + HashSet new_funcs; + for (List::Element *E = current_builtin_funcs.front(); E; E = E->next()) { + if (!old_funcs.has(E->get())) { + new_funcs.insert(E->get()); + } + } + return new_funcs; +} + +String ShaderDeprecatedConverter::get_builtin_rename(const String &p_name) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + return renamed_builtins[i].replacement; + } + } + return String(); +} + +bool ShaderDeprecatedConverter::has_builtin_rename(RS::ShaderMode p_mode, const String &p_name, const String &p_function) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + for (int j = 0; j < renamed_builtins[i].mode_functions.size(); j++) { + if (renamed_builtins[i].mode_functions[j].first == p_mode) { + if (p_function == "") { // Empty function means don't check function. + return true; + } + for (int k = 0; k < renamed_builtins[i].mode_functions[j].second.size(); k++) { + if (renamed_builtins[i].mode_functions[j].second[k] == p_function) { + return true; + } + } + } + } + } + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::get_removed_builtin_uniform_type(const String &p_name) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + return removed_builtins[i].uniform_type; + } + } + return SL::TK_EMPTY; +} + +Vector ShaderDeprecatedConverter::get_removed_builtin_hints(const String &p_name) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + return removed_builtins[i].hints; + } + } + return Vector(); +} + +bool ShaderDeprecatedConverter::_rename_has_special_handling(const String &p_name) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + return renamed_builtins[i].special_handling; + } + } + return false; +} + +void ShaderDeprecatedConverter::_get_builtin_renames_list(List *r_list) { + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + r_list->push_back(renamed_builtins[i].name); + } +} + +void ShaderDeprecatedConverter::_get_render_mode_renames_list(List *r_list) { + for (int i = 0; renamed_render_modes[i].name != nullptr; i++) { + r_list->push_back(renamed_render_modes[i].name); + } +} + +void ShaderDeprecatedConverter::_get_hint_renames_list(List *r_list) { + for (int i = 0; renamed_hints[i].name != nullptr; i++) { + r_list->push_back(renamed_hints[i].name); + } +} + +void ShaderDeprecatedConverter::_get_function_renames_list(List *r_list) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + r_list->push_back(renamed_functions[i].name); + } +} + +void ShaderDeprecatedConverter::_get_render_mode_removals_list(List *r_list) { + for (int i = 0; removed_render_modes[i].name != nullptr; i++) { + r_list->push_back(removed_render_modes[i].name); + } +} + +void ShaderDeprecatedConverter::_get_builtin_removals_list(List *r_list) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + r_list->push_back(removed_builtins[i].name); + } +} + +void ShaderDeprecatedConverter::_get_type_removals_list(List *r_list) { + for (int i = 0; removed_types[i] != nullptr; i++) { + r_list->push_back(removed_types[i]); + } +} + +Vector ShaderDeprecatedConverter::_get_funcs_builtin_rename(RS::ShaderMode p_mode, const String &p_name) { + Vector funcs; + for (int i = 0; renamed_builtins[i].name != nullptr; i++) { + if (renamed_builtins[i].name == p_name) { + for (int j = 0; j < renamed_builtins[i].mode_functions.size(); j++) { + if (renamed_builtins[i].mode_functions[j].first == p_mode) { + for (int k = 0; k < renamed_builtins[i].mode_functions[j].second.size(); k++) { + funcs.push_back(renamed_builtins[i].mode_functions[j].second[k]); + } + } + } + } + } + return funcs; +} + +Vector ShaderDeprecatedConverter::_get_funcs_builtin_removal(RS::ShaderMode p_mode, const String &p_name) { + Vector funcs; + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + for (int j = 0; j < removed_builtins[i].mode_functions.size(); j++) { + if (removed_builtins[i].mode_functions[j].first == p_mode) { + for (int k = 0; k < removed_builtins[i].mode_functions[j].second.size(); k++) { + funcs.push_back(removed_builtins[i].mode_functions[j].second[k]); + } + } + } + } + } + return funcs; +} + +bool ShaderDeprecatedConverter::is_removed_builtin(RS::ShaderMode p_mode, const String &p_name, const String &p_function) { + for (int i = 0; removed_builtins[i].name != nullptr; i++) { + if (removed_builtins[i].name == p_name) { + for (int j = 0; j < removed_builtins[i].mode_functions.size(); j++) { + if (removed_builtins[i].mode_functions[j].first == p_mode) { + if (p_function == "") { // Empty function means don't check function. + return true; + } + for (int k = 0; k < removed_builtins[i].mode_functions[j].second.size(); k++) { + if (removed_builtins[i].mode_functions[j].second[k] == p_function) { + return true; + } + } + } + } + } + } + return false; +} + +bool ShaderDeprecatedConverter::has_hint_replacement(const String &p_name) { + for (int i = 0; renamed_hints[i].name != nullptr; i++) { + if (renamed_hints[i].name == p_name) { + return true; + } + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::get_hint_replacement(const String &p_name) { + for (int i = 0; renamed_hints[i].name != nullptr; i++) { + if (renamed_hints[i].name == p_name) { + return renamed_hints[i].replacement; + } + } + return {}; +} + +bool ShaderDeprecatedConverter::is_renamed_render_mode(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; renamed_render_modes[i].name != nullptr; i++) { + if (renamed_render_modes[i].mode == p_mode && renamed_render_modes[i].name == p_name) { + return true; + } + } + return false; +} + +String ShaderDeprecatedConverter::get_render_mode_rename(const String &p_name) { + for (int i = 0; renamed_render_modes[i].name != nullptr; i++) { + if (renamed_render_modes[i].name == p_name) { + return renamed_render_modes[i].replacement; + } + } + return {}; +} + +bool ShaderDeprecatedConverter::is_renamed_main_function(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].mode == p_mode && renamed_functions[i].name == p_name) { + return true; + } + } + return false; +} + +bool ShaderDeprecatedConverter::is_renamee_main_function(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].mode == p_mode && renamed_functions[i].replacement == p_name) { + return true; + } + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::get_renamed_function_type(const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].name == p_name) { + return renamed_functions[i].type; + } + } + return SL::TK_MAX; +} + +int ShaderDeprecatedConverter::get_renamed_function_arg_count(const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].name == p_name) { + return renamed_functions[i].arg_count; + } + } + return -1; +} + +bool ShaderDeprecatedConverter::FunctionDecl::is_renamed_main_function(RS::ShaderMode p_mode) const { + if (!name_pos || !type_pos) { + return false; + } + if (ShaderDeprecatedConverter::is_renamed_main_function(p_mode, name_pos->get().text) && type_pos->get().type == get_renamed_function_type(name_pos->get().text) && arg_count == get_renamed_function_arg_count(name_pos->get().text)) { + return true; + } + return false; +} + +bool ShaderDeprecatedConverter::FunctionDecl::is_new_main_function(RS::ShaderMode p_mode) const { + if (!name_pos || !type_pos) { + return false; + } + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].mode == p_mode && renamed_functions[i].replacement == name_pos->get().text && renamed_functions[i].type == type_pos->get().type && arg_count == renamed_functions[i].arg_count) { + return true; + } + } + return false; +} + +String ShaderDeprecatedConverter::get_main_function_rename(const String &p_name) { + for (int i = 0; renamed_functions[i].name != nullptr; i++) { + if (renamed_functions[i].name == p_name) { + return renamed_functions[i].replacement; + } + } + return String(); +} + +bool ShaderDeprecatedConverter::has_removed_render_mode(RS::ShaderMode p_mode, const String &p_name) { + for (int i = 0; removed_render_modes[i].name != nullptr; i++) { + if (removed_render_modes[i].mode == p_mode && removed_render_modes[i].name == p_name) { + return true; + } + } + return false; +} + +bool ShaderDeprecatedConverter::can_remove_render_mode(const String &p_name) { + for (int i = 0; removed_render_modes[i].name != nullptr; i++) { + if (removed_render_modes[i].name == p_name) { + return removed_render_modes[i].can_remove; + } + } + return false; +} + +bool ShaderDeprecatedConverter::has_removed_type(const String &p_name) { + for (int i = 0; removed_types[i] != nullptr; i++) { + if (removed_types[i] == p_name) { + return true; + } + } + return false; +} + +static constexpr const char *token_to_str[] = { + "", // TK_EMPTY + "", // TK_IDENTIFIER + "true", + "false", + "", // TK_FLOAT_CONSTANT + "", // TK_INT_CONSTANT + "", // TK_UINT_CONSTANT + "", // TK_STRING_CONSTANT + "void", + "bool", + "bvec2", + "bvec3", + "bvec4", + "int", + "ivec2", + "ivec3", + "ivec4", + "uint", + "uvec2", + "uvec3", + "uvec4", + "float", + "vec2", + "vec3", + "vec4", + "mat2", + "mat3", + "mat4", + "sampler2D", + "isampler2D", + "usampler2D", + "sampler2DArray", + "isampler2DArray", + "usampler2DArray", + "sampler3D", + "isampler3D", + "usampler3D", + "samplerCube", + "samplerCubeArray", + "samplerExternalOES", + "flat", + "smooth", + "const", + "struct", + "lowp", + "mediump", + "highp", + "==", + "!=", + "<", + "<=", + ">", + ">=", + "&&", + "||", + "!", + "+", + "-", + "*", + "/", + "%", + "<<", + ">>", + "=", + "+=", + "-=", + "*=", + "/=", + "%=", + "<<=", + ">>=", + "&=", + "|=", + "^=", + "&", + "|", + "^", + "~", + "++", + "--", + "if", + "else", + "for", + "while", + "do", + "switch", + "case", + "default", + "break", + "continue", + "return", + "discard", + "[", + "]", + "{", + "}", + "(", + ")", + "?", + ",", + ":", + ";", + ".", + "uniform", + "group_uniforms", + "instance", + "global", + "varying", + "in", + "out", + "inout", + "render_mode", + "hint_default_white", + "hint_default_black", + "hint_default_transparent", + "hint_normal", + "hint_roughness_normal", + "hint_roughness_r", + "hint_roughness_g", + "hint_roughness_b", + "hint_roughness_a", + "hint_roughness_gray", + "hint_anisotropy", + "source_color", + "hint_range", + "hint_enum", + "instance_index", + "hint_screen_texture", + "hint_normal_roughness_texture", + "hint_depth_texture", + "filter_nearest", + "filter_linear", + "filter_nearest_mipmap", + "filter_linear_mipmap", + "filter_nearest_mipmap_anisotropic", + "filter_linear_mipmap_anisotropic", + "repeat_enable", + "repeat_disable", + "shader_type", + "", // TK_CURSOR + "", // TK_ERROR + "", // TK_EOF + "\t", + "\r", + " ", + "\n", + "", // TK_BLOCK_COMMENT + "", // TK_LINE_COMMENT + "", // TK_PREPROC_DIRECTIVE +}; +static_assert(ShaderLanguage::TK_MAX == sizeof(token_to_str) / sizeof(token_to_str[0]), "token_to_str length does not match token count (Did TK_MAX change?)"); + +bool ShaderDeprecatedConverter::token_is_skippable(const Token &p_tk) { + switch (p_tk.type) { + case ShaderLanguage::TK_TAB: + case ShaderLanguage::TK_CR: + case ShaderLanguage::TK_SPACE: + case ShaderLanguage::TK_NEWLINE: + case ShaderLanguage::TK_BLOCK_COMMENT: + case ShaderLanguage::TK_LINE_COMMENT: + case ShaderLanguage::TK_PREPROC_DIRECTIVE: + return true; + default: + break; + } + return false; +} + +List::Element *ShaderDeprecatedConverter::_get_next_token_ptr(List::Element *p_curr_ptr) const { + ERR_FAIL_COND_V(p_curr_ptr == nullptr, p_curr_ptr); + if (p_curr_ptr->next() == nullptr) { + return p_curr_ptr; + } + p_curr_ptr = p_curr_ptr->next(); + while (token_is_skippable(p_curr_ptr->get())) { + if (p_curr_ptr->next() == nullptr) { + return p_curr_ptr; + } + p_curr_ptr = p_curr_ptr->next(); + } + return p_curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::_get_prev_token_ptr(List::Element *_curr_ptr) const { + ERR_FAIL_COND_V(_curr_ptr == nullptr, _curr_ptr); + if (_curr_ptr->prev() == nullptr) { + return _curr_ptr; + } + _curr_ptr = _curr_ptr->prev(); + while (token_is_skippable(_curr_ptr->get())) { + if (_curr_ptr->prev() == nullptr) { + return _curr_ptr; + } + _curr_ptr = _curr_ptr->prev(); + } + return _curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::get_next_token() { + curr_ptr = _get_next_token_ptr(curr_ptr); + return curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::get_prev_token() { + curr_ptr = _get_prev_token_ptr(curr_ptr); + return curr_ptr; +} + +List::Element *ShaderDeprecatedConverter::remove_cur_and_get_next() { + ERR_FAIL_COND_V(!curr_ptr, nullptr); + List::Element *prev = curr_ptr->prev(); + if (!prev) { + prev = curr_ptr->next(); + code_tokens.erase(curr_ptr); + while (token_is_skippable(prev->get())) { + if (prev->next() == nullptr) { + return prev; + } + prev = prev->next(); + } + return prev; + } + code_tokens.erase(curr_ptr); + curr_ptr = prev; + return get_next_token(); +} + +SL::TokenType ShaderDeprecatedConverter::_peek_tk_type(int64_t p_count, List::Element **r_pos) const { + ERR_FAIL_COND_V(!curr_ptr, ShaderLanguage::TK_EOF); + if (p_count == 0) { + return curr_ptr->get().type; + } + + bool backwards = p_count < 0; + uint64_t max_count = abs(p_count); + TokenE *start_ptr = curr_ptr; + for (uint64_t i = 0; i < max_count; i++) { + TokenE *_ptr = backwards ? _get_prev_token_ptr(start_ptr) : _get_next_token_ptr(start_ptr); + if (!_ptr) { + if (r_pos) { + *r_pos = start_ptr; + } + return ShaderLanguage::TK_EOF; + } + start_ptr = _ptr; + } + if (r_pos) { + *r_pos = start_ptr; + } + return start_ptr->get().type; +} + +bool ShaderDeprecatedConverter::scope_has_decl(const String &p_scope, const String &p_name) const { + if (uniform_decls.has(p_name) || + (scope_declarations.has("") && scope_declarations[""].has(p_name)) || + (scope_declarations.has(p_scope) && scope_declarations[p_scope].has(p_name))) { + return true; + } + return false; +} + +SL::TokenType ShaderDeprecatedConverter::peek_next_tk_type(uint32_t p_count) const { + return _peek_tk_type(p_count); +} + +SL::TokenType ShaderDeprecatedConverter::peek_prev_tk_type(uint32_t p_count) const { + return _peek_tk_type(-((int64_t)p_count)); +} + +List::Element *ShaderDeprecatedConverter::get_pos() const { + ERR_FAIL_COND_V(!curr_ptr, nullptr); + return curr_ptr; +} + +bool ShaderDeprecatedConverter::reset_to(List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + curr_ptr = p_pos; + return true; +} + +bool ShaderDeprecatedConverter::insert_after(const Vector &p_token_list, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + for (int i = p_token_list.size() - 1; i >= 0; i--) { + const Token &tk = p_token_list[i]; + code_tokens.insert_after(p_pos, { tk.type, tk.text, tk.constant, tk.line, tk.length, NEW_IDENT }); + } + return true; +} + +bool ShaderDeprecatedConverter::insert_before(const Vector &p_token_list, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + for (const Token &tk : p_token_list) { + code_tokens.insert_before(p_pos, { tk.type, tk.text, tk.constant, tk.line, tk.length, NEW_IDENT }); + } + return true; +} + +bool ShaderDeprecatedConverter::insert_after(const Token &p_token, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + Token new_token = p_token; + new_token.pos = NEW_IDENT; + code_tokens.insert_after(p_pos, new_token); + return true; +} + +bool ShaderDeprecatedConverter::insert_before(const Token &p_token, List::Element *p_pos) { + ERR_FAIL_COND_V(p_pos == nullptr, false); + Token new_token = p_token; + new_token.pos = NEW_IDENT; + code_tokens.insert_before(p_pos, new_token); + return true; +} + +List::Element *ShaderDeprecatedConverter::replace_curr(const Token &p_token, String comment_format) { + ERR_FAIL_COND_V(curr_ptr == nullptr, nullptr); + Token new_token = p_token; + new_token.pos = NEW_IDENT; + List::Element *prev = curr_ptr; + if (!comment_format.is_empty()) { + int count = comment_format.count("%s"); + if (count == 0) { + comment_format = RTR(comment_format) + " '%s' renamed to '%s'"; + } else if (count == 1) { + comment_format = RTR(comment_format) + ", renamed to '%s'"; + } + _add_comment_before(vformat(RTR(comment_format), get_token_literal_text(curr_ptr->get()), p_token.text), curr_ptr, false); + } + curr_ptr = code_tokens.insert_before(curr_ptr, new_token); + ERR_FAIL_COND_V(!code_tokens.erase(prev), nullptr); + return curr_ptr; +} + +SL::Token ShaderDeprecatedConverter::mkTok(TokenType p_type, const StringName &p_text, double p_constant, uint16_t p_line) { + return { p_type, p_text, p_constant, p_line, 0, NEW_IDENT }; +} + +bool ShaderDeprecatedConverter::_insert_uniform_declaration(const String &p_name) { + if (after_shader_decl == nullptr) { + return false; + } + TokenType type = get_removed_builtin_uniform_type(p_name); + Vector hints = get_removed_builtin_hints(p_name); + Vector uni_decl = { mkTok(TT::TK_NEWLINE), mkTok(TT::TK_UNIFORM), mkTok(TT::TK_SPACE), mkTok(type), + mkTok(TT::TK_SPACE), mkTok(TT::TK_IDENTIFIER, p_name), mkTok(TT::TK_SPACE), mkTok(TT::TK_COLON), + mkTok(TT::TK_SPACE) }; + for (int i = 0; i < hints.size(); i++) { + uni_decl.append(mkTok(hints[i])); + if (i < hints.size() - 1) { + uni_decl.append(mkTok(TT::TK_COMMA)); + uni_decl.append(mkTok(TT::TK_SPACE)); + } + } + uni_decl.append_array({ mkTok(TT::TK_SEMICOLON), mkTok(TT::TK_NEWLINE) }); + if (!insert_after(uni_decl, after_shader_decl)) { + return false; + } + TokenE *cur_pos = get_pos(); + reset_to(after_shader_decl); + UniformDecl uni; + uni.start_pos = get_next_token(); // uniform + uni.type_pos = get_next_token(); // type + uni.name_pos = get_next_token(); // id + get_next_token(); // colon + for (int i = 0; i < hints.size(); i++) { + uni.hint_poses.push_back(get_next_token()); // hint + if (i < hints.size() - 1) { + get_next_token(); // comma + } + } + uni.end_pos = get_next_token(); + uniform_decls[p_name] = uni; + _add_comment_before(vformat(RTR("Usage of deprecated built-in '%s' requires a uniform declaration."), p_name), uni.start_pos, false); + reset_to(cur_pos); + return true; +} + +RS::ShaderMode ShaderDeprecatedConverter::get_shader_mode_from_string(const String &p_mode) { + if (p_mode == "spatial") { + return RS::SHADER_SPATIAL; + } else if (p_mode == "canvas_item") { + return RS::SHADER_CANVAS_ITEM; + } else if (p_mode == "particles") { + return RS::SHADER_PARTICLES; + } else { // 3.x didn't support anything else. + return RS::SHADER_MAX; + } +} +// Remove from the current token to end (exclsusive) and return the new current token. +List::Element *ShaderDeprecatedConverter::_remove_from_curr_to(TokenE *p_end) { + ERR_FAIL_COND_V(p_end == nullptr, nullptr); + while (curr_ptr != p_end) { + TokenE *next = curr_ptr->next(); + code_tokens.erase(curr_ptr); + curr_ptr = next; + } + return curr_ptr; +} + +SL::TokenType ShaderDeprecatedConverter::get_tokentype_from_text(const String &p_text) { + for (int i = 0; i < SL::TK_MAX; i++) { + if (token_to_str[i] == p_text) { + return static_cast(i); + } + } + return SL::TK_MAX; +} + +String ShaderDeprecatedConverter::get_tokentype_text(TokenType p_tk_type) { + return token_to_str[p_tk_type]; +} + +List::Element *ShaderDeprecatedConverter::_get_end_of_closure() { + int additional_closures = 0; + TokenE *ptr = curr_ptr; + bool start_is_scope_start = false; + switch (ptr->get().type) { + case TT::TK_CURLY_BRACKET_OPEN: + case TT::TK_PARENTHESIS_OPEN: + case TT::TK_BRACKET_OPEN: + start_is_scope_start = true; + break; + default: + break; + } + for (; ptr; ptr = ptr->next()) { + switch (ptr->get().type) { + case TT::TK_CURLY_BRACKET_OPEN: + case TT::TK_PARENTHESIS_OPEN: + case TT::TK_BRACKET_OPEN: { + additional_closures++; + } break; + case TT::TK_CURLY_BRACKET_CLOSE: + case TT::TK_PARENTHESIS_CLOSE: + case TT::TK_BRACKET_CLOSE: { + if (additional_closures > 0) { + additional_closures--; + if (start_is_scope_start && additional_closures == 0) { + return ptr; + } + } else { + return ptr; + } + } break; + case TT::TK_SEMICOLON: + case TT::TK_COMMA: { + if (additional_closures <= 0) { + return _get_prev_token_ptr(ptr); + } + } break; + case TT::TK_EOF: + case TT::TK_ERROR: { + err_line = curr_ptr->get().line + 1; + err_str = ptr->get().type == TT::TK_ERROR ? vformat(RTR("Parser Error (%s) ", ptr->get().text)) : vformat(RTR("Could not find end of closure for token '%s'"), get_tokentype_text(curr_ptr->get().type)); + return ptr; + } break; + default: + break; + } + } + return ptr; +} + +bool ShaderDeprecatedConverter::token_is_type(const Token &p_tk) { + return (ShaderLanguage::is_token_datatype(p_tk.type)) || struct_decls.has(get_token_literal_text(p_tk)) || (p_tk.type == TT::TK_IDENTIFIER && (has_removed_type(p_tk.text))); +} + +bool ShaderDeprecatedConverter::token_is_hint(const Token &p_tk) { + if (p_tk.type == TT::TK_IDENTIFIER) { + return has_hint_replacement(p_tk.text); + } + return SL::is_token_hint(p_tk.type); +} + +String ShaderDeprecatedConverter::get_token_literal_text(const Token &p_tk) const { + switch (p_tk.type) { + case TT::TK_PREPROC_DIRECTIVE: + case TT::TK_LINE_COMMENT: + case TT::TK_BLOCK_COMMENT: + case TT::TK_IDENTIFIER: { // Identifiers prefixed with `__` are modified to `_dup_` by the SL parser + if (p_tk.pos == NEW_IDENT) { + return p_tk.text; + } else { + return old_code.substr(p_tk.pos, p_tk.length); + } + } break; + case TT::TK_INT_CONSTANT: + case TT::TK_FLOAT_CONSTANT: + case TT::TK_UINT_CONSTANT: { + if (p_tk.pos == NEW_IDENT) { + // Fix for 3.x float constants not having a decimal point. + if (!p_tk.is_integer_constant() && p_tk.text != "") { + return p_tk.text; + } + String const_str = rtos(p_tk.constant); + if (!p_tk.is_integer_constant() && !const_str.contains(".")) { + const_str += ".0"; + } + return const_str; + } else { + return old_code.substr(p_tk.pos, p_tk.length); + } + } break; + case TT::TK_ERROR: + case TT::TK_EOF: { + return ""; + } break; + default: + break; + } + return token_to_str[p_tk.type]; +} + +bool ShaderDeprecatedConverter::tokentype_is_identifier(const TokenType &p_tk_type) { + return p_tk_type == TT::TK_IDENTIFIER || tokentype_is_new_reserved_keyword(p_tk_type); +} + +bool ShaderDeprecatedConverter::tokentype_is_new_type(const TokenType &p_type) { + // the following types are in both 3.x and 4.x + switch (p_type) { + case TT::TK_TYPE_VOID: + case TT::TK_TYPE_BOOL: + case TT::TK_TYPE_BVEC2: + case TT::TK_TYPE_BVEC3: + case TT::TK_TYPE_BVEC4: + case TT::TK_TYPE_INT: + case TT::TK_TYPE_IVEC2: + case TT::TK_TYPE_IVEC3: + case TT::TK_TYPE_IVEC4: + case TT::TK_TYPE_UINT: + case TT::TK_TYPE_UVEC2: + case TT::TK_TYPE_UVEC3: + case TT::TK_TYPE_UVEC4: + case TT::TK_TYPE_FLOAT: + case TT::TK_TYPE_VEC2: + case TT::TK_TYPE_VEC3: + case TT::TK_TYPE_VEC4: + case TT::TK_TYPE_MAT2: + case TT::TK_TYPE_MAT3: + case TT::TK_TYPE_MAT4: + case TT::TK_TYPE_SAMPLER2D: + case TT::TK_TYPE_ISAMPLER2D: + case TT::TK_TYPE_USAMPLER2D: + case TT::TK_TYPE_SAMPLER2DARRAY: + case TT::TK_TYPE_ISAMPLER2DARRAY: + case TT::TK_TYPE_USAMPLER2DARRAY: + case TT::TK_TYPE_SAMPLER3D: + case TT::TK_TYPE_ISAMPLER3D: + case TT::TK_TYPE_USAMPLER3D: + case TT::TK_TYPE_SAMPLERCUBE: + case TT::TK_TYPE_SAMPLEREXT: + return false; + default: + break; + } + return SL::is_token_datatype(p_type); +} + +// checks for reserved keywords only found in 4.x +bool ShaderDeprecatedConverter::tokentype_is_new_reserved_keyword(const TokenType &tk_type) { + switch (tk_type) { + // The following keyword tokens are in both 3.x and 4.x. + case TT::TK_ARG_IN: + case TT::TK_ARG_INOUT: + case TT::TK_ARG_OUT: + case TT::TK_CF_BREAK: + case TT::TK_CF_CASE: + case TT::TK_CF_CONTINUE: + case TT::TK_CF_DEFAULT: + case TT::TK_CF_DISCARD: + case TT::TK_CF_DO: + case TT::TK_CF_ELSE: + case TT::TK_CF_FOR: + case TT::TK_CF_IF: + case TT::TK_CF_RETURN: + case TT::TK_CF_SWITCH: + case TT::TK_CF_WHILE: + case TT::TK_CONST: + case TT::TK_ERROR: + case TT::TK_FALSE: + case TT::TK_HINT_NORMAL_TEXTURE: + case TT::TK_HINT_RANGE: + case TT::TK_INTERPOLATION_FLAT: + case TT::TK_INTERPOLATION_SMOOTH: + case TT::TK_PRECISION_HIGH: + case TT::TK_PRECISION_LOW: + case TT::TK_PRECISION_MID: + case TT::TK_RENDER_MODE: + case TT::TK_SHADER_TYPE: + case TT::TK_STRUCT: + case TT::TK_TRUE: + case TT::TK_TYPE_BOOL: + case TT::TK_TYPE_BVEC2: + case TT::TK_TYPE_BVEC3: + case TT::TK_TYPE_BVEC4: + case TT::TK_TYPE_FLOAT: + case TT::TK_TYPE_INT: + case TT::TK_TYPE_ISAMPLER2D: + case TT::TK_TYPE_ISAMPLER2DARRAY: + case TT::TK_TYPE_ISAMPLER3D: + case TT::TK_TYPE_IVEC2: + case TT::TK_TYPE_IVEC3: + case TT::TK_TYPE_IVEC4: + case TT::TK_TYPE_MAT2: + case TT::TK_TYPE_MAT3: + case TT::TK_TYPE_MAT4: + case TT::TK_TYPE_SAMPLER2D: + case TT::TK_TYPE_SAMPLER2DARRAY: + case TT::TK_TYPE_SAMPLER3D: + case TT::TK_TYPE_SAMPLERCUBE: + case TT::TK_TYPE_SAMPLEREXT: + case TT::TK_TYPE_UINT: + case TT::TK_TYPE_USAMPLER2D: + case TT::TK_TYPE_USAMPLER2DARRAY: + case TT::TK_TYPE_USAMPLER3D: + case TT::TK_TYPE_UVEC2: + case TT::TK_TYPE_UVEC3: + case TT::TK_TYPE_UVEC4: + case TT::TK_TYPE_VEC2: + case TT::TK_TYPE_VEC3: + case TT::TK_TYPE_VEC4: + case TT::TK_TYPE_VOID: + case TT::TK_UNIFORM: + case TT::TK_VARYING: + case TT::TK_MAX: + return false; + default: + break; + } + return SL::is_token_keyword(tk_type); +} + +bool ShaderDeprecatedConverter::tokentype_is_new_hint(const TokenType &tk_type) { + switch (tk_type) { + case TT::TK_HINT_NORMAL_TEXTURE: // These two are in both 3.x and 4.x. + case TT::TK_HINT_RANGE: + return false; + default: + break; + } + return SL::is_token_hint(tk_type); +} + +bool ShaderDeprecatedConverter::id_is_new_builtin_func(const String &p_name) { + if (_new_builtin_funcs.is_empty()) { + _new_builtin_funcs = _construct_new_builtin_funcs(); + } + return _new_builtin_funcs.has(p_name); +} + +void ShaderDeprecatedConverter::_get_new_builtin_funcs_list(List *r_list) { + if (_new_builtin_funcs.is_empty()) { + _new_builtin_funcs = _construct_new_builtin_funcs(); + } + for (const String &k : _new_builtin_funcs) { + r_list->push_back(k); + } +} + +String ShaderDeprecatedConverter::get_report() { + String report_str; + for (const KeyValue> &p : report) { + for (const String &v : p.value) { + report_str += vformat(RTR("Line %d: %s\n"), p.key, v); + } + } + return report_str; +} + +bool ShaderDeprecatedConverter::_add_to_report(int p_line, const String &p_msg, int level) { + String message = p_msg; + if (level == 1) { + message = "WARNING: " + message; + } else if (level == 2) { + message = "ERROR: " + message; + } + if (!report.has(p_line)) { + report[p_line] = Vector(); + } + if (report[p_line].has(p_msg)) { + return false; + } + report[p_line].push_back(p_msg); + return true; +} + +bool ShaderDeprecatedConverter::_add_comment_before(const String &p_comment, List::Element *p_pos, bool warning) { + // Peek back until we hit a newline or the start of the file (EOF). + TokenE *start_pos = p_pos; + if (!start_pos) { + return false; + } + if (!_add_to_report(start_pos->get().line, p_comment, warning ? 1 : 0)) { + // Already added. + return true; + } + while (start_pos->prev() && start_pos->get().type != TT::TK_NEWLINE && start_pos->get().type != TT::TK_EOF) { + start_pos = start_pos->prev(); + } + String block_comment = vformat("/* !convert%s */\n", (warning ? " WARNING: " : ": ") + p_comment); + // In case this has been run through the converter before, check if the token before this is a block comment and has the same comment. + TokenE *prev = start_pos->prev(); + while (prev && prev->get().type == TT::TK_BLOCK_COMMENT) { + if (get_token_literal_text(start_pos->next()->get()).strip_edges() == block_comment.strip_edges()) { + return true; + } + prev = prev->prev(); + } + return insert_after(mkTok(TT::TK_BLOCK_COMMENT, block_comment), start_pos); +} + +bool ShaderDeprecatedConverter::_add_comment_at_eol(const String &p_comment, List::Element *p_pos) { + // Peek forward until we hit a newline or the end of the file (EOF). + TokenE *start_pos = p_pos ? p_pos : get_pos(); + if (!start_pos) { + return false; + } + while (start_pos->get().type != TT::TK_NEWLINE && start_pos->get().type != TT::TK_EOF) { + start_pos = start_pos->next(); + } + String comment = "/* !convert: " + p_comment + " */"; + if (start_pos->prev() && start_pos->prev()->get().type == TT::TK_BLOCK_COMMENT && get_token_literal_text(start_pos->prev()->get()) == comment) { + return true; + } + return insert_before(mkTok(TT::TK_BLOCK_COMMENT, comment), start_pos); +} + +void ShaderDeprecatedConverter::reset() { + ShaderLanguage sl; + code_tokens.clear(); + sl.token_debug_stream(old_code, code_tokens, true); + code_tokens.push_back(eof_token); + code_tokens.push_front(eof_token); + uniform_decls.clear(); + var_decls.clear(); + struct_decls.clear(); + function_decls.clear(); + scope_declarations.clear(); + after_shader_decl = code_tokens.front(); + curr_ptr = code_tokens.front(); + report.clear(); +} + +#define COND_MSG_FAIL(m_cond, m_msg) \ + if (unlikely(m_cond)) { \ + err_str = m_msg; \ + return false; \ + } +#define COND_LINE_MSG_FAIL(m_cond, m_line, m_msg) \ + if (unlikely(m_cond)) { \ + err_line = m_line + 1; \ + err_str = m_msg; \ + _add_to_report(err_line, err_str, 2); \ + return false; \ + } +#define LINE_MSG_FAIL(m_line, m_msg) \ + err_line = m_line + 1; \ + err_str = m_msg; \ + return false; +#define MSG_FAIL(m_msg) \ + err_str = m_msg; \ + return false; + +#define EOF_FAIL(m_tok_E) \ + COND_MSG_FAIL(m_tok_E == nullptr, RTR("Unexpected end of file")); \ + COND_LINE_MSG_FAIL(m_tok_E->get().type == TT::TK_EOF || m_tok_E->get().type == TT::TK_ERROR, m_tok_E->get().line, m_tok_E->get().type == TT::TK_ERROR ? vformat(RTR("Parser Error (%s) ", m_tok_E->get().text)) : RTR("Unexpected end of file")); +#define CLOSURE_FAIL(m_tok_E) \ + COND_MSG_FAIL(m_tok_E == nullptr, RTR("Unexpected end of file")); \ + if (unlikely(m_tok_E->get().type == TT::TK_EOF || m_tok_E->get().type == TT::TK_ERROR)) { \ + return false; \ + } + +// At uniform statement. +bool ShaderDeprecatedConverter::_parse_uniform() { + UniformDecl uni; + uni.start_pos = get_pos(); + DEV_ASSERT(uni.start_pos && uni.start_pos->get().type == TT::TK_UNIFORM); + uni.uniform_stmt_pos = uni.start_pos; + if (SL::is_token_uniform_qual(peek_prev_tk_type())) { // 3.x doesn't support these. + uni.start_pos = get_prev_token(); + get_next_token(); // Back to the uniform. + } + TokenE *next_tk = get_next_token(); + EOF_FAIL(next_tk); + while (SL::is_token_precision(next_tk->get().type) || SL::is_token_interpolation(next_tk->get().type)) { + if (SL::is_token_interpolation(next_tk->get().type)) { // Interpolations are not supported for uniforms in newer versions of Godot. + uni.interp_qual_pos = next_tk; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + COND_LINE_MSG_FAIL(!token_is_type(next_tk->get()), next_tk->get().line, RTR("Expected type after 'uniform'")); + uni.type_pos = next_tk; + next_tk = get_next_token(); + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + uni.is_array = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + COND_LINE_MSG_FAIL(!tokentype_is_identifier(next_tk->get().type), next_tk->get().line, RTR("Expected identifier after uniform type")); + String name = get_token_literal_text(next_tk->get()); + uni.name_pos = next_tk; + next_tk = get_next_token(); + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + uni.is_array = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + if (next_tk->get().type == TT::TK_COLON) { + while (true) { + next_tk = get_next_token(); + EOF_FAIL(next_tk); + COND_LINE_MSG_FAIL(!token_is_hint(next_tk->get()), next_tk->get().line, RTR("Expected hint after ':' in uniform declaration")); + uni.hint_poses.push_back(next_tk); + next_tk = get_next_token(); + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_PARENTHESIS_OPEN) { + next_tk = _get_end_of_closure(); + CLOSURE_FAIL(next_tk); + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_PARENTHESIS_CLOSE, next_tk->get().line, RTR("Expected ')' after hint range")); + reset_to(next_tk); // Skip the hint range. + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + if (next_tk->get().type != TT::TK_COMMA) { + break; + } + } + } + if (next_tk->get().type == TT::TK_OP_ASSIGN) { + next_tk = _get_end_of_closure(); + CLOSURE_FAIL(next_tk); + reset_to(next_tk); // Skip the assignment. + next_tk = get_next_token(); + } + uni.end_pos = next_tk; + EOF_FAIL(uni.end_pos); + COND_LINE_MSG_FAIL(uni.end_pos->get().type != TT::TK_SEMICOLON, uni.end_pos->get().line, RTR("Expected ';' after uniform declaration")); + uniform_decls[name] = uni; + return true; +} + +bool ShaderDeprecatedConverter::_skip_uniform() { + TokenE *cur_tok = get_pos(); + DEV_ASSERT(cur_tok && cur_tok->get().type == TT::TK_UNIFORM); + for (KeyValue &kv : uniform_decls) { + if (kv.value.uniform_stmt_pos == cur_tok) { + reset_to(kv.value.end_pos); + return true; + } + } + LINE_MSG_FAIL(cur_tok->get().line, RTR("Uniform declaration not found")); +} + +bool ShaderDeprecatedConverter::_skip_array_size() { + TokenE *next_tk = get_pos(); + DEV_ASSERT(next_tk && next_tk->get().type == TT::TK_BRACKET_OPEN); + next_tk = _get_end_of_closure(); + CLOSURE_FAIL(next_tk); + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_BRACKET_CLOSE, next_tk->get().line, RTR("Expected ']' after array type")); + reset_to(next_tk); // Skip to end. + return true; +} + +bool ShaderDeprecatedConverter::_skip_struct() { + DEV_ASSERT(get_pos() && get_pos()->get().type == TT::TK_STRUCT); + TokenE *struct_name = get_next_token(); + EOF_FAIL(struct_name); + TokenE *struct_body_start; + if (struct_name->get().type == TT::TK_CURLY_BRACKET_OPEN) { + struct_body_start = struct_name; + } else { + struct_body_start = get_next_token(); + } + EOF_FAIL(struct_body_start); + COND_LINE_MSG_FAIL(struct_body_start->get().type != TT::TK_CURLY_BRACKET_OPEN, struct_body_start->get().line, RTR("Expected '{' after struct declaration")); + TokenE *struct_body_end = _get_end_of_closure(); + CLOSURE_FAIL(struct_body_end); + COND_LINE_MSG_FAIL(struct_body_end->get().type != TT::TK_CURLY_BRACKET_CLOSE, struct_body_start->get().line, RTR("Expected '}' bracket at end of struct declaration")); + reset_to(struct_body_end); + return true; +} + +bool ShaderDeprecatedConverter::_tok_is_start_of_decl(const Token &p_tk) { + return token_is_type(p_tk) || p_tk.type == TT::TK_CONST || p_tk.type == TT::TK_VARYING || SL::is_token_precision(p_tk.type) || SL::is_token_interpolation(p_tk.type); +} + +bool ShaderDeprecatedConverter::_parse_struct() { + DEV_ASSERT(get_pos() && get_pos()->get().type == TT::TK_STRUCT); + TokenE *struct_start = get_pos(); + TokenE *struct_name_pos = get_next_token(); + EOF_FAIL(struct_name_pos); + if (struct_name_pos->get().type == TT::TK_CURLY_BRACKET_OPEN) { + return false; // No anonymous structs. + } + TokenE *struct_body_start = get_next_token(); + EOF_FAIL(struct_body_start); + COND_LINE_MSG_FAIL(struct_body_start->get().type != TT::TK_CURLY_BRACKET_OPEN, struct_body_start->get().line, RTR("Expected '{' after struct declaration")); + TokenE *struct_body_end = _get_end_of_closure(); + CLOSURE_FAIL(struct_body_end); + COND_LINE_MSG_FAIL(struct_body_end->get().type != TT::TK_CURLY_BRACKET_CLOSE, struct_body_start->get().line, RTR("Expected '}' bracket at end of struct declaration")); + + COND_LINE_MSG_FAIL(!tokentype_is_identifier(struct_name_pos->get().type), struct_name_pos->get().line, RTR("Expected identifier after 'struct'")); + String struct_name = get_token_literal_text(struct_name_pos->get()); + StructDecl struct_decl; + struct_decl.start_pos = struct_start; + struct_decl.name_pos = struct_name_pos; + struct_decl.body_start_pos = struct_body_start; + struct_decl.body_end_pos = struct_body_end; + struct_decls[struct_name] = struct_decl; + String struct_scope = "struct." + struct_name; + for (TokenE *tk = struct_body_start; tk != struct_body_end; tk = get_next_token()) { + if (!_process_decl_if_exist(struct_scope, false)) { + return false; + } + } + + reset_to(struct_body_end); + return true; +} + +// Past the start and type tokens, at the id or bracket open token. +bool ShaderDeprecatedConverter::_process_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, const String &p_scope, bool p_func_args) { + while (true) { + EOF_FAIL(p_start_tok); + EOF_FAIL(p_type_tok); + COND_LINE_MSG_FAIL(!token_is_type(p_type_tok->get()), p_type_tok->get().line, RTR("Expected type in declaration")); + TokenE *next_tk = get_pos(); + VarDecl var; + var.start_pos = p_start_tok; + var.type_pos = p_type_tok; + var.is_func_arg = p_func_args; + EOF_FAIL(next_tk); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + var.is_array = true; + var.new_arr_style_decl = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + COND_LINE_MSG_FAIL(!tokentype_is_identifier(next_tk->get().type), next_tk->get().line, RTR("Expected identifier after type in declaration")); + var.name_pos = next_tk; + String name = get_token_literal_text(var.name_pos->get()); + next_tk = get_next_token(); + EOF_FAIL(next_tk); + TokenE *end_pos = next_tk; + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + var.is_array = true; + if (!_skip_array_size()) { + return false; + } + end_pos = get_next_token(); + next_tk = end_pos; + EOF_FAIL(next_tk); + } + if (next_tk->get().type == TT::TK_OP_ASSIGN) { + end_pos = _get_end_of_closure(); + CLOSURE_FAIL(end_pos); + reset_to(end_pos); // Skip the assignment. + if (end_pos->get().type == TT::TK_PARENTHESIS_CLOSE && p_func_args) { + next_tk = end_pos; + end_pos = end_pos->prev(); // Including whitespace before parenthesis. + } else { + next_tk = get_next_token(); // comma or semi-colon + EOF_FAIL(next_tk); + end_pos = next_tk; + } + } + var.end_pos = end_pos; + COND_LINE_MSG_FAIL(p_func_args && !(next_tk->get().type == TT::TK_COMMA || next_tk->get().type == TT::TK_PARENTHESIS_CLOSE), next_tk->get().line, RTR("Expected ',' , or ')' after function argument declaration")); + COND_LINE_MSG_FAIL(!p_func_args && !(next_tk->get().type == TT::TK_SEMICOLON || next_tk->get().type == TT::TK_COMMA), next_tk->get().line, RTR("Expected ',' or ';' after declaration")); + if (!p_scope.begins_with("struct.")) { + if (var_decls.has(name)) { + var_decls[name].push_back(var); + } else { + var_decls[name] = { var }; + } + if (!scope_declarations.has(p_scope)) { + scope_declarations[p_scope] = HashSet(); + } + scope_declarations[p_scope].insert(name); + } else { + String struct_name = p_scope.substr(7); + struct_decls[struct_name].members[name] = var; + } + if (next_tk->get().type == TT::TK_COMMA) { + next_tk = get_next_token(); + EOF_FAIL(next_tk); + p_start_tok = next_tk; + if (p_func_args) { + while (next_tk->get().type == TT::TK_CONST || + SL::is_token_precision(next_tk->get().type) || + SL::is_token_arg_qual(next_tk->get().type) || + SL::is_token_interpolation(next_tk->get().type)) { + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + p_type_tok = next_tk; // next_tk is type + COND_LINE_MSG_FAIL(!token_is_type(p_type_tok->get()), p_type_tok->get().line, RTR("Expected type after comma in function argument declaration")); + next_tk = get_next_token(); // id + EOF_FAIL(next_tk); + } // otherwise, this is a compound declaration, leave type_tok as is + } else if (next_tk->get().type == TT::TK_PARENTHESIS_CLOSE) { + break; + } else if (next_tk->get().type == TT::TK_SEMICOLON) { + break; + } + } + return true; +}; + +// Past the start and type tokens, at the id or bracket open token. +bool ShaderDeprecatedConverter::_process_func_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, bool p_first_pass) { + FunctionDecl func; + func.start_pos = p_start_tok; // type or const + func.type_pos = p_type_tok; // type + TokenE *next_tk = get_pos(); // id or array size + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + func.has_array_return_type = true; + if (!_skip_array_size()) { + return false; + } + next_tk = get_next_token(); + EOF_FAIL(next_tk); + } + func.name_pos = next_tk; // id + String name = get_token_literal_text(func.name_pos->get()); + func.args_start_pos = get_next_token(); // paren + EOF_FAIL(func.args_start_pos); + if (peek_next_tk_type() == TT::TK_PARENTHESIS_CLOSE) { + func.args_end_pos = get_next_token(); + func.arg_count = 0; + } else { // Args are present. + func.args_end_pos = _get_end_of_closure(); + CLOSURE_FAIL(func.args_end_pos); + COND_LINE_MSG_FAIL(func.args_end_pos->get().type != TT::TK_PARENTHESIS_CLOSE, func.args_end_pos->get().line, RTR("Expected ')' after function arguments")); + if (!p_first_pass) { // first_pass == false means we've already parsed the args. + // Skip the args. + reset_to(func.args_end_pos); + } else { + TokenE *start_pos = get_next_token(); + TokenE *type_pos = start_pos; + while (type_pos->get().type == TT::TK_CONST || SL::is_token_precision(type_pos->get().type) || SL::is_token_arg_qual(type_pos->get().type) || SL::is_token_interpolation(type_pos->get().type)) { + type_pos = get_next_token(); + EOF_FAIL(type_pos); + } + COND_LINE_MSG_FAIL(!token_is_type(type_pos->get()), type_pos->get().line, RTR("Expected type in function argument declaration")); + get_next_token(); // id or open bracket + int var_count = var_decls.size(); + if (!_process_decl_statement(start_pos, type_pos, name, true)) { + return false; + } + COND_LINE_MSG_FAIL(get_pos() != func.args_end_pos, get_pos()->get().line, RTR("Expected ')' after function arguments")); + func.arg_count = var_decls.size() - var_count; + } + } + // Currently at paren close. + func.body_start_pos = get_next_token(); // Curly open. + EOF_FAIL(func.body_start_pos); + COND_LINE_MSG_FAIL(func.body_start_pos->get().type != TT::TK_CURLY_BRACKET_OPEN, func.body_start_pos->get().line, RTR("Expected '{' after function declaration")); + func.body_end_pos = _get_end_of_closure(); + CLOSURE_FAIL(func.body_end_pos); + COND_LINE_MSG_FAIL(func.body_end_pos->get().type != TT::TK_CURLY_BRACKET_CLOSE, func.body_start_pos->get().line, RTR("Expected '}' bracket")); + if (p_first_pass) { // p_first_pass == false means the functions have already been processed. + function_decls[name] = func; +#ifdef DEBUG_ENABLED + } else { + if (!function_decls.has(name)) { + LINE_MSG_FAIL(func.start_pos->get().line, vformat(RTR("Function declaration not found in third pass (%s)"), name)); + } else { + // Compare our values to ensure they match. + FunctionDecl &first_pass = function_decls[name]; + // Don't check arg count, as it's not set in the second pass. + bool matches = first_pass.start_pos == func.start_pos && first_pass.type_pos == func.type_pos && first_pass.name_pos == func.name_pos && first_pass.args_start_pos == func.args_start_pos && first_pass.args_end_pos == func.args_end_pos && first_pass.body_start_pos == func.body_start_pos && first_pass.body_end_pos == func.body_end_pos; + COND_LINE_MSG_FAIL(!matches, func.start_pos->get().line, vformat(RTR("Function declaration mismatch in third pass (%s)"), name)); + } +#endif + } + return true; +} + +bool ShaderDeprecatedConverter::_parse_decls(bool p_first_pass) { + reset_to(after_shader_decl); + String curr_func = ""; + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + + if (!p_first_pass) { + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.args_start_pos) { + curr_func = E.key; + } else if (cur_tok == func.body_end_pos) { + curr_func = ""; + } + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + if (cur_tok->get().type == TT::TK_UNIFORM) { + if (!_skip_uniform()) { + return false; + } + continue; + } + if (!_process_decl_if_exist(curr_func, p_first_pass)) { + return false; + } + } + return true; +} + +bool ShaderDeprecatedConverter::_process_decl_if_exist(String p_curr_func, bool p_first_pass) { + TokenE *cur_tok = get_pos(); + + TokenE *start_pos = cur_tok; + if (!_tok_is_start_of_decl(cur_tok->get())) { + return true; + } + while (_tok_is_start_of_decl(cur_tok->get())) { + if (token_is_type(cur_tok->get())) { + break; + } + cur_tok = get_next_token(); + EOF_FAIL(cur_tok); + } + COND_LINE_MSG_FAIL(!token_is_type(cur_tok->get()), cur_tok->get().line, RTR("Expected type in declaration")); + TokenE *type_pos = cur_tok; + + bool is_decl = tokentype_is_identifier(peek_next_tk_type()); + bool is_function = peek_next_tk_type(2) == TT::TK_PARENTHESIS_OPEN; + if (!is_decl) { + // Check if this is an array declaration. + TokenE *next_tk = get_next_token(); + if (next_tk->get().type == TT::TK_BRACKET_OPEN) { + if (!_skip_array_size()) { + return true; + } + next_tk = get_pos(); + EOF_FAIL(next_tk); + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_BRACKET_CLOSE, next_tk->get().line, RTR("Expected ']' after array type")); + TokenE *next_next_tk = get_next_token(); + if (next_next_tk && next_next_tk->get().type == TT::TK_IDENTIFIER) { + is_decl = true; + if (peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN) { + is_function = true; + } else { + is_function = false; + } + } + } + reset_to(cur_tok); // Backup to the Bracket open. + } + COND_LINE_MSG_FAIL(is_function && p_curr_func != "", cur_tok->get().line, RTR("Unexpected function declaration")); + if (!is_decl) { + return true; + } + TokenE *id_tok = get_next_token(); // Id or bracket open. + EOF_FAIL(id_tok); + if (is_function) { // Function declaration. + if (!_process_func_decl_statement(start_pos, type_pos, p_first_pass)) { + return false; + } + // Backup to before the curly bracket open. + get_prev_token(); + } else if (!p_first_pass) { // Other non-uniform declaration (global const, varying, locals, etc.). + if (!_process_decl_statement(start_pos, type_pos, p_curr_func)) { + return false; + } + } + return true; +} + +bool ShaderDeprecatedConverter::_parse_uniforms() { + reset_to(after_shader_decl); + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + switch (cur_tok->get().type) { + case TT::TK_UNIFORM: { + if (!_parse_uniform()) { + return false; + } + } break; + default: + break; + } + } + return true; +} + +bool ShaderDeprecatedConverter::_parse_structs() { + reset_to(after_shader_decl); + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + switch (cur_tok->get().type) { + case TT::TK_STRUCT: { + if (!_parse_struct()) { + return false; + } + } break; + default: + break; + } + } + return true; +} + +bool ShaderDeprecatedConverter::_preprocess_code() { + COND_MSG_FAIL(code_tokens.size() == 0, RTR("Empty shader file")); + StringName mode_string; + { + COND_MSG_FAIL(code_tokens.size() < 3, RTR("Invalid shader file")); + TokenE *first_token = get_next_token(); + EOF_FAIL(first_token); + COND_LINE_MSG_FAIL(first_token->get().type != TT::TK_SHADER_TYPE, first_token->get().line, RTR("Shader type must be first token")); + TokenE *id_token = get_next_token(); + EOF_FAIL(id_token); + COND_LINE_MSG_FAIL(id_token->get().type != TT::TK_IDENTIFIER, id_token->get().line, RTR("Invalid shader type")); + mode_string = id_token->get().text; + TokenE *token = get_next_token(); + EOF_FAIL(token); + COND_LINE_MSG_FAIL(token->get().type != TT::TK_SEMICOLON, token->get().line, RTR("Expected semi-colon after shader type")); + shader_mode = get_shader_mode_from_string(mode_string); + } + after_shader_decl = get_pos(); + + /*** + * The first pass gets the uniform declarations; we require this is to ensure idempotency for inserting new uniforms and replacing type hints. + * The second pass gets the function declarations; these are used for determining if a renamed built-in is valid in the current scope. + * The third pass gets the variable declarations; These are used for determining if renamed built-ins have been previously declared, and for detecting new keywords used as identifiers. + */ + // first pass, get uniform declarations. + if (!_parse_uniforms()) { + err_str = vformat(RTR("First pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + if (!_parse_structs()) { + err_str = vformat(RTR("First pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + // Second pass, get function declarations. + if (!_parse_decls(true)) { + function_pass_failed = true; + err_str = vformat(RTR("Second pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + // Third pass, get variable declarations. + if (!_parse_decls(false)) { + var_pass_failed = true; + err_str = vformat(RTR("Third pre-process pass failed: %s"), err_str); + curr_ptr = code_tokens.front(); + return false; + } + curr_ptr = code_tokens.front(); + return true; +} + +int ShaderDeprecatedConverter::get_error_line() const { + return err_line; +} + +bool ShaderDeprecatedConverter::is_code_deprecated(const String &p_code) { + // Quick check to see if it's a shader file with a deprecated type. + String mode_str = SL::get_shader_type(p_code); + if (mode_str.is_empty()) { + // If it failed, it's because it was prefixed with a preproc directive (4.x only) or it's not a shader file. + return false; + } + RS::ShaderMode mode = get_shader_mode_from_string(mode_str); + if (mode == RS::SHADER_MAX) { + return false; + } + old_code = p_code; + reset(); + if (_has_any_preprocessor_directives()) { + return false; + } + if (!_preprocess_code()) { + return false; + } + return _is_code_deprecated(); +} + +bool ShaderDeprecatedConverter::_has_any_preprocessor_directives() { + TokenE *cur_tok = code_tokens.front(); + while (cur_tok) { + if (cur_tok->get().type == TT::TK_PREPROC_DIRECTIVE) { + return true; + } + cur_tok = cur_tok->next(); + } + return false; +} + +bool ShaderDeprecatedConverter::_is_code_deprecated() { + reset_to(after_shader_decl); + + // Negative cases first, then positive cases. + bool is_3x = false; + + // Check declarations for negative cases. + for (const KeyValue &E : uniform_decls) { + const UniformDecl &uni = E.value; + if (uni.is_array) { // 3.x did not have array uniforms. + return false; + } else if (tokentype_is_new_type(uni.type_pos->get().type)) { // Usage of new type. + return false; + } else if (uni.has_uniform_qual()) { // 3.x did not have uniform qualifiers. + return false; + } + for (const TokenE *hint : uni.hint_poses) { + if (tokentype_is_new_hint(hint->get().type)) { // Usage of new hint. + return false; + } + } + } + + for (const KeyValue &E : function_decls) { + const FunctionDecl &func = E.value; + if (func.has_array_return_type) { // 3.x did not have array return types. + return false; + } else if (tokentype_is_new_type(func.type_pos->get().type) && !struct_decls.has(get_tokentype_text(func.type_pos->get().type))) { // Usage of new type. + return false; + } else if (func.is_new_main_function(shader_mode)) { // Has the process function with the same signature. + return false; + } + } + + for (const KeyValue &E : struct_decls) { + const StructDecl &str = E.value; + for (const KeyValue &v : str.members) { + if (tokentype_is_new_type(v.value.type_pos->get().type) && !struct_decls.has(get_tokentype_text(v.value.type_pos->get().type))) { // Usage of new type. + return false; + } + } + } + + for (const KeyValue> &E : var_decls) { + for (const VarDecl &var_decl : E.value) { + if (var_decl.is_array && var_decl.is_func_arg) { // 3.x did not allow array function arguments. + return false; + } else if (var_decl.new_arr_style_decl) { // 3.x did not have the `float[] x` style of array declarations for non-struct members. + return false; + } else if (tokentype_is_new_type(var_decl.type_pos->get().type) && !struct_decls.has(get_tokentype_text(var_decl.type_pos->get().type))) { // Usage of new type. + return false; + } + } + } + + // Check token stream for negative cases. + { + reset_to(after_shader_decl); + String curr_func = ""; + while (true) { + TokenE *cur_tok = get_next_token(); + DEV_ASSERT(cur_tok); + if (cur_tok->get().type == TT::TK_EOF || cur_tok->get().type == TT::TK_ERROR) { + break; + } + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.args_start_pos) { + curr_func = E.key; + break; + } else if (cur_tok == func.body_end_pos) { + curr_func = ""; + break; + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + String id = get_token_literal_text(cur_tok->get()); + if (cur_tok->get().type == TT::TK_IDENTIFIER) { + if (has_builtin_rename(shader_mode, id, curr_func) || is_removed_builtin(shader_mode, id, curr_func)) { + if (scope_has_decl(curr_func, id) || function_decls.has(id)) { + // The renamed built-ins are global identifiers in 3.x and can't be redefined in either the global scope or the function scope they're valid for. + // If they were declared previously within the global or current scope, this would be a 4.x shader. + return false; + } + } else if (id_is_new_builtin_func(id) && peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN && !function_decls.has(id)) { // Use of a new built-in function without a corresponding declaration. + return false; + } + } + } + } + + // Positive cases. + + // Check declarations for positive cases. + for (const KeyValue &E : uniform_decls) { + const UniformDecl &uni = E.value; + if (uni.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(uni.type_pos->get()))) { // Unported 3.x type. + return true; + } else if (tokentype_is_new_reserved_keyword(uni.name_pos->get().type)) { // Uniform name is a new reserved keyword. + return true; + } else if (uni.has_interp_qual()) { // Newer versions of Godot disallow interpolation qualifiers for uniforms. + return true; + } + for (const TokenE *hint : uni.hint_poses) { + if (hint->get().type == TT::TK_IDENTIFIER && has_hint_replacement(get_token_literal_text(hint->get()))) { + return true; + } + } + } + + for (const KeyValue &E : struct_decls) { + const StructDecl &struct_decl = E.value; + if (tokentype_is_new_reserved_keyword(struct_decl.name_pos->get().type)) { // Struct identifier is new reserved keyword. + return true; + } + for (const KeyValue &v : struct_decl.members) { + if (v.value.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(v.value.type_pos->get()))) { // Unported 3.x type. + return true; + } else if (tokentype_is_new_reserved_keyword(v.value.name_pos->get().type)) { // Struct member identifier is new reserved keyword. + return true; + } + } + } + + for (const KeyValue &E : function_decls) { + const FunctionDecl &func = E.value; + String name = get_token_literal_text(func.name_pos->get()); + String type_name = get_token_literal_text(func.type_pos->get()); + if (func.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(type_name)) { // Unported 3.x type. + return true; + } else if (func.is_renamed_main_function(shader_mode)) { // Matching renamed function. + return true; + } else if (tokentype_is_new_reserved_keyword(func.name_pos->get().type)) { // Function identifier is new reserved keyword. + return true; + } else if (id_is_new_builtin_func(name)) { // Declaration of function with the same name as a new built-in function. + return true; + } + } + + for (const KeyValue> &E : var_decls) { + for (const VarDecl &var_decl : E.value) { + if (var_decl.type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(var_decl.type_pos->get()))) { // Unported 3.x type. + return true; + } else if (tokentype_is_new_reserved_keyword(var_decl.name_pos->get().type)) { // Id is new reserved keyword. + return true; + } + } + } + + String curr_func = ""; + reset_to(after_shader_decl); + // Check token stream for positive cases. + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF || cur_tok->get().type == TT::TK_ERROR) { + break; + } + + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.body_start_pos) { + curr_func = E.key; + break; + } else if (cur_tok == func.body_end_pos) { + curr_func = ""; + break; + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + + switch (cur_tok->get().type) { + case TT::TK_FLOAT_CONSTANT: { + String const_str = get_token_literal_text(cur_tok->get()).to_lower(); + // 3.x float constants allowed for a value without a decimal point if it ended in `f` (e.g. `1f`). + if (const_str.ends_with("f") && const_str.find(".") == -1 && const_str.find("e") == -1) { + return true; + } + } break; + case TT::TK_RENDER_MODE: { + while (true) { + TokenE *next_tk = get_next_token(); + if (next_tk->get().type == TT::TK_IDENTIFIER) { + String id_text = get_token_literal_text(next_tk->get()); + if (is_renamed_render_mode(shader_mode, id_text) || has_removed_render_mode(shader_mode, id_text)) { + return true; + } + } else { + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_COMMA && next_tk->get().type != TT::TK_SEMICOLON, next_tk->get().line, "Invalid render mode declaration"); + } + if (next_tk->get().type == TT::TK_SEMICOLON) { + break; + } + } + } break; + case TT::TK_IDENTIFIER: { + String id = get_token_literal_text(cur_tok->get()); + if (has_builtin_rename(shader_mode, id, curr_func) || is_removed_builtin(shader_mode, id, curr_func)) { + if ((!scope_has_decl(curr_func, id) && !function_decls.has(id))) { + return true; + } + } else if (has_removed_type(id) && peek_next_tk_type() == TT::TK_IDENTIFIER) { + // Declaration with unported 3.x type. + return true; + } + } break; + default: + break; + } + } + return is_3x; +} + +String ShaderDeprecatedConverter::get_error_text() const { + return err_str; +} + +bool ShaderDeprecatedConverter::_check_deprecated_type(TokenE *p_type_pos) { + if (p_type_pos->get().type == TT::TK_IDENTIFIER && has_removed_type(get_token_literal_text(p_type_pos->get()))) { + const String i_err_msg = vformat(RTR("Deprecated type '%s' is not supported by this version of Godot."), get_token_literal_text(p_type_pos->get())); + COND_LINE_MSG_FAIL(fail_on_unported, p_type_pos->get().line, i_err_msg); + _add_comment_before(i_err_msg, p_type_pos); + } + return true; +} + +ShaderDeprecatedConverter::TokenE *ShaderDeprecatedConverter::_rename_keyword_id(TokenE *pos, bool p_detected_3x, const HashMap &p_new_reserved_word_renames) { + String rename = p_new_reserved_word_renames[pos->get().type]; + reset_to(pos); + return replace_curr(mkTok(TT::TK_IDENTIFIER, rename), "Identifier '%s' is a reserved word in this version of Godot, renamed to '%s'."); +} + +bool ShaderDeprecatedConverter::_handle_new_keyword_rename(TokenE *pos, bool p_detected_3x, HashMap &p_new_reserved_word_renames) { + TokenType tk_type = pos->get().type; + String p_name = get_token_literal_text(pos->get()); + if (tokentype_is_new_reserved_keyword(tk_type)) { + if (!p_detected_3x) { + // If we're not sure it's a 3.x shader, just add a comment. + _add_comment_before(vformat(RTR("Identifier '%s' is a reserved word in this version of Godot."), p_name), pos); + return false; + } + if (!p_new_reserved_word_renames.has(tk_type)) { + String rename = p_name + String("_"); + while (function_decls.has(rename) || uniform_decls.has(rename) || var_decls.has(rename) || struct_decls.has(rename)) { + rename += "_"; + } + p_new_reserved_word_renames[tk_type] = rename; + } + return true; + } + return false; +} + +bool ShaderDeprecatedConverter::convert_code(const String &p_code) { + /** + * We need to do the following: + * * Replace everything in RenamesMap3To4::shaders_renames + * * the usage of SCREEN_TEXTURE, DEPTH_TEXTURE, and NORMAL_ROUGHNESS_TEXTURE necessitates adding a uniform declaration at the top of the file + * * async_visible and async_hidden render modes need to be removed + * * If shader_type is "particles", need to rename the function "void vertex()" to "void process()" + * * Invert all usages of CLEARCOAT_GLOSS: + * * Invert all lefthand assignments: + * - `CLEARCOAT_GLOSS = 5.0 / foo;` + * becomes: `CLEARCOAT_ROUGHNESS = (1.0 - (5.0 / foo));`, + * - `CLEARCOAT_GLOSS *= 1.1;` + * becomes `CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 1.1));` + * * Invert all righthand usages + * - `foo = CLEARCOAT_GLOSS;` + * becomes: `foo = (1.0 - CLEARCOAT_ROUGHNESS);` + * * Wrap `INDEX` in `int()` casts if necessary. + * * Check for use of `specular_blinn` and `specular_phong` render modes; not supported in 4.x, throw an error. + * * Check for use of `MODULATE`; not supported in 4.x, throw an error. + * * Check for use of new reserved keywords as identifiers; rename them if necessary. + * * Check for use of new built-in functions with a corresponding declaration; rename them if necessary. + */ + old_code = p_code; + reset(); + if (_has_any_preprocessor_directives()) { // We refuse to process any shader with preprocessor directives, as they're not supported in 3.x and they make our parsing assumptions invalid. + err_str = RTR("Cannot convert new shader with pre-processor directives."); + return false; + } + if (!_preprocess_code()) { + return false; + } + bool detected_3x = _is_code_deprecated(); // Calls preprocess_code(). + if (!detected_3x && err_str != "") { + return false; + } // We don't fail if the code is detected as not deprecated, as the user may have forced it; instead, we just avoid doing the more dicey replacements, like renaming new keywords. + COND_MSG_FAIL(shader_mode == RS::SHADER_MAX, RTR("Detected Shader type is not a 3.x type.")); // However, we do fail if it's a new shader type, because we don't do any replacements for those. + err_str = ""; + curr_ptr = after_shader_decl; + + // Renaming changed hints. + Vector all_hints; + for (KeyValue &E : uniform_decls) { + UniformDecl &uni = E.value; + if (uni.has_interp_qual()) { // Removing interpolation qualifiers before the type name, which was allowed in 3.x. + reset_to(uni.interp_qual_pos); + _add_comment_before(vformat(RTR("Interpolation qualifiers not supported in this version of Godot, '%s' removed."), get_token_literal_text(uni.interp_qual_pos->get())), uni.start_pos, false); + remove_cur_and_get_next(); + uni.interp_qual_pos = nullptr; + reset_to(after_shader_decl); + } + String name = get_token_literal_text(uni.name_pos->get()); + for (int i = 0; i < uni.hint_poses.size(); i++) { + TokenE *hint = uni.hint_poses[i]; + String hint_name = get_token_literal_text(hint->get()); + if (hint->get().type == TT::TK_IDENTIFIER && has_hint_replacement(hint_name)) { + // replace the hint + reset_to(hint); + TT hint_rename = get_hint_replacement(hint_name); + hint = replace_curr(mkTok(hint_rename), "Hint '%s' renamed to '%s'."); + uni.hint_poses.write[i] = hint; + reset_to(after_shader_decl); + } + all_hints.push_back(hint); + } + } + + // Renaming new reserved keywords used as identifiers (e.g "global", "instance"). + // To ensure idempotency, we only do this if we know for certain that the new keyword was used in a declaration. + HashMap new_reserved_word_renames; + HashMap func_renames; + HashMap struct_renames; + HashMap struct_member_renames; + HashMap nonfunc_globals_renames; // Only used if a function is renamed and an existing global conflicts with the rename. + + for (KeyValue &E : uniform_decls) { + UniformDecl &uni = E.value; + if (!_check_deprecated_type(uni.type_pos)) { + return false; + } + + if (_handle_new_keyword_rename(uni.name_pos, detected_3x, new_reserved_word_renames)) { + uni.name_pos = _rename_keyword_id(uni.name_pos, detected_3x, new_reserved_word_renames); + reset_to(after_shader_decl); + } + } + + for (KeyValue &E : struct_decls) { + StructDecl &struct_decl = E.value; + if (_handle_new_keyword_rename(struct_decl.name_pos, detected_3x, new_reserved_word_renames)) { + struct_decl.name_pos = _rename_keyword_id(struct_decl.name_pos, detected_3x, new_reserved_word_renames); + struct_renames[E.key] = get_token_literal_text(struct_decl.name_pos->get()); + reset_to(after_shader_decl); + } + for (KeyValue &M : struct_decl.members) { + VarDecl &var = M.value; + if (!_check_deprecated_type(var.type_pos)) { + return false; + } + String type = get_token_literal_text(var.type_pos->get()); + if (_handle_new_keyword_rename(var.name_pos, detected_3x, new_reserved_word_renames)) { + var.name_pos = _rename_keyword_id(var.name_pos, detected_3x, new_reserved_word_renames); + struct_member_renames[M.key] = get_token_literal_text(var.name_pos->get()); + reset_to(after_shader_decl); + } + if (struct_renames.has(type)) { + reset_to(var.type_pos); + var.type_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, struct_renames[type]), "Struct type '%s' renamed to '%s'."); + reset_to(after_shader_decl); + } + } + } + for (KeyValue> &E : var_decls) { + if (E.value.is_empty()) { + continue; + } + // Check for deprecated type. + for (VarDecl &var : E.value) { + if (!_check_deprecated_type(var.type_pos)) { + return false; + } + String type = get_token_literal_text(var.type_pos->get()); + if (struct_renames.has(type)) { + reset_to(var.type_pos); + var.type_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, struct_renames[type]), "Struct type '%s' renamed to '%s'."); + reset_to(after_shader_decl); + } + } + + if (_handle_new_keyword_rename(E.value[0].name_pos, detected_3x, new_reserved_word_renames)) { + for (VarDecl &varDecl : E.value) { + // replace the identifier + reset_to(varDecl.name_pos); + if (varDecl.name_pos == varDecl.start_pos) { + varDecl.name_pos = _rename_keyword_id(varDecl.name_pos, detected_3x, new_reserved_word_renames); + varDecl.start_pos = varDecl.name_pos; + } else { + varDecl.name_pos = _rename_keyword_id(varDecl.name_pos, detected_3x, new_reserved_word_renames); + } + reset_to(after_shader_decl); + } + } + } + bool has_new_main_function = false; + Vector new_main_function_names; + for (KeyValue &E : function_decls) { + if (E.value.is_new_main_function(shader_mode)) { + has_new_main_function = true; + break; + } + } + static const char *conflict_comment_prefix = "Renamed %s to avoid conflict with new main function, %s."; + for (KeyValue &E : function_decls) { + FunctionDecl &var = E.value; + if (!_check_deprecated_type(var.type_pos)) { + return false; + } + String type = get_token_literal_text(var.type_pos->get()); + if (struct_renames.has(type)) { + reset_to(var.type_pos); + var.type_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, struct_renames[type]), "Struct type '%s' renamed to '%s'."); + reset_to(after_shader_decl); + } + String name = get_token_literal_text(var.name_pos->get()); + if (!has_new_main_function && var.is_renamed_main_function(shader_mode)) { + // replace the function name + String rename = get_main_function_rename(name); + reset_to(var.name_pos); + var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rename), "Function '%s' renamed to '%s' in this version of Godot."); + reset_to(after_shader_decl); + func_renames[name] = rename; + // Only doing this because "process" is a common word and we don't want to clobber an existing function/global named that. + // Won't clobber a pre-exising "process" function that has the correct main signature because of the check before. + if (function_decls.has(rename) || scope_has_decl("", rename)) { + String rerename = rename + String("_"); + while (function_decls.has(rerename) || uniform_decls.has(rerename) || var_decls.has(rerename)) { + rerename += "_"; + } + if (function_decls.has(rename)) { + func_renames[rename] = rerename; + FunctionDecl &rere_func = function_decls[rename]; + reset_to(rere_func.name_pos); + rere_func.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rerename), conflict_comment_prefix); + reset_to(after_shader_decl); + } else if (uniform_decls.has(rename)) { + nonfunc_globals_renames[rename] = rerename; + UniformDecl &rere_uni = uniform_decls[rename]; + reset_to(rere_uni.name_pos); + rere_uni.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rerename), conflict_comment_prefix); + reset_to(after_shader_decl); + } else if (var_decls.has(rename) && scope_declarations.has("") && scope_declarations[""].has(rename)) { + nonfunc_globals_renames[rename] = rerename; + for (int i = 0; i < var_decls[rename].size(); i++) { + VarDecl &rere_var = var_decls[rename].write[i]; + reset_to(rere_var.name_pos); + rere_var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rerename), conflict_comment_prefix); + reset_to(after_shader_decl); + } + } + } + } else if (id_is_new_builtin_func(name)) { + String rename = name + String("_"); + while (function_decls.has(rename) || uniform_decls.has(rename) || var_decls.has(rename)) { + rename += "_"; + } + func_renames[name] = rename; + // replace the function name + reset_to(var.name_pos); + var.name_pos = replace_curr(mkTok(TT::TK_IDENTIFIER, rename), "Function '%s' is a built-in function in this version of Godot, renamed to '%s'."); + reset_to(after_shader_decl); + } else if (_handle_new_keyword_rename(var.name_pos, detected_3x, new_reserved_word_renames)) { + var.name_pos = _rename_keyword_id(var.name_pos, detected_3x, new_reserved_word_renames); + reset_to(after_shader_decl); + } + } + bool in_function = false; + String curr_func = ""; + reset_to(after_shader_decl); + static Vector uniform_qualifiers = { "global", "instance" }; + while (true) { + TokenE *cur_tok = get_next_token(); + if (cur_tok->get().type == TT::TK_EOF) { + break; + } + for (KeyValue &E : function_decls) { + FunctionDecl &func = E.value; + if (cur_tok == func.args_start_pos) { + in_function = true; + curr_func = E.key; // The key is the ORIGINAL function name, not the potentially renamed one. + } else if (in_function && cur_tok == func.body_end_pos) { + in_function = false; + curr_func = ""; + } + } + if (cur_tok->get().type == TT::TK_STRUCT) { + if (!_skip_struct()) { + return false; + } + continue; + } + String cur_tok_text = get_token_literal_text(cur_tok->get()); + if (cur_tok->get().pos != NEW_IDENT && new_reserved_word_renames.has(cur_tok->get().type)) { + if (peek_prev_tk_type() == TT::TK_PERIOD && struct_member_renames.has(cur_tok_text)) { + cur_tok = _rename_keyword_id(cur_tok, detected_3x, new_reserved_word_renames); + continue; + } + if (!(scope_has_decl(curr_func, cur_tok_text) || (function_decls.has(cur_tok_text) && peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN))) { + continue; // Don't replace if this rename is not in the current scope or is not a renamed function call. + } + // Just extra insurance against replacing legit new keywords. + if (uniform_qualifiers.has(cur_tok_text)) { + if (peek_next_tk_type() == TT::TK_UNIFORM) { + continue; // Don't replace uniform qualifiers. + } + } else if (all_hints.has(cur_tok)) { + continue; // Hint, don't replace it. + } + cur_tok = _rename_keyword_id(cur_tok, detected_3x, new_reserved_word_renames); + continue; + } + switch (cur_tok->get().type) { + case TT::TK_FLOAT_CONSTANT: { + // Earlier versions of Godot 3.x (< 3.5) allowed the use of the `f` sigil with float constants without a decimal place. + if (cur_tok_text.ends_with("f") && !cur_tok_text.contains(".") && !cur_tok_text.contains("e")) { + cur_tok_text = cur_tok_text.substr(0, cur_tok_text.length() - 1) + ".0f"; + _add_comment_before(RTR("Float constant without decimal point requires a trailing '.0' in this version of Godot."), cur_tok, false); + cur_tok = replace_curr(mkTok(TT::TK_FLOAT_CONSTANT, cur_tok_text, 0xdeadbeef)); + } + } break; + case TT::TK_RENDER_MODE: { + // we only care about the ones for spatial + if (shader_mode == RenderingServer::ShaderMode::SHADER_SPATIAL) { + while (true) { + TokenE *next_tk = get_next_token(); + if (next_tk->get().type == TT::TK_IDENTIFIER) { + String id_text = get_token_literal_text(next_tk->get()); + if (has_removed_render_mode(shader_mode, id_text)) { + String comment = "Deprecated render mode '%s' is not supported by this version of Godot."; + if (!can_remove_render_mode(id_text)) { + const String i_err_msg = vformat(RTR(comment), id_text); + COND_LINE_MSG_FAIL(fail_on_unported, next_tk->get().line, i_err_msg); + _add_comment_before(i_err_msg, next_tk); + } else { + comment = comment.substr(0, comment.length() - 1) + ", removed."; + _add_comment_before(vformat(RTR(comment), id_text), next_tk, true); + if (peek_next_tk_type() == TT::TK_COMMA) { + TokenE *comma = get_next_token(); + reset_to(next_tk); // Reset to the identifier. + EOF_FAIL(comma->next()); + next_tk = _remove_from_curr_to(comma->next()); // Inclusive of comma. + } else if (peek_prev_tk_type() == TT::TK_COMMA && peek_next_tk_type() == TT::TK_SEMICOLON) { + TokenE *end = get_next_token(); + reset_to(next_tk); // Back to identifier. + next_tk = get_prev_token(); // comma + next_tk = _remove_from_curr_to(end); // Exclusive of semi-colon. + break; // We're at the end of the render_mode declaration. + } else if (peek_prev_tk_type() == TT::TK_RENDER_MODE && peek_next_tk_type() == TT::TK_SEMICOLON) { + // Remove the whole line. + TokenE *semi = get_next_token(); + COND_LINE_MSG_FAIL(!semi->next(), semi->get().line, "Unexpected EOF???"); // We should always have an EOF token at the end of the stream. + reset_to(next_tk); // Back to identifier. + next_tk = get_prev_token(); // render_mode + next_tk = _remove_from_curr_to(semi->next()); // Inclusive of semi-colon. + break; + } else { + // we shouldn't be here + LINE_MSG_FAIL(next_tk->get().line, RTR("Unexpected token after render mode declaration.")); + } + } + } else if (is_renamed_render_mode(shader_mode, id_text)) { + next_tk = replace_curr(mkTok(TT::TK_IDENTIFIER, get_render_mode_rename(id_text)), "Render mode "); + } + } else { + COND_LINE_MSG_FAIL(next_tk->get().type != TT::TK_COMMA && next_tk->get().type != TT::TK_SEMICOLON, next_tk->get().line, RTR("Expected ',' or ';' after render mode declaration.")); + } + if (next_tk->get().type == TT::TK_SEMICOLON) { + break; + } + } + } + } break; + case TT::TK_IDENTIFIER: { + if (cur_tok->get().pos == NEW_IDENT) { // Skip already-replaced identifiers. + break; + } + if (peek_prev_tk_type() == TT::TK_PERIOD) { + break; // Struct member access, don't replace it. + } + if (func_renames.has(cur_tok_text) && peek_next_tk_type() == TT::TK_PARENTHESIS_OPEN) { // Function call. + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, func_renames[cur_tok_text]), "Function call "); + } else if (nonfunc_globals_renames.has(cur_tok_text) && peek_next_tk_type() != TT::TK_PARENTHESIS_OPEN) { + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, nonfunc_globals_renames[cur_tok_text]), conflict_comment_prefix); + } else if (is_removed_builtin(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + if (get_removed_builtin_uniform_type(cur_tok_text) == TT::TK_ERROR) { + const String i_err_str = vformat(RTR("Deprecated built-in '%s' is not supported by this version of Godot"), cur_tok_text); + COND_LINE_MSG_FAIL(fail_on_unported, cur_tok->get().line, i_err_str); + _add_comment_before(i_err_str, cur_tok); + } + COND_LINE_MSG_FAIL(!_insert_uniform_declaration(cur_tok_text), cur_tok->get().line, RTR("Failed to insert uniform declaration")); + } else if (cur_tok_text == "INDEX" && has_builtin_rename(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + // INDEX was an int in 3.x, but is a uint in later versions. + // Need to wrap it in a `int()` cast. + // This is safe because this will only trigger if the `particles` function is "vertex" (which is renamed to "process"). + + // Don't do this if it's already wrapped in a int(), uint() or float(). + if (peek_prev_tk_type() == TT::TK_PARENTHESIS_OPEN && peek_next_tk_type() == TT::TK_PARENTHESIS_CLOSE) { + TT peeked_type = peek_prev_tk_type(2); + if (peeked_type == TT::TK_TYPE_INT || peeked_type == TT::TK_TYPE_UINT || peeked_type == TT::TK_TYPE_FLOAT) { + break; + } + } + _add_comment_before(vformat(RTR("INDEX is uint in this version of Godot, wrapped INDEX in 'int()' cast.")), cur_tok, false); + insert_before({ mkTok(TT::TK_TYPE_INT), mkTok(TT::TK_PARENTHESIS_OPEN) }, cur_tok); + insert_after(mkTok(TT::TK_PARENTHESIS_CLOSE), cur_tok); + } else if (cur_tok_text == "CLEARCOAT_GLOSS" && has_builtin_rename(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, "CLEARCOAT_ROUGHNESS"), "Inverting usage of '%s' to '%s'."); + List::Element *assign_closure_end = nullptr; + switch (peek_next_tk_type()) { + case TT::TK_OP_ASSIGN: + case TT::TK_OP_ASSIGN_ADD: + case TT::TK_OP_ASSIGN_SUB: + case TT::TK_OP_ASSIGN_MUL: + case TT::TK_OP_ASSIGN_DIV: { + assign_closure_end = _get_end_of_closure(); + CLOSURE_FAIL(assign_closure_end); + + TokenE *assign_tk = get_next_token(); + TokenE *insert_pos = assign_tk; + if (assign_tk->next() && assign_tk->next()->get().type == TT::TK_SPACE) { + insert_pos = assign_tk->next(); + } + // " = (1.0 - (" + Vector assign_prefix = { + mkTok(TT::TK_OP_ASSIGN), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_PARENTHESIS_OPEN), + mkTok(TT::TK_FLOAT_CONSTANT, {}, 1.0), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_OP_SUB), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_PARENTHESIS_OPEN), + }; + if (assign_tk->get().type != TT::TK_OP_ASSIGN) { + // " = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) {op} + assign_prefix.append_array( + { mkTok(TT::TK_PARENTHESIS_OPEN), + mkTok(TT::TK_FLOAT_CONSTANT, {}, 1.0), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_OP_SUB), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_IDENTIFIER, "CLEARCOAT_ROUGHNESS"), + mkTok(TT::TK_PARENTHESIS_CLOSE), + mkTok(TT::TK_SPACE) }); + } + switch (assign_tk->get().type) { + case TT::TK_OP_ASSIGN_ADD: { + assign_prefix.append_array({ mkTok(TT::TK_OP_ADD), mkTok(TT::TK_SPACE) }); + } break; + case TT::TK_OP_ASSIGN_SUB: { + assign_prefix.append_array({ mkTok(TT::TK_OP_SUB), mkTok(TT::TK_SPACE) }); + } break; + case TT::TK_OP_ASSIGN_MUL: { + assign_prefix.append_array({ mkTok(TT::TK_OP_MUL), mkTok(TT::TK_SPACE) }); + } break; + case TT::TK_OP_ASSIGN_DIV: { + assign_prefix.append_array({ mkTok(TT::TK_OP_DIV), mkTok(TT::TK_SPACE) }); + } break; + default: + break; + } + insert_after(assign_prefix, insert_pos); + + // remove the assignment token + if (assign_tk != insert_pos && insert_pos->next()) { + // remove the extraneous space too if necessary + _remove_from_curr_to(insert_pos->next()); // Exclusive of the token after the space + } else { + remove_cur_and_get_next(); + } + // "))" + insert_after({ mkTok(TT::TK_PARENTHESIS_CLOSE), mkTok(TT::TK_PARENTHESIS_CLOSE) }, assign_closure_end); + reset_to(cur_tok); + + } break; + + default: + break; + } + + // Check for right-hand usage: if previous token is anything but a `{`, `}` or `;`. + if (peek_prev_tk_type() == TT::TK_SEMICOLON || + peek_prev_tk_type() == TT::TK_CURLY_BRACKET_OPEN || + peek_prev_tk_type() == TT::TK_CURLY_BRACKET_CLOSE) { + break; + } + + // Invert right-hand usage. + Vector right_hand_prefix = { // "(1.0 - ("; + mkTok(TT::TK_PARENTHESIS_OPEN), + mkTok(TT::TK_FLOAT_CONSTANT, {}, 1.0), + mkTok(TT::TK_SPACE), + mkTok(TT::TK_OP_SUB), + mkTok(TT::TK_SPACE) + }; + if (assign_closure_end) { + right_hand_prefix.append_array({ mkTok(TT::TK_PARENTHESIS_OPEN) }); + insert_after({ mkTok(TT::TK_PARENTHESIS_CLOSE), mkTok(TT::TK_PARENTHESIS_CLOSE) }, assign_closure_end); + } else { + insert_after(mkTok(TT::TK_PARENTHESIS_CLOSE), cur_tok); + } + insert_before(right_hand_prefix, cur_tok); + } else if (has_builtin_rename(shader_mode, cur_tok_text, curr_func) && !scope_has_decl(curr_func, cur_tok_text)) { + String rename = get_builtin_rename(cur_tok_text); + cur_tok = replace_curr(mkTok(TT::TK_IDENTIFIER, rename), "Built-in '%s' is renamed to '%s'."); + } + } break; // End of identifier case. + case TT::TK_ERROR: { + LINE_MSG_FAIL(cur_tok->get().line, "Parser error ( " + cur_tok->get().text + ")"); + } break; + default: + break; + } + } + return true; +} + +String ShaderDeprecatedConverter::emit_code() const { + if (code_tokens.size() == 0) { + return ""; + } + String new_code = ""; + const TokenE *start = code_tokens.front()->next(); // skip TK_EOF token at start + for (const TokenE *E = start; E; E = E->next()) { + const Token &tk = E->get(); + ERR_FAIL_COND_V(tk.type < 0 || tk.type > TT::TK_MAX, ""); + bool end = false; + String tok_text = get_token_literal_text(tk); + switch (tk.type) { + case TT::TK_ERROR: + case TT::TK_EOF: { + end = true; + } break; + case TT::TK_BLOCK_COMMENT: { + if (tk.pos == NEW_IDENT) { + if (warning_comments && tok_text.contains("!convert WARNING:")) { + new_code += tok_text; + } else if (verbose_comments && tok_text.contains("!convert")) { + new_code += tok_text; + } else { + break; + } + } else { + new_code += tok_text; + } + } break; + default: { + new_code += tok_text; + } break; + } + if (end) { + break; + } + } + + return new_code; +} + +void ShaderDeprecatedConverter::set_warning_comments(bool p_add_comments) { + warning_comments = p_add_comments; +} +void ShaderDeprecatedConverter::set_fail_on_unported(bool p_fail_on_unported) { + fail_on_unported = p_fail_on_unported; +} +void ShaderDeprecatedConverter::set_verbose_comments(bool p_verbose_comments) { + verbose_comments = p_verbose_comments; +} + +#endif // DISABLE_DEPRECATED diff --git a/servers/rendering/shader_converter.h b/servers/rendering/shader_converter.h new file mode 100644 index 000000000000..b32eba02ed82 --- /dev/null +++ b/servers/rendering/shader_converter.h @@ -0,0 +1,310 @@ +/**************************************************************************/ +/* shader_converter.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef SHADER_CONVERTER_H +#define SHADER_CONVERTER_H + +#ifndef DISABLE_DEPRECATED + +#include "core/templates/pair.h" +#include "servers/rendering/shader_language.h" +#include "servers/rendering_server.h" + +class ShaderDeprecatedConverter { +public: + using TokenType = ShaderLanguage::TokenType; + using Token = ShaderLanguage::Token; + using TT = TokenType; + using TokenE = List::Element; + + ShaderDeprecatedConverter() {}; + bool is_code_deprecated(const String &p_code); + bool convert_code(const String &p_code); + String get_error_text() const; + int get_error_line() const; + String emit_code() const; + void set_warning_comments(bool p_add_comments); + void set_fail_on_unported(bool p_fail_on_unported); + void set_assume_correct(bool p_assume_correct); + void set_force_reserved_word_replacement(bool p_force_reserved_word_replacement); + void set_verbose_comments(bool p_verbose_comments); + String get_report(); + + static bool tokentype_is_identifier(const TokenType &p_tk_type); + static bool tokentype_is_new_reserved_keyword(const TokenType &p_tk_type); + static bool tokentype_is_new_type(const TokenType &p_tk_type); + static bool tokentype_is_new_hint(const TokenType &p_tk); + + static bool id_is_new_builtin_func(const String &p_name); + + static String get_tokentype_text(TokenType p_tk_type); + static TokenType get_tokentype_from_text(const String &p_text); + + static bool has_builtin_rename(RS::ShaderMode p_mode, const String &p_name, const String &p_function = ""); + static String get_builtin_rename(const String &p_name); + + static bool has_hint_replacement(const String &p_name); + static TokenType get_hint_replacement(const String &p_name); + + static bool is_renamed_render_mode(RS::ShaderMode p_mode, const String &p_name); + static String get_render_mode_rename(const String &p_name); + + static bool has_removed_render_mode(RS::ShaderMode p_mode, const String &p_name); + static bool can_remove_render_mode(const String &p_name); + + static bool has_removed_type(const String &p_name); + + static bool is_renamed_main_function(RS::ShaderMode p_mode, const String &p_name); + static bool is_renamee_main_function(RS::ShaderMode p_mode, const String &p_name); + static String get_main_function_rename(const String &p_name); + static TokenType get_renamed_function_type(const String &p_name); + static int get_renamed_function_arg_count(const String &p_name); + + static bool is_removed_builtin(RS::ShaderMode p_mode, const String &p_name, const String &p_function = ""); + static TokenType get_removed_builtin_uniform_type(const String &p_name); + static Vector get_removed_builtin_hints(const String &p_name); + + static bool _rename_has_special_handling(const String &p_name); + + static void _get_builtin_renames_list(List *r_list); + static void _get_render_mode_renames_list(List *r_list); + static void _get_hint_renames_list(List *r_list); + static void _get_function_renames_list(List *r_list); + static void _get_render_mode_removals_list(List *r_list); + static void _get_builtin_removals_list(List *r_list); + static void _get_type_removals_list(List *r_list); + static void _get_new_builtin_funcs_list(List *r_list); + static Vector _get_funcs_builtin_rename(RS::ShaderMode p_mode, const String &p_name); + static Vector _get_funcs_builtin_removal(RS::ShaderMode p_mode, const String &p_name); + + struct RenamedBuiltins { + const char *name; + const char *replacement; + const Vector>> mode_functions; + const bool special_handling; + }; + + struct RenamedRenderModes { + const RS::ShaderMode mode; + const char *name; + const char *replacement; + }; + + struct RenamedHints { + const char *name; + const ShaderLanguage::TokenType replacement; + }; + + struct RenamedFunctions { + const RS::ShaderMode mode; + const ShaderLanguage::TokenType type; + const int arg_count; + const char *name; + const char *replacement; + }; + + struct RemovedRenderModes { + const RS::ShaderMode mode; + const char *name; + const bool can_remove; + }; + + struct RemovedBuiltins { + const char *name; + const ShaderLanguage::TokenType uniform_type; + const Vector hints; + const Vector>> mode_functions; + }; + +private: + struct UniformDecl { + List::Element *start_pos = nullptr; + List::Element *uniform_stmt_pos = nullptr; + List::Element *end_pos = nullptr; + List::Element *interp_qual_pos = nullptr; + List::Element *type_pos = nullptr; + List::Element *name_pos = nullptr; + Vector::Element *> hint_poses; + + bool is_array = false; + bool has_uniform_qual() const { + return start_pos != nullptr && ShaderLanguage::is_token_uniform_qual(start_pos->get().type); + } + bool has_interp_qual() const { + return interp_qual_pos != nullptr; + } + }; + struct VarDecl { + List::Element *start_pos = nullptr; + List::Element *end_pos = nullptr; // semicolon or comma or right paren + List::Element *type_pos = nullptr; + List::Element *name_pos = nullptr; + bool is_array = false; + bool new_arr_style_decl = false; + bool is_func_arg = false; + void clear() { + start_pos = nullptr; + end_pos = nullptr; + type_pos = nullptr; + name_pos = nullptr; + } + }; + + struct StructDecl { + List::Element *start_pos = nullptr; + List::Element *name_pos = nullptr; + List::Element *body_start_pos = nullptr; // left curly + List::Element *body_end_pos = nullptr; // right curly - end of struct + HashMap members; + void clear() { + start_pos = nullptr; + name_pos = nullptr; + body_start_pos = nullptr; + body_end_pos = nullptr; + } + }; + + struct FunctionDecl { + List::Element *start_pos = nullptr; + List::Element *type_pos = nullptr; + List::Element *name_pos = nullptr; + List::Element *args_start_pos = nullptr; // left paren + List::Element *args_end_pos = nullptr; // right paren + List::Element *body_start_pos = nullptr; // left curly + List::Element *body_end_pos = nullptr; // right curly - end of function + bool is_renamed_main_function(RS::ShaderMode p_mode) const; + bool is_new_main_function(RS::ShaderMode p_mode) const; + + int arg_count = 0; + bool has_array_return_type = false; + void clear() { + type_pos = nullptr; + name_pos = nullptr; + args_start_pos = nullptr; + args_end_pos = nullptr; + body_start_pos = nullptr; + body_end_pos = nullptr; + } + }; + static const RenamedBuiltins renamed_builtins[]; + static const RenamedRenderModes renamed_render_modes[]; + static const RenamedHints renamed_hints[]; + static const RenamedFunctions renamed_functions[]; + static const RemovedRenderModes removed_render_modes[]; + static const RemovedBuiltins removed_builtins[]; + static const char *removed_types[]; + static const char *old_builtin_funcs[]; + static HashSet _new_builtin_funcs; + String old_code; + List code_tokens; + List::Element *curr_ptr = nullptr; + List::Element *after_shader_decl = nullptr; + + HashMap uniform_decls; + HashMap> var_decls; + HashMap function_decls; + HashMap> scope_declarations; + + HashMap struct_decls; + RenderingServer::ShaderMode shader_mode = RenderingServer::ShaderMode::SHADER_MAX; + + bool warning_comments = true; + bool verbose_comments = false; + bool fail_on_unported = true; + + bool function_pass_failed = false; + bool var_pass_failed = false; + String err_str; + int err_line = 0; + + Token eof_token{ ShaderLanguage::TK_EOF, {}, 0, 0, 0, 0 }; + + HashMap> report; + + static RS::ShaderMode get_shader_mode_from_string(const String &p_mode); + + String get_token_literal_text(const Token &p_tk) const; + static Token mkTok(TokenType p_type, const StringName &p_text = StringName(), double constant = 0, uint16_t p_line = 0); + static bool token_is_skippable(const Token &p_tk); + bool token_is_type(const Token &p_tk); + static bool token_is_hint(const Token &p_tk); + + void reset(); + bool _preprocess_code(); + List::Element *get_next_token(); + List::Element *get_prev_token(); + List::Element *remove_cur_and_get_next(); + TokenType peek_next_tk_type(uint32_t p_count = 1) const; + TokenType peek_prev_tk_type(uint32_t p_count = 1) const; + List::Element *get_pos() const; + bool reset_to(List::Element *p_pos); + bool insert_after(const Vector &p_token_list, List::Element *p_pos); + bool insert_before(const Vector &p_token_list, List::Element *p_pos); + bool insert_after(const Token &p_token, List::Element *p_pos); + bool insert_before(const Token &p_token, List::Element *p_pos); + List::Element *replace_curr(const Token &p_token, String comment_prefix = String()); + List::Element *_get_next_token_ptr(List::Element *p_curr_ptr) const; + List::Element *_get_prev_token_ptr(List::Element *p_curr_ptr) const; + TokenType _peek_tk_type(int64_t p_count, List::Element **r_pos = nullptr) const; + + bool scope_has_decl(const String &p_scope, const String &p_name) const; + bool _handle_new_keyword_rename(TokenE *pos, bool p_detected_3x, HashMap &p_func_renames); + TokenE *_rename_keyword_id(TokenE *pos, bool p_detected_3x, const HashMap &p_func_renames); + + bool _has_any_preprocessor_directives(); + bool _is_code_deprecated(); + bool _parse_uniform(); + bool _tok_is_start_of_decl(const Token &p_tk); + bool _skip_uniform(); + bool _parse_uniforms(); + bool _parse_structs(); + bool _skip_array_size(); + bool _parse_struct(); + bool _skip_struct(); + bool _check_deprecated_type(TokenE *p_type_tok); + bool _add_to_report(int p_line, const String &p_msg, int level = 0); + bool _add_comment_before(const String &p_comment, List::Element *p_pos, bool warning = true); + bool _add_comment_at_eol(const String &p_comment, List::Element *p_pos); + bool _process_func_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, bool p_second_pass = false); + bool _process_decl_statement(TokenE *p_start_tok, TokenE *p_type_tok, const String &p_scope = "", bool p_func_args = false); + bool _parse_decls(bool p_first_pass); + bool _process_decl_if_exist(String p_curr_func, bool p_first_pass); + bool _insert_uniform_declaration(const String &p_name); + List::Element *_remove_from_curr_to(List::Element *p_end); + List::Element *_get_end_of_closure(); + static HashSet _construct_new_builtin_funcs(); + + enum { + NEW_IDENT = -1 + }; +}; +#endif // DISABLE_DEPRECATED + +#endif // SHADER_CONVERTER_H diff --git a/servers/rendering/shader_language.cpp b/servers/rendering/shader_language.cpp index 879a83f5198c..ec05f5dc5bff 100644 --- a/servers/rendering/shader_language.cpp +++ b/servers/rendering/shader_language.cpp @@ -230,6 +230,13 @@ const char *ShaderLanguage::token_names[TK_MAX] = { "CURSOR", "ERROR", "EOF", + "TAB", + "CR", + "SPACE", + "NEWLINE", + "BLOCK_COMMENT", + "LINE_COMMENT", + "PREPROC_DIRECTIVE", }; String ShaderLanguage::get_token_text(Token p_token) { @@ -250,6 +257,8 @@ ShaderLanguage::Token ShaderLanguage::_make_token(TokenType p_type, const String tk.type = p_type; tk.text = p_text; tk.line = tk_line; + tk.pos = tk_start_pos; + tk.length = char_idx - tk_start_pos; if (tk.type == TK_ERROR) { _set_error(p_text); } @@ -402,8 +411,11 @@ const ShaderLanguage::KeyWord ShaderLanguage::keyword_list[] = { ShaderLanguage::Token ShaderLanguage::_get_token() { #define GETCHAR(m_idx) (((char_idx + m_idx) < code.length()) ? code[char_idx + m_idx] : char32_t(0)) - +#define IF_DBG_MK_TK(m_type) \ + if (unlikely(debug_parse)) \ + return _make_token(m_type); while (true) { + tk_start_pos = char_idx; char_idx++; switch (GETCHAR(-1)) { case 0: @@ -411,11 +423,17 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { case 0xFFFF: return _make_token(TK_CURSOR); //for completion case '\t': + IF_DBG_MK_TK(TK_TAB); + continue; case '\r': + IF_DBG_MK_TK(TK_CR); + continue; case ' ': + IF_DBG_MK_TK(TK_SPACE); continue; case '\n': tk_line++; + IF_DBG_MK_TK(TK_NEWLINE); continue; case '/': { switch (GETCHAR(0)) { @@ -424,10 +442,12 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { char_idx++; while (true) { if (GETCHAR(0) == 0) { + IF_DBG_MK_TK(TK_BLOCK_COMMENT); return _make_token(TK_EOF); } if (GETCHAR(0) == '*' && GETCHAR(1) == '/') { char_idx += 2; + IF_DBG_MK_TK(TK_BLOCK_COMMENT); break; } else if (GETCHAR(0) == '\n') { tk_line++; @@ -441,11 +461,13 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { while (true) { if (GETCHAR(0) == '\n') { + IF_DBG_MK_TK(TK_LINE_COMMENT); tk_line++; char_idx++; break; } if (GETCHAR(0) == 0) { + IF_DBG_MK_TK(TK_LINE_COMMENT); return _make_token(TK_EOF); } char_idx++; @@ -702,6 +724,23 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { return _make_token(TK_ERROR, "Invalid include enter/exit hint token (@@> and @@<)"); } } break; + case '#': { + if (!debug_parse) { // We shouldn't get here if the preprocessor is enabled and doing a non-debug parse. + return _make_token(TK_ERROR, "Unexpected pre-processor directive (Is the pre-processor enabled?)"); + } + while (true) { + char32_t c = GETCHAR(0); + if (c == '\\' && GETCHAR(1) == '\n') { + char_idx += 2; + tk_line++; + continue; + } + if (c == '\n' || c == 0) { + return _make_token(TK_PREPROC_DIRECTIVE); + } + char_idx++; + } + } break; default: { char_idx--; //go back one, since we have no idea what this is @@ -767,7 +806,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { lut_case = CASE_EXPONENT; } else if (symbol == 'f' && !hexa_found) { if (!period_found && !exponent_found) { - error = true; + error = !debug_parse; } float_suffix_found = true; end_suffix_found = true; @@ -886,6 +925,8 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { tk.constant = str.to_float(); } tk.line = tk_line; + tk.pos = tk_start_pos; + tk.length = char_idx - tk_start_pos; return tk; } @@ -934,6 +975,7 @@ ShaderLanguage::Token ShaderLanguage::_get_token() { return Token(); #undef GETCHAR +#undef IF_DBG_MK_TK } bool ShaderLanguage::_lookup_next(Token &r_tk) { @@ -956,22 +998,46 @@ ShaderLanguage::Token ShaderLanguage::_peek() { return tk; } -String ShaderLanguage::token_debug(const String &p_code) { +String ShaderLanguage::token_debug(const String &p_code, bool p_debug_parse) { clear(); code = p_code; + debug_parse = p_debug_parse; String output; - Token tk = _get_token(); + while (tk.type != TK_EOF && tk.type != TK_ERROR) { - output += itos(tk_line) + ": " + get_token_text(tk) + "\n"; + String literal_text = p_code.substr(tk.pos, tk.length); + output += itos(tk_line) + " (" + itos(tk.pos) + ":" + itos(tk.pos + tk.length) + "): " + get_token_text(tk) + " [" + literal_text; + String suffix = "]\n"; + // add error string if invalid float constant + if (debug_parse && tk.type == TK_FLOAT_CONSTANT && literal_text.ends_with("f") && !literal_text.contains(".") && !literal_text.contains("e")) { + output += " (Invalid float constant)]\n"; + } else { + output += "]\n"; + } tk = _get_token(); } + debug_parse = false; return output; } +void ShaderLanguage::token_debug_stream(const String &p_code, List &r_output, bool p_debug_parse) { + clear(); + + code = p_code; + debug_parse = p_debug_parse; + Token tk = _get_token(); + + while (tk.type != TK_EOF && tk.type != TK_ERROR) { + r_output.push_back(tk); + tk = _get_token(); + } + debug_parse = false; +} + bool ShaderLanguage::is_token_variable_datatype(TokenType p_type) { return ( p_type == TK_TYPE_VOID || @@ -1064,6 +1130,12 @@ bool ShaderLanguage::is_token_arg_qual(TokenType p_type) { p_type == TK_ARG_INOUT); } +bool ShaderLanguage::is_token_uniform_qual(TokenType p_type) { + return ( + p_type == TK_INSTANCE || + p_type == TK_GLOBAL); +} + ShaderLanguage::DataPrecision ShaderLanguage::get_token_precision(TokenType p_type) { if (p_type == TK_PRECISION_LOW) { return PRECISION_LOWP; @@ -4118,6 +4190,17 @@ bool ShaderLanguage::is_token_hint(TokenType p_type) { return int(p_type) > int(TK_RENDER_MODE) && int(p_type) < int(TK_SHADER_TYPE); } +bool ShaderLanguage::is_token_keyword(TokenType p_type) { + int idx = 0; + while (keyword_list[idx].text) { + if (keyword_list[idx].token == p_type) { + return true; + } + idx++; + } + return false; +} + bool ShaderLanguage::convert_constant(ConstantNode *p_constant, DataType p_to_type, Scalar *p_value) { if (p_constant->datatype == p_to_type) { if (p_value) { diff --git a/servers/rendering/shader_language.h b/servers/rendering/shader_language.h index 48df77f6bbfc..10ac3dd4bcd4 100644 --- a/servers/rendering/shader_language.h +++ b/servers/rendering/shader_language.h @@ -194,7 +194,15 @@ class ShaderLanguage { TK_CURSOR, TK_ERROR, TK_EOF, - TK_MAX + TK_TAB, // for debug purposes + TK_CR, + TK_SPACE, + TK_NEWLINE, + TK_BLOCK_COMMENT, + TK_LINE_COMMENT, + TK_PREPROC_DIRECTIVE, + TK_MAX, + TK_REG_MAX = TK_TAB, }; /* COMPILER */ @@ -788,6 +796,8 @@ class ShaderLanguage { StringName text; double constant; uint16_t line; + uint16_t length; + int32_t pos; bool is_integer_constant() const { return type == TK_INT_CONSTANT || type == TK_UINT_CONSTANT; } @@ -803,6 +813,7 @@ class ShaderLanguage { static DataInterpolation get_token_interpolation(TokenType p_type); static bool is_token_precision(TokenType p_type); static bool is_token_arg_qual(TokenType p_type); + static bool is_token_uniform_qual(TokenType p_type); static DataPrecision get_token_precision(TokenType p_type); static String get_precision_name(DataPrecision p_type); static String get_interpolation_name(DataInterpolation p_interpolation); @@ -814,6 +825,7 @@ class ShaderLanguage { static bool is_token_operator(TokenType p_type); static bool is_token_operator_assign(TokenType p_type); static bool is_token_hint(TokenType p_type); + static bool is_token_keyword(TokenType p_type); static bool convert_constant(ConstantNode *p_constant, DataType p_to_type, Scalar *p_value = nullptr); static DataType get_scalar_type(DataType p_type); @@ -1006,6 +1018,7 @@ class ShaderLanguage { String code; int char_idx = 0; int tk_line = 0; + int tk_start_pos = 0; StringName shader_type_identifier; StringName current_function; @@ -1140,6 +1153,8 @@ class ShaderLanguage { StringName completion_struct; int completion_argument = 0; + bool debug_parse = false; + #ifdef DEBUG_ENABLED uint32_t keyword_completion_context; #endif // DEBUG_ENABLED @@ -1222,7 +1237,10 @@ class ShaderLanguage { ShaderNode *get_shader(); - String token_debug(const String &p_code); + String token_debug(const String &p_code, bool p_debug_parse = false); + void token_debug_stream(const String &p_code, List &r_output, bool p_debug_parse); + + ShaderLanguage::Operator get_op(const TokenType &p_token) const; ShaderLanguage(); ~ShaderLanguage(); diff --git a/tests/servers/rendering/test_shader_converter.h b/tests/servers/rendering/test_shader_converter.h new file mode 100644 index 000000000000..af714577fb13 --- /dev/null +++ b/tests/servers/rendering/test_shader_converter.h @@ -0,0 +1,936 @@ +/**************************************************************************/ +/* test_shader_converter.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef TEST_SHADER_CONVERTER_H +#define TEST_SHADER_CONVERTER_H + +#ifndef DISABLE_DEPRECATED +#include "servers/rendering/shader_converter.h" +#include "servers/rendering/shader_language.h" +#include "servers/rendering/shader_types.h" +#include "tests/test_macros.h" + +#include + +namespace TestShaderConverter { + +void erase_all_empty(Vector &p_vec) { + int idx = p_vec.find(" "); + while (idx >= 0) { + p_vec.remove_at(idx); + idx = p_vec.find(" "); + } +} + +bool is_variable_char(unsigned char c) { + return std::isalnum(c) || c == '_'; +} + +bool is_operator_char(unsigned char c) { + return (c == '*') || (c == '+') || (c == '-') || (c == '/') || ((c >= '<') && (c <= '>')); +} + +// Remove unnecessary spaces from a line. +String remove_spaces(String &p_str) { + String res; + // Result is guaranteed to not be longer than the input. + res.resize(p_str.size()); + int wp = 0; + char32_t last = 0; + bool has_removed = false; + + for (int n = 0; n < p_str.size(); n++) { + // These test cases only use ASCII. + unsigned char c = static_cast(p_str[n]); + if (std::isblank(c)) { + has_removed = true; + } else { + if (has_removed) { + // Insert a space to avoid joining things that could potentially form a new token. + // E.g. "float x" or "- -". + if ((is_variable_char(c) && is_variable_char(last)) || + (is_operator_char(c) && is_operator_char(last))) { + res[wp++] = ' '; + } + has_removed = false; + } + res[wp++] = c; + last = c; + } + } + res.resize(wp); + return res; +} + +// The pre-processor changes indentation and inserts spaces when inserting macros. +// Re-format the code, without changing its meaning, to make it easier to compare. +String compact_spaces(String &p_str) { + Vector lines = p_str.split("\n", false); + erase_all_empty(lines); + for (String &line : lines) { + line = remove_spaces(line); + } + return String("\n").join(lines); +} + +void get_keyword_set(HashSet &p_keywords) { + List keywords; + ShaderLanguage::get_keyword_list(&keywords); + for (const String &keyword : keywords) { + p_keywords.insert(keyword); + } +} + +#define CHECK_SHADER_EQ(a, b) CHECK_EQ(compact_spaces(a), compact_spaces(b)) +#define CHECK_SHADER_NE(a, b) CHECK_NE(compact_spaces(a), compact_spaces(b)) + +void get_compile_info(ShaderLanguage::ShaderCompileInfo &info, RenderingServer::ShaderMode p_mode) { + info.functions = ShaderTypes::get_singleton()->get_functions(p_mode); + info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode); + info.shader_types = ShaderTypes::get_singleton()->get_types(); + // Only used by editor for completion, so it's not important for these tests. + info.global_shader_uniform_type_func = [](const StringName &p_name) -> ShaderLanguage::DataType { + return ShaderLanguage::TYPE_SAMPLER2D; + }; +} + +RenderingServer::ShaderMode get_shader_mode(const String &p_mode_string) { + if (p_mode_string == "canvas_item") { + return RS::SHADER_CANVAS_ITEM; + } else if (p_mode_string == "particles") { + return RS::SHADER_PARTICLES; + } else if (p_mode_string == "spatial") { + return RS::SHADER_SPATIAL; + } else if (p_mode_string == "sky") { + return RS::SHADER_SKY; + } else if (p_mode_string == "fog") { + return RS::SHADER_FOG; + } else { + return RS::SHADER_MAX; + } +} + +String get_shader_mode_name(const RenderingServer::ShaderMode &p_mode_string) { + switch (p_mode_string) { + case RS::SHADER_CANVAS_ITEM: + return "canvas_item"; + case RS::SHADER_PARTICLES: + return "particles"; + case RS::SHADER_SPATIAL: + return "spatial"; + case RS::SHADER_SKY: + return "sky"; + case RS::SHADER_FOG: + return "fog"; + default: + return "unknown"; + } +} +using SL = ShaderLanguage; +using SDC = ShaderDeprecatedConverter; + +#define TEST_CONVERSION(m_old_code, m_expected, m_is_deprecated) \ + { \ + ShaderDeprecatedConverter _i_converter; \ + CHECK_EQ(_i_converter.is_code_deprecated(m_old_code), m_is_deprecated); \ + CHECK(_i_converter.convert_code(m_old_code)); \ + String _i_new_code = _i_converter.emit_code(); \ + CHECK_SHADER_EQ(_i_new_code, m_expected); \ + } + +#define TEST_CONVERSION_COMPILE(m_old_code, m_expected, m_is_deprecated) \ + { \ + ShaderLanguage _i_language; \ + String _i_type = SL::get_shader_type(m_old_code); \ + ShaderLanguage::ShaderCompileInfo _i_info; \ + get_compile_info(_i_info, get_shader_mode(_i_type)); \ + ShaderDeprecatedConverter _i_converter; \ + CHECK_EQ(_i_converter.is_code_deprecated(m_old_code), m_is_deprecated); \ + CHECK_EQ(_i_converter.get_error_text(), ""); \ + CHECK(_i_converter.convert_code(m_old_code)); \ + String _i_new_code = _i_converter.emit_code(); \ + CHECK_SHADER_EQ(_i_new_code, m_expected); \ + Error ret = _i_language.compile(_i_new_code, _i_info); \ + CHECK_EQ(_i_language.get_error_text(), ""); \ + CHECK_EQ(ret, Error::OK); \ + } + +TEST_CASE("[ShaderDeprecatedConverter] Simple conversion with arrays") { + String code = "shader_type particles; void vertex() { float xy[2] = {1.0,1.1}; }"; + String expected = "shader_type particles; void process() { float xy[2] = {1.0,1.1}; }"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] Test warning comments") { + // Test that warning comments are added when fail_on_unported is false and warning_comments is true + String code = "shader_type spatial;\nrender_mode specular_phong;"; + String expected = "shader_type spatial;\n/* !convert WARNING: Deprecated render mode 'specular_phong' is not supported by this version of Godot. */\nrender_mode specular_phong;"; + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_warning_comments(true); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + CHECK_EQ(new_code, expected); +} + +TEST_CASE("[ShaderDeprecatedConverter] Simple conversion with arrays and structs") { + String code = "shader_type particles; struct foo{float bar;} void vertex() { float xy[2] = {1.0,1.1}; }"; + String expected = "shader_type particles; struct foo{float bar;} void process() { float xy[2] = {1.0,1.1}; }"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] new-style array declaration") { + String code = "shader_type spatial; void vertex() { float[2] xy = {1.0,1.1}; }"; + // code should be the same + TEST_CONVERSION(code, code, false); +} + +TEST_CASE("[ShaderDeprecatedConverter] Simple conversion") { + String code = "shader_type particles; void vertex() { float x = 1.0; }"; + String expected = "shader_type particles; void process() { float x = 1.0; }"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] Replace non-conformant float literals") { + String code = "shader_type spatial; const float x = 1f;"; + String expected = "shader_type spatial; const float x = 1.0f;"; + TEST_CONVERSION(code, expected, true); +} + +TEST_CASE("[ShaderDeprecatedConverter] particles::vertex() -> particles::process()") { + SUBCASE("basic") { + String code = "shader_type particles; void vertex() { float x = 1.0; }"; + String expected = "shader_type particles; void process() { float x = 1.0; }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("with another function named `process` without correct signature") { + String code = "shader_type particles; void vertex() {} float process() { return 1.0; }"; + String expected = "shader_type particles; void process() {} float process_() { return 1.0; }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("with another function named `process` with correct signature") { + String code = "shader_type particles; void vertex() {} void process() {}"; + // Should be unchanged. + TEST_CONVERSION(code, code, false); + } + + SUBCASE("with another function named `process` that is called") { + String code = "shader_type particles; float process() { return 1.0; } void vertex() { float foo = process(); }"; + String expected = "shader_type particles; float process_() { return 1.0; } void process() { float foo = process_(); }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("with another function named `process` which calls `vertex`") { + String code = "shader_type particles; float process() {foo(); return 1.0;} void vertex() {} void foo() { vertex(); }"; + String expected = "shader_type particles; float process_() {foo(); return 1.0;} void process() {} void foo() { process(); }"; + TEST_CONVERSION(code, expected, true); + } + SUBCASE("No function named `vertex`") { + String code = "shader_type particles; void process() {}"; + // Should be unchanged. + TEST_CONVERSION(code, code, false); + } +} + +TEST_CASE("[ShaderDeprecatedConverter] CLEARCOAT_GLOSS -> CLEARCOAT_ROUGHNESS") { + SUBCASE("Left-hand simple assignment") { + String code("shader_type spatial; void fragment() {\n" + "CLEARCOAT_GLOSS = 1.0;\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "CLEARCOAT_ROUGHNESS = (1.0 - (1.0));\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Left-hand *= assignment") { + String code("shader_type spatial; void fragment() {\n" + "CLEARCOAT_GLOSS *= 0.5;\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 0.5));\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Right-hand usage") { + String code("shader_type spatial; void fragment() {\n" + "float foo = CLEARCOAT_GLOSS;\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "float foo = (1.0 - CLEARCOAT_ROUGHNESS);\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } + SUBCASE("both usages") { + String code("shader_type spatial; void fragment() {\n" + "float foo = (CLEARCOAT_GLOSS *= 0.5);\n" + "}\n"); + String expected("shader_type spatial; void fragment() {\n" + "float foo = ((1.0 - (CLEARCOAT_ROUGHNESS = (1.0 - ((1.0 - CLEARCOAT_ROUGHNESS) * 0.5)))));\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } +} +TEST_CASE("[ShaderDeprecatedConverter] Wrap INDEX in int()") { + SUBCASE("basic") { + String code("shader_type particles; void vertex() {\n" + "int foo = INDEX/2;\n" + "}\n"); + String expected("shader_type particles; void process() {\n" + "int foo = int(INDEX)/2;\n" + "}\n"); + + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Wrapped in pre-existing cast") { + String code("shader_type particles; void vertex() {\n" + "float foo = float(INDEX/2);\n" + "}\n"); + String expected("shader_type particles; void process() {\n" + "float foo = float(int(INDEX)/2);\n" + "}\n"); + + TEST_CONVERSION(code, expected, true); + } + SUBCASE("Without clobbering existing casts") { + String code("shader_type particles; void vertex() {\n" + "int foo = int(INDEX/2) * int(INDEX) * 2;\n" + "float bar = float(INDEX);\n" + "}\n"); + String expected("shader_type particles; void process() {\n" + "int foo = int(int(INDEX)/2) * int(INDEX) * 2;\n" + "float bar = float(INDEX);\n" + "}\n"); + TEST_CONVERSION(code, expected, true); + } +} + +TEST_CASE("[ShaderDeprecatedConverter] All hint renames") { + String code_template = "shader_type spatial; uniform sampler2D foo : %s;"; + // get all the hint renames + List hints; + ShaderDeprecatedConverter::_get_hint_renames_list(&hints); + + SUBCASE("No renamed hints present in current keyword list") { + HashSet keywords_set; + get_keyword_set(keywords_set); + for (const String &hint : hints) { + CHECK_FALSE(keywords_set.has(hint)); + } + } + + SUBCASE("All renamed hints are replaced") { + for (const String &hint : hints) { + ShaderDeprecatedConverter::TokenType type = ShaderDeprecatedConverter::get_hint_replacement(hint); + String rename = ShaderDeprecatedConverter::get_tokentype_text(type); + String code = vformat(code_template, hint); + String expected = vformat(code_template, rename); + TEST_CONVERSION(code, expected, true); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Built-in renames") { + // Get all the built-in renames. + List builtins; + ShaderDeprecatedConverter::_get_builtin_renames_list(&builtins); + // remove built-ins that have special handling, we test those above + for (List::Element *E = builtins.front(); E; E = E->next()) { + if (ShaderDeprecatedConverter::_rename_has_special_handling(E->get())) { + List::Element *prev = E->prev(); + builtins.erase(E); + E = prev; + } + } + Vector modes = { RS::SHADER_SPATIAL, RS::SHADER_CANVAS_ITEM, RS::SHADER_PARTICLES }; + HashMap>> rename_func_map; + for (RS::ShaderMode mode : modes) { + rename_func_map[mode] = HashMap>(); + for (const String &builtin : builtins) { + rename_func_map[mode][builtin] = ShaderDeprecatedConverter::_get_funcs_builtin_rename(mode, builtin); + } + } + + SUBCASE("All renamed built-ins are not currently built-in") { + for (RS::ShaderMode mode : modes) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, mode); + for (const String &builtin : builtins) { + // Now get the funcs applicable for this mode and built-in. + for (const String &func : rename_func_map[mode][builtin]) { + // The built-in should not be present in the built-ins list. + auto &finfo = info.functions[func]; + if (finfo.built_ins.has(builtin)) { + WARN_PRINT(vformat("Renamed 3.x Built-in %s is present in function %s", builtin, func)); + } + CHECK_FALSE(finfo.built_ins.has(builtin)); + } + } + } + } + + SUBCASE("All renamed built-ins are replaced") { + String code_template = "shader_type %s; void %s() { %s; }"; + for (RS::ShaderMode mode : modes) { + for (const String &builtin : builtins) { + // Now get the funcs applicable for this mode and built-in. + String rename = ShaderDeprecatedConverter::get_builtin_rename(builtin); + for (const String &func : rename_func_map[mode][builtin]) { + String code = vformat(code_template, get_shader_mode_name(mode), func, builtin); + String expected = vformat(code_template, get_shader_mode_name(mode), func, rename); + TEST_CONVERSION(code, expected, true); + } + } + } + } + SUBCASE("No renaming built-ins in non-candidate functions") { + String code_template = "shader_type %s; void %s() { float %s = 1.0; %s += 1.0; }"; + for (RS::ShaderMode mode : modes) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, mode); + for (const String &builtin : builtins) { + Vector non_funcs; + for (KeyValue &func : info.functions) { + if (func.key == "global") { + continue; + } + if (!rename_func_map[mode][builtin].has(func.key)) { + non_funcs.push_back(func.key); + } + } + String rename = ShaderDeprecatedConverter::get_builtin_rename(builtin); + for (const String &func : non_funcs) { + String code = vformat(code_template, get_shader_mode_name(mode), func, builtin, builtin); + // The code should not change. + TEST_CONVERSION(code, code, false); + } + } + } + } + SUBCASE("No renaming built-ins in candidate functions with built-in declared") { + String code_template = "shader_type %s; void %s() { float %s = 1.0; %s += 1.0; }"; + for (RS::ShaderMode mode : modes) { + for (const String &builtin : builtins) { + for (const String &func : rename_func_map[mode][builtin]) { + String code = vformat(code_template, get_shader_mode_name(mode), func, builtin, builtin); + // The code should not change. + TEST_CONVERSION(code, code, false); + } + } + } + } +} + +// TODO: Remove this when the MODULATE built-in PR lands. +// If this fails, remove the MODULATE entry from ShaderDeprecatedConverter::removed_builtins, then remove this test and the following test. +TEST_CASE("[ShaderDeprecatedConverter] MODULATE is not a built-in") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::ShaderMode::SHADER_CANVAS_ITEM); + SUBCASE("MODULATE is not a built-in") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + auto &finfo = info.functions[func]; + CHECK_FALSE(finfo.built_ins.has("MODULATE")); + } + } +} + +// Don't remove this one if the above doesn't fail too. +TEST_CASE("[ShaderDeprecatedConverter] MODULATE handling") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::ShaderMode::SHADER_CANVAS_ITEM); + SUBCASE("Fails to compile") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func); + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + } + SUBCASE("Fails to convert on fail_on_unported=true") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func); + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_fail_on_unported(true); + CHECK_FALSE(converter.convert_code(code)); + } + } + + SUBCASE("Conversion succeeds on fail_on_unported=false") { + for (const String &func : Vector{ "vertex", "fragment", "light" }) { + String code = vformat("shader_type canvas_item; void %s() { MODULATE; }", func); + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + CHECK(new_code.find("/*") != -1); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Uniform declarations for removed builtins") { + // Test uniform declaration inserts for removed builtins for all shader types. + String code_template = "shader_type %s;%s void %s() { %s; }"; + String uniform_template = "\nuniform %s %s : %s;\n"; + // Get all the removed built-ins. + List builtins; + ShaderDeprecatedConverter::_get_builtin_removals_list(&builtins); + Vector modes = { RS::SHADER_SPATIAL, RS::SHADER_CANVAS_ITEM, RS::SHADER_PARTICLES }; + HashMap compiler_infos; + + SUBCASE("Removed built-ins are not currently built-in") { + for (RS::ShaderMode mode : modes) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, mode); + for (const String &builtin : builtins) { + Vector funcs = ShaderDeprecatedConverter::_get_funcs_builtin_removal(mode, builtin); + for (const String &func : funcs) { + const ShaderLanguage::FunctionInfo &finfo = info.functions[func]; + if (finfo.built_ins.has(builtin)) { + WARN_PRINT(vformat("Removed 3.x Built-in %s is present in function %s", builtin, func)); + } + CHECK_FALSE(finfo.built_ins.has(builtin)); + } + } + } + } + + SUBCASE("All removed built-ins have uniform declarations") { + for (RS::ShaderMode mode : modes) { + for (const String &builtin : builtins) { + // now get the funcs applicable for this mode and builtins + ShaderLanguage::TokenType type = ShaderDeprecatedConverter::get_removed_builtin_uniform_type(builtin); + if (type == ShaderDeprecatedConverter::TokenType::TK_ERROR) { + continue; + } + Vector hints = ShaderDeprecatedConverter::get_removed_builtin_hints(builtin); + Vector funcs = ShaderDeprecatedConverter::_get_funcs_builtin_removal(mode, builtin); + String hint_string = ""; + for (int i = 0; i < hints.size(); i++) { + hint_string += ShaderDeprecatedConverter::get_tokentype_text(hints[i]); + if (i < hints.size() - 1) { + hint_string += ", "; + } + } + String uniform_decl = vformat(uniform_template, ShaderDeprecatedConverter::get_tokentype_text(type), builtin, hint_string); + for (const String &func : funcs) { + String code = vformat(code_template, get_shader_mode_name(mode), "", func, builtin); + if (type == ShaderDeprecatedConverter::TokenType::TK_ERROR) { // Unported builtins with no uniform declaration + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + CHECK_FALSE(converter.convert_code(code)); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + continue; + } + String expected = vformat(code_template, get_shader_mode_name(mode), uniform_decl, func, builtin); + TEST_CONVERSION(code, expected, true); + } + } + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Handling of renamed render_modes") { + List render_modes; + ShaderDeprecatedConverter::_get_render_mode_renames_list(&render_modes); + HashSet render_modes_set; + for (const String &render_mode : render_modes) { + render_modes_set.insert(render_mode); + } + // static const char *code_template = "shader_type %s; render_mode %s;"; + static const char *code_template = "shader_type %s; render_mode blend_mix, %s, depth_draw_always;"; + SUBCASE("Renamed render modes are not currently valid render modes") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + for (const auto &render_mode : info.render_modes) { + if (render_modes_set.has(render_mode.name)) { + WARN_PRINT(vformat("Renamed 3.x Render Mode %s is present in render modes", render_mode.name)); + } + CHECK_FALSE(render_modes_set.has(render_mode.name)); + } + } + for (const String &render_mode : render_modes) { + SUBCASE((render_mode + "is renamed").utf8().get_data()) { + String rename = ShaderDeprecatedConverter::get_render_mode_rename(render_mode); + String code = vformat(code_template, "spatial", render_mode); + String expected = vformat(code_template, "spatial", rename); + TEST_CONVERSION(code, expected, true); + } + SUBCASE((render_mode + "is renamed").utf8().get_data()) { + String rename = ShaderDeprecatedConverter::get_render_mode_rename(render_mode); + String code = vformat(code_template, "spatial", render_mode); + String expected = vformat(code_template, "spatial", rename); + TEST_CONVERSION(code, expected, true); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Handling of removed render_modes") { + List removed_render_modes; + ShaderDeprecatedConverter::_get_render_mode_removals_list(&removed_render_modes); + HashSet render_modes_set; + for (const String &render_mode : removed_render_modes) { + render_modes_set.insert(render_mode); + } + SUBCASE("Removed render modes are not currently valid render modes") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + for (const auto &render_mode : info.render_modes) { + if (render_modes_set.has(render_mode.name)) { + WARN_PRINT(vformat("Removed 3.x Render Mode %s is present in render modes", render_mode.name)); + } + CHECK_FALSE(render_modes_set.has(render_mode.name)); + } + } + const char *code_template = "shader_type %s; %s"; + static const char *render_mode_template = "render_mode blend_mix, %s, depth_draw_always;"; + static const char *render_mode_removed = "render_mode blend_mix, depth_draw_always;"; + + for (const String &render_mode : removed_render_modes) { + SUBCASE((render_mode + " is handled").utf8().get_data()) { + String render_mode_decl = vformat(render_mode_template, render_mode); + String code = vformat(code_template, "spatial", render_mode_decl); + String expected = vformat(code_template, "spatial", ShaderDeprecatedConverter::can_remove_render_mode(render_mode) ? render_mode_removed : render_mode_decl); + ShaderDeprecatedConverter converter; + CHECK(converter.is_code_deprecated(code)); + converter.set_warning_comments(false); + converter.set_fail_on_unported(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + CHECK_SHADER_EQ(new_code, expected); + // Check for warning comment + converter.set_warning_comments(true); + new_code = converter.emit_code(); + CHECK(new_code.contains("/* !convert WARNING:")); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Renaming of functions with the same name as new built-in functions") { + List builtins; + ShaderDeprecatedConverter::_get_new_builtin_funcs_list(&builtins); + + for (const String &builtin : builtins) { + SUBCASE((builtin + " renamed").utf8().get_data()) { + String code = vformat("shader_type spatial; void %s() { %s(); }", builtin, builtin); + String expected = vformat("shader_type spatial; void %s_() { %s_(); }", builtin, builtin); + TEST_CONVERSION(code, expected, true); + } + + SUBCASE((builtin + " usage without declaration not replaced").utf8().get_data()) { + String code = vformat("shader_type spatial; void foo() { %s(); }", builtin); + TEST_CONVERSION(code, code, false); + } + SUBCASE((builtin + " renamed while handling usage as variable").utf8().get_data()) { + const char *test_format = "shader_type spatial;\nvoid foo(){%s();}\nvoid %s() { float %s = 3.0; }\n"; + String code = vformat(test_format, builtin, builtin, builtin); + String last_builtin = builtin; + if (ShaderDeprecatedConverter::tokentype_is_new_reserved_keyword(ShaderDeprecatedConverter::get_tokentype_from_text(builtin))) { + last_builtin += "_"; + } + String expected = vformat(test_format, builtin + "_", builtin + "_", last_builtin); + TEST_CONVERSION(code, expected, true); + } + } +} + +// Reserved keywords (i.e. non-built-in function keywords that have a discrete token type) +TEST_CASE("[ShaderDeprecatedConverter] Replacement of reserved keywords used as identifiers") { + Vector keywords; + for (int i = 0; i < ShaderLanguage::TK_MAX; i++) { + if (ShaderDeprecatedConverter::tokentype_is_new_reserved_keyword(static_cast(i))) { + keywords.push_back(ShaderDeprecatedConverter::get_tokentype_text(static_cast(i))); + } + } + Vector hint_keywords; + for (int i = 0; i < SL::TK_MAX; i++) { + if (SDC::tokentype_is_new_hint(static_cast(i))) { + hint_keywords.push_back(SDC::get_tokentype_text(static_cast(i))); + } + } + Vector uniform_quals; + for (int i = 0; i < SL::TK_MAX; i++) { + if (SL::is_token_uniform_qual(static_cast(i))) { + uniform_quals.push_back(SDC::get_tokentype_text(static_cast(i))); + } + } + Vector shader_types_to_test = { "spatial", "canvas_item", "particles" }; + + static const char *decl_test_template[]{ + "shader_type %s;\nvoid %s() {}\nvoid foo(){%s();}\n", + "shader_type %s;\nvoid test_func() {float %s; %s = 1.0;}\n", + "shader_type %s;\nuniform float %s;\nvoid foo(){float bar = %s * 3.0;}\n", + "shader_type %s;\nconst float %s = 1.0;\nvoid foo(){float bar = %s * 3.0;}\n", + "shader_type %s;\nvarying float %s;\nvoid foo(){float bar = %s * 3.0;}\n", + "shader_type %s;\nstruct foo{float %s;};\nvoid bar(){foo f; f.%s = 1.0;}\n", + "shader_type %s;\nstruct %s{float foo;};\nvoid bar(){%s f; f.foo = 1.0;}\n", + nullptr + }; + // NOTE: if this fails, the current behavior of the converter to replace these has to be changed. + SUBCASE("Code with reserved keywords used as identifiers fail to compile") { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + for (const String &shader_type : shader_types_to_test) { + for (const String &keyword : keywords) { + for (int i = 0; decl_test_template[i] != nullptr; i++) { + String code = vformat(decl_test_template[i], shader_type, keyword, keyword); + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + } + } + } + SUBCASE("Code with reserved keywords used as identifiers is converted successfully") { + for (const String &shader_type : shader_types_to_test) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, get_shader_mode(shader_type)); + for (const String &keyword : keywords) { + for (int i = 0; decl_test_template[i] != nullptr; i++) { + if (shader_type == "particles" && String(decl_test_template[i]).contains("varying")) { + continue; + } + String code = vformat(decl_test_template[i], shader_type, keyword, keyword); + String expected = vformat(decl_test_template[i], shader_type, keyword + "_", keyword + "_"); + TEST_CONVERSION_COMPILE(code, expected, true); + } + } + } + } + static const char *new_hint_test = "shader_type spatial;\nuniform sampler2D foo : %s; const float %s = 1.0;\n"; + SUBCASE("New hints used as hints are not replaced") { + for (const String &hint : hint_keywords) { + String code = vformat(new_hint_test, hint, "bar"); + // Code should not change. + TEST_CONVERSION(code, code, false); + } + } + + SUBCASE("Mixed new hints used as hints and new hints used as identifiers") { + for (const String &hint : hint_keywords) { + String code = vformat(new_hint_test, hint, hint); + // Should not change. + ShaderDeprecatedConverter converter; + CHECK_FALSE(converter.is_code_deprecated(code)); // Should be detected as not deprecated. + converter.set_warning_comments(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + // Code should not change + CHECK_EQ(new_code, code); + // Check for warning comment + converter.set_warning_comments(true); + new_code = converter.emit_code(); + CHECK(new_code.contains("/* !convert WARNING:")); + } + } + static const char *non_id_keyword_test = "shader_type spatial;\n%s uniform sampler2D foo; const float %s = 1.0;\n"; + SUBCASE("New keywords not used as identifiers are not replaced") { + for (const String &qual : uniform_quals) { + // e.g. "shader_type spatial;\nglobal uniform sampler2D foo; const float bar = 1.0;\n" + String code = vformat(non_id_keyword_test, qual, "bar"); + // Code should not change. + TEST_CONVERSION(code, code, false); + } + } + + SUBCASE("Mixed idiomatic new reserved words and new reserved words used as identifiers") { + for (const String &qual : uniform_quals) { + // e.g. "shader_type spatial;\nglobal uniform sampler2D foo; const float global = 1.0;\n" + String code = vformat(non_id_keyword_test, qual, qual); + // Should not change. + ShaderDeprecatedConverter converter; + CHECK_FALSE(converter.is_code_deprecated(code)); // Should be detected as not deprecated. + converter.set_warning_comments(false); + CHECK(converter.convert_code(code)); + String new_code = converter.emit_code(); + // Code should not change + CHECK_EQ(new_code, code); + // Check for warning comment + converter.set_warning_comments(true); + new_code = converter.emit_code(); + CHECK(new_code.contains("/* !convert WARNING:")); + } + } +} + +TEST_CASE("[ShaderDeprecatedConverter] Convert default 3.x nodetree shader") { + static const char *default_3x_nodtree_shader = + R"(shader_type spatial; +render_mode blend_mix, depth_draw_always, cull_back, diffuse_burley, specular_schlick_ggx; + +uniform sampler2D texture_0: hint_albedo; + + +void node_bsdf_principled(vec4 color, float subsurface, vec4 subsurface_color, + float metallic, float specular, float roughness, float clearcoat, + float clearcoat_roughness, float anisotropy, float transmission, + float IOR, out vec3 albedo, out float sss_strength_out, + out float metallic_out, out float specular_out, + out float roughness_out, out float clearcoat_out, + out float clearcoat_gloss_out, out float anisotropy_out, + out float transmission_out, out float ior) { + metallic = clamp(metallic, 0.0, 1.0); + transmission = clamp(transmission, 0.0, 1.0); + + subsurface = subsurface * (1.0 - metallic); + + albedo = mix(color.rgb, subsurface_color.rgb, subsurface); + sss_strength_out = subsurface; + metallic_out = metallic; + specular_out = pow((IOR - 1.0)/(IOR + 1.0), 2)/0.08; + roughness_out = roughness; + clearcoat_out = clearcoat * (1.0 - transmission); + clearcoat_gloss_out = 1.0 - clearcoat_roughness; + anisotropy_out = clamp(anisotropy, 0.0, 1.0); + transmission_out = (1.0 - transmission) * (1.0 - metallic); + ior = IOR; +} + + +void node_tex_image(vec3 co, sampler2D ima, out vec4 color, out float alpha) { + color = texture(ima, co.xy); + alpha = color.a; +} + +void vertex () { +} + +void fragment () { + + // node: 'Image Texture' + // type: 'ShaderNodeTexImage' + // input sockets handling + vec3 node0_in0_vector = vec3(0.0, 0.0, 0.0); + // output sockets definitions + vec4 node0_out0_color; + float node0_out1_alpha; + + node0_in0_vector = vec3(UV, 0.0); + node_tex_image(node0_in0_vector, texture_0, node0_out0_color, node0_out1_alpha); + + // node: 'Principled BSDF' + // type: 'ShaderNodeBsdfPrincipled' + // input sockets handling + vec4 node1_in0_basecolor = node0_out0_color; + float node1_in1_subsurface = float(0.0); + vec3 node1_in2_subsurfaceradius = vec3(1.0, 0.20000000298023224, + 0.10000000149011612); + vec4 node1_in3_subsurfacecolor = vec4(0.800000011920929, 0.800000011920929, + 0.800000011920929, 1.0); + float node1_in4_metallic = float(0.0); + float node1_in5_specular = float(0.5); + float node1_in6_speculartint = float(0.0); + float node1_in7_roughness = float(1.0); + float node1_in8_anisotropic = float(0.0); + float node1_in9_anisotropicrotation = float(0.0); + float node1_in10_sheen = float(0.0); + float node1_in11_sheentint = float(0.5); + float node1_in12_clearcoat = float(0.0); + float node1_in13_clearcoatroughness = float(0.029999999329447746); + float node1_in14_ior = float(1.4500000476837158); + float node1_in15_transmission = float(0.0); + float node1_in16_transmissionroughness = float(0.0); + vec4 node1_in17_emission = vec4(0.0, 0.0, 0.0, 1.0); + float node1_in18_emissionstrength = float(1.0); + float node1_in19_alpha = float(1.0); + vec3 node1_in20_normal = NORMAL; + vec3 node1_in21_clearcoatnormal = vec3(0.0, 0.0, 0.0); + vec3 node1_in22_tangent = TANGENT; + // output sockets definitions + vec3 node1_bsdf_out0_albedo; + float node1_bsdf_out1_sss_strength; + float node1_bsdf_out3_specular; + float node1_bsdf_out2_metallic; + float node1_bsdf_out4_roughness; + float node1_bsdf_out5_clearcoat; + float node1_bsdf_out6_clearcoat_gloss; + float node1_bsdf_out7_anisotropy; + float node1_bsdf_out8_transmission; + float node1_bsdf_out9_ior; + + node_bsdf_principled(node1_in0_basecolor, node1_in1_subsurface, + node1_in3_subsurfacecolor, node1_in4_metallic, node1_in5_specular, + node1_in7_roughness, node1_in12_clearcoat, node1_in13_clearcoatroughness, + node1_in8_anisotropic, node1_in15_transmission, node1_in14_ior, + node1_bsdf_out0_albedo, node1_bsdf_out1_sss_strength, node1_bsdf_out2_metallic, + node1_bsdf_out3_specular, node1_bsdf_out4_roughness, node1_bsdf_out5_clearcoat, + node1_bsdf_out6_clearcoat_gloss, node1_bsdf_out7_anisotropy, + node1_bsdf_out8_transmission, node1_bsdf_out9_ior); + + ALBEDO = node1_bsdf_out0_albedo; + SSS_STRENGTH = node1_bsdf_out1_sss_strength; + SPECULAR = node1_bsdf_out3_specular; + METALLIC = node1_bsdf_out2_metallic; + ROUGHNESS = node1_bsdf_out4_roughness; + CLEARCOAT = node1_bsdf_out5_clearcoat; + CLEARCOAT_GLOSS = node1_bsdf_out6_clearcoat_gloss; + NORMAL = node1_in20_normal; + // uncomment it when you need it + // TRANSMISSION = vec3(1.0, 1.0, 1.0) * node1_bsdf_out8_transmission; + // uncomment it when you are modifying TANGENT + // TANGENT = normalize(cross(cross(node1_in22_tangent, NORMAL), NORMAL)); + // BINORMAL = cross(TANGENT, NORMAL); + // uncomment it when you have tangent(UV) set + // ANISOTROPY = node1_bsdf_out7_anisotropy; +} + )"; + + ShaderLanguage sl; + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + SUBCASE("Default 3.x nodetree shader does not compile") { + CHECK_NE(sl.compile(default_3x_nodtree_shader, info), Error::OK); + } + sl.clear(); + SUBCASE("Convert default 3.x nodetree shader") { + ShaderDeprecatedConverter converter; + CHECK(converter.convert_code(default_3x_nodtree_shader)); + String new_code = converter.emit_code(); + CHECK(new_code.find("/*") == -1); + converter.set_verbose_comments(true); + new_code = converter.emit_code(); + CHECK(new_code.find("/*") != -1); + CHECK_FALSE(converter.get_error_line()); + } + + SUBCASE("Converted default 3.x nodetree shader compiles") { + ShaderDeprecatedConverter converter; + CHECK(converter.convert_code(default_3x_nodtree_shader)); + String new_code = converter.emit_code(); + CHECK_EQ(sl.compile(new_code, info), Error::OK); + } + sl.clear(); +} + +} // namespace TestShaderConverter +#undef TEST_CONVERSION +#undef TEST_CONVERSION_COMPILE +#endif // DISABLE_DEPRECATED + +#endif // TEST_SHADER_CONVERTER_H diff --git a/tests/servers/rendering/test_shader_language.h b/tests/servers/rendering/test_shader_language.h new file mode 100644 index 000000000000..b51d567983ad --- /dev/null +++ b/tests/servers/rendering/test_shader_language.h @@ -0,0 +1,144 @@ +/**************************************************************************/ +/* test_shader_language.h */ +/**************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/**************************************************************************/ +/* Copyright (c) 2014-present Godot Engine contributors (see AUTHORS.md). */ +/* Copyright (c) 2007-2014 Juan Linietsky, Ariel Manzur. */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. */ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/**************************************************************************/ + +#ifndef TEST_SHADER_LANGUAGE_H +#define TEST_SHADER_LANGUAGE_H + +#include "servers/rendering/shader_language.h" +#include "servers/rendering/shader_types.h" + +#include "tests/test_macros.h" + +#include + +namespace TestShaderLanguage { + +void get_compile_info(ShaderLanguage::ShaderCompileInfo &info, RenderingServer::ShaderMode p_mode) { + info.functions = ShaderTypes::get_singleton()->get_functions(p_mode); + info.render_modes = ShaderTypes::get_singleton()->get_modes(p_mode); + info.shader_types = ShaderTypes::get_singleton()->get_types(); + // Only used by editor for completion, so it's not important for these tests. + info.global_shader_uniform_type_func = [](const StringName &p_name) -> ShaderLanguage::DataType { + return ShaderLanguage::TYPE_SAMPLER2D; + }; +} + +RenderingServer::ShaderMode get_shader_mode(const String &p_mode_string) { + if (p_mode_string == "canvas_item") { + return RS::SHADER_CANVAS_ITEM; + } else if (p_mode_string == "particles") { + return RS::SHADER_PARTICLES; + } else if (p_mode_string == "spatial") { + return RS::SHADER_SPATIAL; + } else if (p_mode_string == "sky") { + return RS::SHADER_SKY; + } else if (p_mode_string == "fog") { + return RS::SHADER_FOG; + } else { + return RS::SHADER_MAX; + } +} + +TEST_CASE("[ShaderLanguage] Minimal Script") { + ShaderLanguage sl; + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, RS::SHADER_SPATIAL); + String code = "shader_type spatial;"; + CHECK_EQ(sl.compile(code, info), Error::OK); +} + +// No keywords (except for built-in functions) should be valid identifiers. +TEST_CASE("[ShaderLanguage] Ensure no reserved keywords are valid identifiers") { + List keywords; + List builtin_functions; + ShaderLanguage::get_keyword_list(&keywords); + ShaderLanguage::get_builtin_funcs(&builtin_functions); + + HashSet builtin_set; + for (const String &keyword : builtin_functions) { + builtin_set.insert(keyword); + } + + HashSet non_func_keywords_set; + for (const String &keyword : keywords) { + if (!builtin_set.has(keyword)) { + non_func_keywords_set.insert(keyword); + } + } + + static const char *decl_test_template[]{ + "shader_type %s;\nvoid %s() {}\n", + "shader_type %s;\nvoid vertex() {float %s;}\n", + "shader_type %s;\nuniform sampler2D %s;\n", + "shader_type %s;\nconst float %s = 1.0;\n", + nullptr + }; + static const char *varying_template = "shader_type %s;\nvarying float %s;\n"; + Vector non_varying_types = { "particles", "sky", "fog" }; + + auto shader_types_to_test = ShaderTypes::get_singleton()->get_types(); + for (auto shader_type : shader_types_to_test) { + ShaderLanguage::ShaderCompileInfo info; + get_compile_info(info, get_shader_mode(shader_type)); + // test templates with non-keyword identifiers + + for (int i = 0; decl_test_template[i] != nullptr; i++) { + String code = vformat(decl_test_template[i], shader_type, "foo"); + String result; + ShaderLanguage sl; + CHECK_EQ(sl.compile(code, info), Error::OK); + } + if (!non_varying_types.has(shader_type)) { + String code = vformat(varying_template, shader_type, "foo"); + String result; + ShaderLanguage sl; + CHECK_EQ(sl.compile(code, info), Error::OK); + } + + for (const String &keyword : non_func_keywords_set) { + for (int i = 0; decl_test_template[i] != nullptr; i++) { + String code = vformat(decl_test_template[i], shader_type, keyword); + String result; + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + if (!non_varying_types.has(shader_type)) { + String code = vformat(varying_template, shader_type, keyword); + String result; + ShaderLanguage sl; + CHECK_NE(sl.compile(code, info), Error::OK); + } + } + } +} + +} // namespace TestShaderLanguage + +#endif // TEST_SHADER_LANGUAGE_H diff --git a/tests/test_main.cpp b/tests/test_main.cpp index 14a78558322a..f33e9bbb4a17 100644 --- a/tests/test_main.cpp +++ b/tests/test_main.cpp @@ -132,6 +132,8 @@ #include "tests/scene/test_viewport.h" #include "tests/scene/test_visual_shader.h" #include "tests/scene/test_window.h" +#include "tests/servers/rendering/test_shader_converter.h" +#include "tests/servers/rendering/test_shader_language.h" #include "tests/servers/rendering/test_shader_preprocessor.h" #include "tests/servers/test_text_server.h" #include "tests/test_validate_testing.h" @@ -270,7 +272,7 @@ struct GodotTestCaseListener : public doctest::IReporter { String name = String(p_in.m_name); String suite_name = String(p_in.m_test_suite); - if (name.contains("[SceneTree]") || name.contains("[Editor]")) { + if (name.contains("[SceneTree]") || name.contains("[Editor]") || name.contains("[ShaderDeprecatedConverter]") || name.contains("[ShaderLanguage]")) { memnew(MessageQueue); memnew(Input);