Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Importer version system for files imported in older Godot versions #84034

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions core/io/resource_importer.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,14 @@ ResourceFormatImporter::ResourceFormatImporter() {

//////////////

int ResourceImporter::get_importer_version() const {
return _importer_version;
}

void ResourceImporter::set_importer_version(int p_version) {
_importer_version = p_version;
}

void ResourceImporter::_bind_methods() {
BIND_ENUM_CONSTANT(IMPORT_ORDER_DEFAULT);
BIND_ENUM_CONSTANT(IMPORT_ORDER_SCENE);
Expand Down
7 changes: 6 additions & 1 deletion core/io/resource_importer.h
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,8 @@ class ResourceFormatImporter : public ResourceFormatLoader {
class ResourceImporter : public RefCounted {
GDCLASS(ResourceImporter, RefCounted);

int _importer_version = get_latest_importer_version();

protected:
static void _bind_methods();

Expand All @@ -110,7 +112,10 @@ class ResourceImporter : public RefCounted {
virtual String get_resource_type() const = 0;
virtual float get_priority() const { return 1.0; }
virtual int get_import_order() const { return IMPORT_ORDER_DEFAULT; }
virtual int get_format_version() const { return 0; }
virtual int get_latest_importer_version(const String &p_file_extension = "") const { return 0; }

int get_importer_version() const;
void set_importer_version(int p_version);

struct ImportOption {
PropertyInfo option;
Expand Down
66 changes: 33 additions & 33 deletions editor/editor_file_system.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -394,7 +394,6 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
String source_md5 = "";
Vector<String> dest_files;
String dest_md5 = "";
int version = 0;
bool found_uid = false;

while (true) {
Expand All @@ -418,8 +417,6 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
for (int i = 0; i < fa.size(); i++) {
to_check.push_back(fa[i]);
}
} else if (assign == "importer_version") {
version = value;
} else if (assign == "importer") {
importer_name = value;
} else if (assign == "uid") {
Expand Down Expand Up @@ -451,10 +448,6 @@ bool EditorFileSystem::_test_for_reimport(const String &p_path, bool p_only_impo
return true; // the importer has possibly changed, try to reimport.
}

if (importer->get_format_version() > version) {
return true; // version changed, reimport
}

// Read the md5's from a separate file (so the import parameters aren't dependent on the file version
String base_path = ResourceFormatImporter::get_singleton()->get_import_base_path(p_path);
Ref<FileAccess> md5s = FileAccess::open(base_path + ".md5", FileAccess::READ, &err);
Expand Down Expand Up @@ -1820,10 +1813,7 @@ Error EditorFileSystem::_reimport_group(const String &p_group_file, const Vector
f->store_line("[remap]");
f->store_line("");
f->store_line("importer=\"" + importer->get_importer_name() + "\"");
int version = importer->get_format_version();
if (version > 0) {
f->store_line("importer_version=" + itos(version));
}
f->store_line("importer_version=" + itos(importer->get_importer_version()));
if (!importer->get_resource_type().is_empty()) {
f->store_line("type=\"" + importer->get_resource_type() + "\"");
}
Expand Down Expand Up @@ -1946,6 +1936,9 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
}

ResourceUID::ID uid = ResourceUID::INVALID_ID;
// When a .import file exists but it has no importer version written, use version 0.
// This is because older Godot versions did not write the version if it was 0.
int importer_version = 0;
Variant generator_parameters;
if (p_generator_parameters) {
generator_parameters = *p_generator_parameters;
Expand All @@ -1971,6 +1964,7 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
if (p_custom_importer.is_empty()) {
importer_name = cf->get_value("remap", "importer");
}
importer_version = cf->get_value("remap", "importer_version", 0);

if (cf->has_section_key("remap", "uid")) {
String uidt = cf->get_value("remap", "uid");
Expand All @@ -1984,6 +1978,10 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
}
}
}
} else {
// When resources do not have a .import file, use the latest importer
// version (to be determined below after getting the importer).
importer_version = -1;
}

if (importer_name == "keep") {
Expand All @@ -2003,17 +2001,22 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(importer_name);
}

const String file_extension = p_file.get_extension().to_lower();
if (importer.is_null()) {
//not found by name, find by extension
importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(p_file.get_extension());
importer = ResourceFormatImporter::get_singleton()->get_importer_by_extension(file_extension);
load_default = true;
if (importer.is_null()) {
ERR_FAIL_V_MSG(ERR_FILE_CANT_OPEN, "BUG: File queued for import, but can't be imported, importer for type '" + importer_name + "' not found.");
}
}

if (importer_version == -1) {
// When resources do not have a .import file, use the latest importer version.
importer_version = importer->get_latest_importer_version(file_extension);
}
importer->set_importer_version(importer_version);
//mix with default params, in case a parameter is missing

List<ResourceImporter::ImportOption> opts;
importer->get_import_options(p_file, &opts);
for (const ResourceImporter::ImportOption &E : opts) {
Expand Down Expand Up @@ -2054,10 +2057,7 @@ Error EditorFileSystem::_reimport_file(const String &p_file, const HashMap<Strin
f->store_line("[remap]");
f->store_line("");
f->store_line("importer=\"" + importer->get_importer_name() + "\"");
int version = importer->get_format_version();
if (version > 0) {
f->store_line("importer_version=" + itos(version));
}
f->store_line("importer_version=" + itos(importer->get_importer_version()));
if (!importer->get_resource_type().is_empty()) {
f->store_line("type=\"" + importer->get_resource_type() + "\"");
}
Expand Down Expand Up @@ -2217,7 +2217,7 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {

EditorProgress pr("reimport", TTR("(Re)Importing Assets"), p_files.size());

Vector<ImportFile> reimport_files;
Vector<ImportFile> files_to_reimport;

HashSet<String> groups_to_reimport;

Expand Down Expand Up @@ -2248,7 +2248,7 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
ifile.path = file;
ResourceFormatImporter::get_singleton()->get_import_order_threads_and_importer(file, ifile.order, ifile.threaded, ifile.importer);
reloads.push_back(file);
reimport_files.push_back(ifile);
files_to_reimport.push_back(ifile);
}

// Group may have changed, so also update group reference.
Expand All @@ -2259,39 +2259,39 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
}
}

reimport_files.sort();
files_to_reimport.sort();

bool use_multiple_threads = GLOBAL_GET("editor/import/use_multiple_threads");

int from = 0;
for (int i = 0; i < reimport_files.size(); i++) {
if (groups_to_reimport.has(reimport_files[i].path)) {
for (int i = 0; i < files_to_reimport.size(); i++) {
if (groups_to_reimport.has(files_to_reimport[i].path)) {
continue;
}

if (use_multiple_threads && reimport_files[i].threaded) {
if (i + 1 == reimport_files.size() || reimport_files[i + 1].importer != reimport_files[from].importer) {
if (use_multiple_threads && files_to_reimport[i].threaded) {
if (i + 1 == files_to_reimport.size() || files_to_reimport[i + 1].importer != files_to_reimport[from].importer) {
if (from - i == 0) {
// Single file, do not use threads.
pr.step(reimport_files[i].path.get_file(), i);
_reimport_file(reimport_files[i].path);
pr.step(files_to_reimport[i].path.get_file(), i);
_reimport_file(files_to_reimport[i].path);
} else {
Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(reimport_files[from].importer);
Ref<ResourceImporter> importer = ResourceFormatImporter::get_singleton()->get_importer_by_name(files_to_reimport[from].importer);
ERR_CONTINUE(!importer.is_valid());

importer->import_threaded_begin();

ImportThreadData tdata;
tdata.max_index = from;
tdata.reimport_from = from;
tdata.reimport_files = reimport_files.ptr();
tdata.reimport_files = files_to_reimport.ptr();

WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_singleton()->add_template_group_task(this, &EditorFileSystem::_reimport_thread, &tdata, i - from + 1, -1, false, vformat(TTR("Import resources of type: %s"), reimport_files[from].importer));
WorkerThreadPool::GroupID group_task = WorkerThreadPool::get_singleton()->add_template_group_task(this, &EditorFileSystem::_reimport_thread, &tdata, i - from + 1, -1, false, vformat(TTR("Import resources of type: %s"), files_to_reimport[from].importer));
int current_index = from - 1;
do {
if (current_index < tdata.max_index) {
current_index = tdata.max_index;
pr.step(reimport_files[current_index].path.get_file(), current_index);
pr.step(files_to_reimport[current_index].path.get_file(), current_index);
}
OS::get_singleton()->delay_usec(1);
} while (!WorkerThreadPool::get_singleton()->is_group_task_completed(group_task));
Expand All @@ -2305,14 +2305,14 @@ void EditorFileSystem::reimport_files(const Vector<String> &p_files) {
}

} else {
pr.step(reimport_files[i].path.get_file(), i);
_reimport_file(reimport_files[i].path);
pr.step(files_to_reimport[i].path.get_file(), i);
_reimport_file(files_to_reimport[i].path);
}
}

// Reimport groups.

from = reimport_files.size();
from = files_to_reimport.size();

if (groups_to_reimport.size()) {
HashMap<String, Vector<String>> group_files;
Expand Down
2 changes: 1 addition & 1 deletion editor/import/resource_importer_obj.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -532,7 +532,7 @@ String ResourceImporterOBJ::get_resource_type() const {
return "Mesh";
}

int ResourceImporterOBJ::get_format_version() const {
int ResourceImporterOBJ::get_latest_importer_version(const String &p_file_extension) const {
return 1;
}

Expand Down
2 changes: 1 addition & 1 deletion editor/import/resource_importer_obj.h
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ class ResourceImporterOBJ : public ResourceImporter {
virtual void get_recognized_extensions(List<String> *p_extensions) const override;
virtual String get_save_extension() const override;
virtual String get_resource_type() const override;
virtual int get_format_version() const override;
virtual int get_latest_importer_version(const String &p_file_extension = "") const override;

virtual int get_preset_count() const override;
virtual String get_preset_name(int p_idx) const override;
Expand Down
54 changes: 32 additions & 22 deletions editor/import/resource_importer_scene.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,14 @@ Variant EditorSceneFormatImporter::get_option_visibility(const String &p_path, b
return ret;
}

int EditorSceneFormatImporter::get_importer_version() const {
return _importer_version;
}

void EditorSceneFormatImporter::set_importer_version(int p_version) {
_importer_version = p_version;
}

void EditorSceneFormatImporter::_bind_methods() {
GDVIRTUAL_BIND(_get_import_flags);
GDVIRTUAL_BIND(_get_extensions);
Expand Down Expand Up @@ -263,7 +271,11 @@ String ResourceImporterScene::get_resource_type() const {
return animation_importer ? "AnimationLibrary" : "PackedScene";
}

int ResourceImporterScene::get_format_version() const {
int ResourceImporterScene::get_latest_importer_version(const String &p_file_extension) const {
Ref<EditorSceneFormatImporter> scene_importer = get_scene_importer_for_file_extension(p_file_extension);
if (scene_importer.is_valid()) {
return scene_importer->get_importer_version();
}
return 1;
}

Expand Down Expand Up @@ -2390,29 +2402,14 @@ Node *ResourceImporterScene::pre_import(const String &p_source_file, const HashM
Error ResourceImporterScene::import(const String &p_source_file, const String &p_save_path, const HashMap<StringName, Variant> &p_options, List<String> *r_platform_variants, List<String> *r_gen_files, Variant *r_metadata) {
const String &src_path = p_source_file;

Ref<EditorSceneFormatImporter> importer;
String ext = src_path.get_extension().to_lower();
Ref<EditorSceneFormatImporter> importer = get_scene_importer_for_file_extension(ext);

EditorProgress progress("import", TTR("Import Scene"), 104);
progress.step(TTR("Importing Scene..."), 0);

for (Ref<EditorSceneFormatImporter> importer_elem : importers) {
List<String> extensions;
importer_elem->get_extensions(&extensions);

for (const String &F : extensions) {
if (F.to_lower() == ext) {
importer = importer_elem;
break;
}
}

if (importer.is_valid()) {
break;
}
}

ERR_FAIL_COND_V(!importer.is_valid(), ERR_FILE_UNRECOGNIZED);
importer->set_importer_version(get_importer_version());

int import_flags = 0;

Expand Down Expand Up @@ -2700,6 +2697,23 @@ void ResourceImporterScene::add_importer(Ref<EditorSceneFormatImporter> p_import
}
}

void ResourceImporterScene::remove_importer(Ref<EditorSceneFormatImporter> p_importer) {
importers.erase(p_importer);
}

Ref<EditorSceneFormatImporter> ResourceImporterScene::get_scene_importer_for_file_extension(const String &p_file_extension) {
for (Ref<EditorSceneFormatImporter> scene_importer : importers) {
List<String> extensions;
scene_importer->get_extensions(&extensions);
for (const String &F : extensions) {
if (F.to_lower() == p_file_extension) {
return scene_importer;
}
}
}
return Ref<EditorSceneFormatImporter>();
}

void ResourceImporterScene::remove_post_importer_plugin(const Ref<EditorScenePostImportPlugin> &p_plugin) {
post_importer_plugins.erase(p_plugin);
}
Expand All @@ -2713,10 +2727,6 @@ void ResourceImporterScene::add_post_importer_plugin(const Ref<EditorScenePostIm
}
}

void ResourceImporterScene::remove_importer(Ref<EditorSceneFormatImporter> p_importer) {
importers.erase(p_importer);
}

void ResourceImporterScene::clean_up_importer_plugins() {
importers.clear();
post_importer_plugins.clear();
Expand Down
8 changes: 7 additions & 1 deletion editor/import/resource_importer_scene.h
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ class ImporterMesh;
class EditorSceneFormatImporter : public RefCounted {
GDCLASS(EditorSceneFormatImporter, RefCounted);

int _importer_version = 1;

protected:
static void _bind_methods();

Expand Down Expand Up @@ -80,6 +82,9 @@ class EditorSceneFormatImporter : public RefCounted {
virtual void get_import_options(const String &p_path, List<ResourceImporter::ImportOption> *r_options);
virtual Variant get_option_visibility(const String &p_path, bool p_for_animation, const String &p_option, const HashMap<StringName, Variant> &p_options);

int get_importer_version() const;
void set_importer_version(int p_version);

EditorSceneFormatImporter() {}
};

Expand Down Expand Up @@ -246,6 +251,7 @@ class ResourceImporterScene : public ResourceImporter {

static void add_importer(Ref<EditorSceneFormatImporter> p_importer, bool p_first_priority = false);
static void remove_importer(Ref<EditorSceneFormatImporter> p_importer);
static Ref<EditorSceneFormatImporter> get_scene_importer_for_file_extension(const String &p_file_extension);

static void clean_up_importer_plugins();

Expand All @@ -254,7 +260,7 @@ class ResourceImporterScene : public ResourceImporter {
virtual void get_recognized_extensions(List<String> *p_extensions) const override;
virtual String get_save_extension() const override;
virtual String get_resource_type() const override;
virtual int get_format_version() const override;
virtual int get_latest_importer_version(const String &p_file_extension = "") const override;

virtual int get_preset_count() const override;
virtual String get_preset_name(int p_idx) const override;
Expand Down
1 change: 1 addition & 0 deletions modules/gltf/editor/editor_scene_importer_gltf.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ Node *EditorSceneFormatImporterGLTF::import_scene(const String &p_path, uint32_t
gltf.instantiate();
Ref<GLTFState> state;
state.instantiate();
gltf->set_importer_version(get_importer_version());
if (p_options.has("gltf/embedded_image_handling")) {
int32_t enum_option = p_options["gltf/embedded_image_handling"];
state->set_handle_binary_image(enum_option);
Expand Down
Loading