Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add gltf-compiler for every internal types #2096

Open
wants to merge 3 commits into
base: main
Choose a base branch
from
Open

Add gltf-compiler for every internal types #2096

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
e4e0440
Add gltf-compiler for every internal types
smartel-legion Jun 16, 2022
fbbe265
Fix tests.
aganea Jun 27, 2022
44d7fc6
Merge branch 'maverick' into city_optim
aganea Jun 27, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 11 additions & 0 deletions .cargo/config.toml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -135,5 +135,16 @@ opt-level = 3
debug = true
debug-assertions = false

[profile.dev.package.image]
opt-level = 3
debug = true
debug-assertions = false

[profile.dev.package.gltf]
opt-level = 3
debug = true
debug-assertions = false


[env]
LGN_TELEMETRY_GRPC_API_KEY = "296bb99c-6921-4233-a353-a29c1045b5f2"
3 changes: 2 additions & 1 deletion .vscode/legionlabs.natvis
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,8 @@
<DisplayString Condition="(*(u64*)&amp;__0 == 0x44e4b6023fb7a8d3)">offline_model({__0,x})</DisplayString>
<DisplayString Condition="(*(u64*)&amp;__0 == 0x5c4b1b522bf5dcb0)">runtime_model({__0,x})</DisplayString>
<DisplayString Condition="(*(u64*)&amp;__0 == 0x0d4207bbf3a2fd08)">gltf({__0,x})</DisplayString>

<DisplayString Condition="(*(u64*)&amp;__0 == 0x491984ba0d752703)">export_gltf({__0,x})</DisplayString>

<DisplayString>{__0,x}</DisplayString>
</Type>

Expand Down
38 changes: 6 additions & 32 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

154 changes: 103 additions & 51 deletions crates/lgn-data-build/src/databuild.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
use std::{
collections::{HashMap, HashSet},
collections::{hash_map::Entry, HashMap, HashSet},
env,
hash::{Hash, Hasher},
io,
Expand Down Expand Up @@ -451,18 +451,34 @@ impl DataBuild {
};

let mut compiler_details = HashMap::new();
for t in unique_transforms {
let (transform, res_path_id) = t;
let (compiler, transform) = self
.compilers
.compilers()
.find_compiler(transform)
.ok_or(Error::CompilerNotFound(transform, res_path_id))?;
let compiler_hash = compiler
.compiler_hash(transform, env)
.await
.map_err(|e| Error::Io(e.into()))?;
compiler_details.insert(transform, (compiler, compiler_hash));
for (transform, res_path_id) in unique_transforms {
// Insert a identity compiler if one exists
let identity_transform = transform.as_identity();
if let Some((identity_compiler, _transform)) =
self.compilers.compilers().find_compiler(identity_transform)
{
if let Entry::Vacant(e) = compiler_details.entry(identity_transform) {
let compiler_hash = identity_compiler
.compiler_hash(identity_transform, env)
.await
.map_err(|e| Error::Io(e.into()))?;
e.insert((identity_compiler, compiler_hash));
}
} else if let Entry::Vacant(e) = compiler_details.entry(transform) {
let (compiler, _transform) = self
.compilers
.compilers()
.find_compiler(transform)
.ok_or_else(|| {
lgn_tracing::error!("Compiler not found {}", transform);
Error::CompilerNotFound(transform, res_path_id)
})?;
let compiler_hash = compiler
.compiler_hash(transform, env)
.await
.map_err(|e| Error::Io(e.into()))?;
e.insert((compiler, compiler_hash));
}
}
compiler_details
};
Expand All @@ -477,7 +493,7 @@ impl DataBuild {
// in the future this should be improved.
//
let mut accumulated_dependencies = vec![];
let mut node_hash = HashMap::<_, (AssetHash, AssetHash)>::new();
let mut node_hash = HashMap::<_, (AssetHash, AssetHash, ResourcePathId)>::new();

let mut compiled_at_node = HashMap::<ResourcePathId, _>::new();
let mut compiled = HashSet::<petgraph::graph::NodeIndex>::new();
Expand Down Expand Up @@ -579,36 +595,54 @@ impl DataBuild {
let mut new_work = vec![];
let num_ready = ready.len();
for compile_node_index in ready {
let compile_node = build_graph.node_weight(compile_node_index).unwrap();
let source_node = build_graph.node_weight(compile_node_index).unwrap().clone();
info!(
"Progress({:?}): {:?} is ready",
compile_node_index, compile_node
compile_node_index, source_node
);
// compile non-source dependencies.
if let Some(direct_dependency) = compile_node.direct_dependency() {
let mut n =
build_graph.neighbors_directed(compile_node_index, petgraph::Incoming);
let direct_dependency_index = n.next().unwrap();

// only one direct dependency supported now. it's ok for the path
// but it needs to be revisited for source (if this ever applies to source).
assert!(n.next().is_none());

assert_eq!(
&direct_dependency,
build_graph.node_weight(direct_dependency_index).unwrap()
);

let transform = compile_node.last_transform().unwrap();
let compile_node = if let Some(transform) = source_node.last_transform() {
// If there's no compiler for this operation but there's an Identity transform on the source, use it instead
if compiler_details.get(&transform).is_none()
&& compiler_details.get(&transform.as_identity()).is_some()
{
ResourcePathId::from(source_node.source_resource())
.push(source_node.source_resource().kind)
} else {
source_node.clone()
}
} else {
// If it's a source node but there's an Identity compiler, replace with Identity transform
let identity_transform = ResourcePathId::from(source_node.source_resource())
.push(source_node.source_resource().kind);
if let Some((_compiler, _compiler_hash)) =
compiler_details.get(&identity_transform.last_transform().unwrap())
{
identity_transform
} else {
source_node.clone()
}
};

// 'name' is dropped as we always compile input as a whole.
let expected_name = compile_node.name();
// compile non-source dependencies.
if let Some(direct_dependency) = compile_node.direct_dependency() {
let compile_node = compile_node.to_unnamed();

// check if the unnamed ResourcePathId has been already compiled and early out.
if let Some(node_index) = compiled_at_node.get(&compile_node) {
node_hash.insert(compile_node_index, *node_hash.get(node_index).unwrap());
node_hash.insert(
compile_node_index,
node_hash.get(node_index).unwrap().clone(),
);

let unnamed = source_node.to_unnamed();
info!(
"Source({:?}) Completed '{}' (reusing result from {:?})",
compile_node_index, source_node, node_index
);
compiled.insert(compile_node_index);
compiling_unnamed.remove(&unnamed);
compiled_unnamed.insert(unnamed);
continue;
}

Expand All @@ -629,13 +663,6 @@ impl DataBuild {
.find_dependencies(&direct_dependency)
.unwrap_or_default();

let (compiler, compiler_hash) = *compiler_details.get(&transform).unwrap();

// todo: not sure if transform is the right thing here. resource_path_id better?
// transform is already defined by the compiler_hash so it seems redundant.
let context_hash =
compute_context_hash(transform, compiler_hash, Self::version());

let source_hash = {
if direct_dependency.is_source() {
//
Expand All @@ -654,19 +681,24 @@ impl DataBuild {
// resource should not read any other resources - but right now
// `accumulated_dependencies` allows to read much more.
//
let (dep_context_hash, dep_source_hash) =
let mut n = build_graph
.neighbors_directed(compile_node_index, petgraph::Incoming);
let direct_dependency_index = n.next().unwrap();
let (dep_context_hash, dep_source_hash, dep_path) =
node_hash.get(&direct_dependency_index).unwrap();

// we can assume there are results of compilation of the `direct_dependency`
let compiled = self
.output_index
.find_compiled(
&direct_dependency.to_unnamed(),
*dep_context_hash,
*dep_source_hash,
)
.find_compiled(dep_path, *dep_context_hash, *dep_source_hash)
.await
.unwrap()
.ok_or_else(|| {
lgn_tracing::error!(
"Output not present for {}",
&direct_dependency
);
Error::OutputNotPresent(direct_dependency.clone(), "".into())
})?
.0;
// can we assume there is a result of a requested name?
// probably no, this should return a compile error.
Expand All @@ -687,7 +719,22 @@ impl DataBuild {
}
};

node_hash.insert(compile_node_index, (context_hash, source_hash));
// Find the compiler and compiler_hash from the last transform
let transform = compile_node.last_transform().unwrap();
let (compiler, context_hash) =
if let Some((compiler, compiler_hash)) = compiler_details.get(&transform) {
(
*compiler,
compute_context_hash(transform, *compiler_hash, Self::version()),
)
} else {
return Err(Error::CompilerNotFound(transform, compile_node.clone()));
};

node_hash.insert(
compile_node_index,
(context_hash, source_hash, compile_node.clone()),
);

let output_index = &self.output_index;
let data_content_provider = Arc::clone(&self.data_content_provider);
Expand All @@ -703,7 +750,9 @@ impl DataBuild {
> = async move {
info!(
"Compiling({:?}) {} ({:?}) ...",
compile_node_index, compile_node, expected_name
compile_node_index,
compile_node,
compile_node.name()
);
let start = std::time::Instant::now();

Expand All @@ -723,7 +772,10 @@ impl DataBuild {
resources.clone(),
)
.await
.map_err(|e| (compile_node_index, e))?;
.map_err(|e| {
lgn_tracing::error!("Failed to compile: {}", e);
(compile_node_index, e)
})?;

info!(
"Compiled({:?}) {:?} ended in {:?}.",
Expand Down
30 changes: 16 additions & 14 deletions crates/lgn-data-compiler/src/compiler_api.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -396,14 +396,15 @@ async fn get_transform_hash(
env: &CompilationEnv,
transform: Transform,
) -> Result<CompilerHash, CompilerError> {
let (compiler, transform) = compilers
.find_compiler(transform)
.ok_or(CompilerError::CompilerNotFound(transform))?;
let (compiler, transform) = compilers.find_compiler(transform).ok_or_else(|| {
lgn_tracing::error!("Not found");
CompilerError::CompilerNotFound(transform)
})?;

let compiler_hash = compiler
.compiler_hash(transform, env)
.await
.map_err(|_e| CompilerError::CompilerNotFound(transform))?;
let compiler_hash = compiler.compiler_hash(transform, env).await.map_err(|e| {
lgn_tracing::error!("{}", e);
CompilerError::CompilerNotFound(transform)
})?;

Ok(compiler_hash)
}
Expand Down Expand Up @@ -508,9 +509,10 @@ async fn run(command: Commands, compilers: CompilerRegistry) -> Result<(), Compi
};

let registry = {
let (compiler, _) = compilers
.find_compiler(transform)
.ok_or(CompilerError::CompilerNotFound(transform))?;
let (compiler, _) = compilers.find_compiler(transform).ok_or_else(|| {
lgn_tracing::error!("Compiler not found");
CompilerError::CompilerNotFound(transform)
})?;

let registry = AssetRegistryOptions::new()
.add_device_cas(Arc::clone(&data_provider), runtime_manifest_id.clone())
Expand All @@ -524,10 +526,10 @@ async fn run(command: Commands, compilers: CompilerRegistry) -> Result<(), Compi

let shell = CompilerNode::new(compilers, registry);

let (compiler, _) = shell
.compilers()
.find_compiler(transform)
.ok_or(CompilerError::CompilerNotFound(transform))?;
let (compiler, _) = shell.compilers().find_compiler(transform).ok_or_else(|| {
lgn_tracing::error!("Compiler not found");
CompilerError::CompilerNotFound(transform)
})?;

let compilation_output = compiler
.compile(
Expand Down
8 changes: 3 additions & 5 deletions crates/lgn-data-compiler/src/compiler_node/compiler_registry.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,11 +161,9 @@ impl fmt::Debug for CompilerRegistry {
impl CompilerRegistry {
/// Returns a reference to the compiler
pub fn find_compiler(&self, transform: Transform) -> Option<(&dyn CompilerStub, Transform)> {
if let Some(compiler_index) = self
.infos
.iter()
.position(|info| info.transform == transform)
{
if let Some(compiler_index) = self.infos.iter().position(|info| {
info.transform == transform || info.transform.is_wildcard_for(&transform)
}) {
let stub_index = self.indices[compiler_index];
return Some((self.compilers[stub_index].as_ref(), transform));
}
Expand Down
Loading