Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use a deterministic depth-first order when managing dependencies #2641

Closed
wants to merge 2 commits into from
Closed

Use a deterministic depth-first order when managing dependencies #2641

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c668ed7
When --dependency_mode is set to STRICT or LOOSE, order dependencies …
ChadKillingsworth Sep 8, 2017
47fbd16
Build a map of provides to inputs so that lookups are constant time.
ChadKillingsworth Oct 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
172 changes: 153 additions & 19 deletions src/com/google/javascript/jscomp/Compiler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,6 @@
import java.io.Serializable;
import java.util.AbstractSet;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
Expand Down Expand Up @@ -1749,7 +1748,19 @@ Node parseInputs() {
options.moduleResolutionMode,
processJsonInputs(inputs));
}
} else {
// Use an empty module loader if we're not actually dealing with modules.
this.moduleLoader = ModuleLoader.EMPTY;
}

if (options.getDependencyOptions().needsManagement()) {
findDependenciesFromEntryPoints(
options.getLanguageIn().toFeatureSet().has(Feature.MODULES),
options.processCommonJSModules,
options.transformAMDToCJSModules);
} else if (options.needsTranspilationFrom(FeatureSet.ES6_MODULES)
|| options.transformAMDToCJSModules
|| options.processCommonJSModules) {
if (options.getLanguageIn().toFeatureSet().has(Feature.MODULES)) {
parsePotentialModules(inputs);
}
Expand Down Expand Up @@ -1782,12 +1793,12 @@ Node parseInputs() {
}
}

if (!inputsToRewrite.isEmpty()) {
forceToEs6Modules(inputsToRewrite.values());
for (CompilerInput input : inputsToRewrite.values()) {
forceInputToPathBasedModule(
input,
options.getLanguageIn().toFeatureSet().has(Feature.MODULES),
options.processCommonJSModules);
}
} else {
// Use an empty module loader if we're not actually dealing with modules.
this.moduleLoader = ModuleLoader.EMPTY;
}

orderInputs();
Expand Down Expand Up @@ -1894,6 +1905,141 @@ void orderInputs() {
}
}

/**
* Find dependencies by recursively traversing each dependency of an input starting with the entry
* points. Causes a full parse of each file, but since the file is reachable by walking the graph,
* this would be required in later compilation passes regardless.
*
* <p>Inputs which are not reachable during graph traversal will be dropped.
*
* <p>If the dependency mode is set to LOOSE, inputs for which the deps package did not find a
* provide statement or detect as a module will be treated as entry points.
*/
void findDependenciesFromEntryPoints(
boolean supportEs6Modules, boolean supportCommonJSModules, boolean supportAmdModules) {
hoistExterns();
List<CompilerInput> entryPoints = new ArrayList<>();
Map<String, CompilerInput> inputsByProvide = new HashMap<>();
Map<String, CompilerInput> inputsByIdentifier = new HashMap<>();
for (CompilerInput input : inputs) {
if (!options.getDependencyOptions().shouldDropMoochers() && input.getProvides().isEmpty()) {
entryPoints.add(input);
}
inputsByIdentifier.put(
ModuleIdentifier.forFile(input.getPath().toString()).toString(), input);
for (String provide : input.getProvides()) {
if (!provide.startsWith("module$")) {
inputsByProvide.put(provide, input);
}
}
}
for (ModuleIdentifier moduleIdentifier : options.getDependencyOptions().getEntryPoints()) {
CompilerInput input = inputsByIdentifier.get(moduleIdentifier.toString());
if (input != null) {
entryPoints.add(input);
}
}

Set<CompilerInput> workingInputSet = new HashSet<>(inputs);
List<CompilerInput> orderedInputs = new ArrayList<>();
for (CompilerInput entryPoint : entryPoints) {
orderedInputs.addAll(
depthFirstDependenciesFromInput(
entryPoint,
false,
workingInputSet,
inputsByIdentifier,
inputsByProvide,
supportEs6Modules,
supportCommonJSModules,
supportAmdModules));
}

// TODO(ChadKillingsworth) Move this into the standard compilation passes
if (supportCommonJSModules) {
for (CompilerInput input : orderedInputs) {
new ProcessCommonJSModules(this).process(null, input.getAstRoot(this), false);
}
}
}

/** For a given input, order it's dependencies in a depth first traversal */
List<CompilerInput> depthFirstDependenciesFromInput(
CompilerInput input,
boolean wasImportedByModule,
Set<CompilerInput> inputs,
Map<String, CompilerInput> inputsByIdentifier,
Map<String, CompilerInput> inputsByProvide,
boolean supportEs6Modules,
boolean supportCommonJSModules,
boolean supportAmdModules) {
List<CompilerInput> orderedInputs = new ArrayList<>();
if (!inputs.remove(input)) {
// It's possible for a module to be included as both a script
// and a module in the same compilation. In these cases, it should
// be forced to be a module.
if (wasImportedByModule && input.getJsModuleType() == CompilerInput.ModuleType.NONE) {
forceInputToPathBasedModule(input, supportEs6Modules, supportCommonJSModules);
}

return orderedInputs;
}

if (supportAmdModules) {
new TransformAMDToCJSModule(this).process(null, input.getAstRoot(this));
}

FindModuleDependencies findDeps =
new FindModuleDependencies(this, supportEs6Modules, supportCommonJSModules);
findDeps.process(input.getAstRoot(this));

// If this input was imported by another module, it is itself a module
// so we force it to be detected as such.
if (wasImportedByModule && input.getJsModuleType() == CompilerInput.ModuleType.NONE) {
forceInputToPathBasedModule(input, supportEs6Modules, supportCommonJSModules);
}

for (String requiredNamespace : input.getRequires()) {
CompilerInput requiredInput = null;
boolean requiredByModuleImport = false;
if (inputsByProvide.containsKey(requiredNamespace)) {
requiredInput = inputsByProvide.get(requiredNamespace);
} else if (inputsByIdentifier.containsKey(requiredNamespace)) {
requiredByModuleImport = true;
requiredInput = inputsByIdentifier.get(requiredNamespace);
}

if (requiredInput != null) {
orderedInputs.addAll(
depthFirstDependenciesFromInput(
requiredInput,
requiredByModuleImport,
inputs,
inputsByIdentifier,
inputsByProvide,
supportEs6Modules,
supportCommonJSModules,
supportAmdModules));
}
}
orderedInputs.add(input);
return orderedInputs;
}

private void forceInputToPathBasedModule(
CompilerInput input, boolean supportEs6Modules, boolean supportCommonJSModules) {

if (supportEs6Modules) {
FindModuleDependencies findDeps =
new FindModuleDependencies(this, supportEs6Modules, supportCommonJSModules);
findDeps.convertToEs6Module(input.getAstRoot(this));
input.setJsModuleType(CompilerInput.ModuleType.ES6);
} else if (supportCommonJSModules) {
new ProcessCommonJSModules(this).process(null, input.getAstRoot(this), true);
input.setJsModuleType(CompilerInput.ModuleType.COMMONJS);
}
}

/**
* Hoists inputs with the @externs annotation into the externs list.
*/
Expand Down Expand Up @@ -2041,18 +2187,6 @@ Map<String, String> processJsonInputs(List<CompilerInput> inputsToProcess) {
return rewriteJson.getPackageJsonMainEntries();
}

void forceToEs6Modules(Collection<CompilerInput> inputsToProcess) {
for (CompilerInput input : inputsToProcess) {
input.setCompiler(this);
input.addProvide(input.getPath().toModuleName());
Node root = input.getAstRoot(this);
if (root == null) {
continue;
}
Es6RewriteModules moduleRewriter = new Es6RewriteModules(this);
moduleRewriter.forceToEs6Module(root);
}
}

private List<CompilerInput> parsePotentialModules(List<CompilerInput> inputsToProcess) {
List<CompilerInput> filteredInputs = new ArrayList<>();
Expand Down Expand Up @@ -2091,7 +2225,7 @@ void processAMDAndCommonJSModules() {
new TransformAMDToCJSModule(this).process(null, root);
}
if (options.processCommonJSModules) {
ProcessCommonJSModules cjs = new ProcessCommonJSModules(this, true);
ProcessCommonJSModules cjs = new ProcessCommonJSModules(this);
cjs.process(null, root);
}
}
Expand Down
45 changes: 38 additions & 7 deletions src/com/google/javascript/jscomp/CompilerInput.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,9 @@ public class CompilerInput implements SourceAst, DependencyInfo {
private DependencyInfo dependencyInfo;
private final List<String> extraRequires = new ArrayList<>();
private final List<String> extraProvides = new ArrayList<>();
private final List<String> orderedRequires = new ArrayList<>();
private boolean hasFullParseDependencyInfo = false;
private ModuleType jsModuleType = ModuleType.NONE;

// An AbstractCompiler for doing parsing.
// We do not want to persist this across serialized state.
Expand Down Expand Up @@ -151,6 +154,10 @@ public void setCompiler(AbstractCompiler compiler) {
/** Gets a list of types depended on by this input. */
@Override
public Collection<String> getRequires() {
if (hasFullParseDependencyInfo) {
return orderedRequires;
}

return getDependencyInfo().getRequires();
}

Expand Down Expand Up @@ -182,19 +189,36 @@ Collection<String> getKnownProvides() {
extraProvides);
}

// TODO(nicksantos): Remove addProvide/addRequire/removeRequire once
// there is better support for discovering non-closure dependencies.

/**
* Registers a type that this input defines.
* Registers a type that this input defines. Includes both explicitly declared namespaces via
* goog.provide and goog.module calls as well as implicit namespaces provided by module rewriting.
*/
public void addProvide(String provide) {
extraProvides.add(provide);
}

/**
* Registers a type that this input depends on.
*/
/** Registers a type that this input depends on in the order seen in the file. */
public boolean addOrderedRequire(String require) {
if (!orderedRequires.contains(require)) {
orderedRequires.add(require);
return true;
}
return false;
}

public void setHasFullParseDependencyInfo(boolean hasFullParseDependencyInfo) {
this.hasFullParseDependencyInfo = hasFullParseDependencyInfo;
}

public ModuleType getJsModuleType() {
return jsModuleType;
}

public void setJsModuleType(ModuleType moduleType) {
jsModuleType = moduleType;
}

/** Registers a type that this input depends on. */
public void addRequire(String require) {
extraRequires.add(require);
}
Expand Down Expand Up @@ -484,4 +508,11 @@ ModulePath getPath() {
public void reset() {
this.module = null;
}

public enum ModuleType {
NONE,
GOOG_MODULE,
ES6,
COMMONJS
}
}
4 changes: 2 additions & 2 deletions src/com/google/javascript/jscomp/DependencyOptions.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import java.io.Serializable;
import java.util.Collection;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.Set;

/**
Expand All @@ -43,7 +43,7 @@ public final class DependencyOptions implements Serializable {
private boolean sortDependencies = false;
private boolean pruneDependencies = false;
private boolean dropMoochers = false;
private final Set<ModuleIdentifier> entryPoints = new HashSet<>();
private final Set<ModuleIdentifier> entryPoints = new LinkedHashSet<>();

/**
* Enables or disables dependency sorting mode.
Expand Down
Loading