Skip to content

Commit

Permalink
Merge pull request #221 from warsaw/zipapp-refactor
Browse files Browse the repository at this point in the history
Refactor to support generic Python "zipapps"
  • Loading branch information
Ethan Hall authored Apr 23, 2018
2 parents 24ea886 + 5a49834 commit 5f446bd
Show file tree
Hide file tree
Showing 12 changed files with 197 additions and 58 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -21,36 +21,56 @@
import java.io.File;


public class PexExtension {

private File pexCache;
private boolean fatPex = OperatingSystem.current().isWindows(); //Defaulting to fat pex's on windows
public class PexExtension implements ZipappExtension {
private File cache;
// Default to fat zipapps on Windows, since our wrappers are fairly POSIX specific.
private boolean isFat = OperatingSystem.current().isWindows();
private boolean pythonWrapper = true;

public PexExtension(Project project) {
pexCache = new File(project.getBuildDir(), "pex-cache");
this.cache = new File(project.getBuildDir(), "pex-cache");
}

public File getPexCache() {
return pexCache;
return cache;
}

public void setPexCache(File pexCache) {
this.pexCache = pexCache;
cache = pexCache;
}

// These are kept for API backward compatibility.

/**
* @return when <code>true</code>, then skinny pex's will be used.
*/
@Deprecated
public boolean isFatPex() {
return fatPex;
return isFat();
}

/**
* @param fatPex when <code>true</code>, wrappers will be made all pointing to a single pex file.
*/
@Deprecated
public void setFatPex(boolean fatPex) {
this.fatPex = fatPex;
isFat = fatPex;
}

// Use these properties instead.

/**
* @return when <code>true</code>, then skinny pex's will be used.
*/
public boolean isFat() {
return isFat;
}

/**
* @param fat when <code>true</code>, wrappers will be made all pointing to a single pex file.
*/
public void setIsFat(boolean isFat) {
this.isFat = isFat;
}

/**
Expand All @@ -65,4 +85,12 @@ public boolean isPythonWrapper() {
public void setPythonWrapper(boolean pythonWrapper) {
this.pythonWrapper = pythonWrapper;
}

public File getCache() {
return cache;
}

public void setCache(File cache) {
this.cache = cache;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,14 +13,17 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.gradle.python.util.internal.pex;
package com.linkedin.gradle.python.extension;

public interface PexGenerator {

public interface ZipappExtension {
/**
* When called will generate entry point files for a pex.
*
* @throws Exception when failing to build entry point
* @return when <code>true</code>, then skinny pex's will be used.
*/
void buildEntryPoints() throws Exception;
public boolean isFat();

/**
* @param fat when <code>true</code>, wrappers will be made all pointing to a single pex file.
*/
public void setIsFat(boolean isFat);
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@
import com.linkedin.gradle.python.util.ExtensionUtils;
import com.linkedin.gradle.python.util.internal.pex.FatPexGenerator;
import com.linkedin.gradle.python.util.internal.pex.ThinPexGenerator;
import com.linkedin.gradle.python.util.pex.DefaultEntryPointTemplateProvider;
import com.linkedin.gradle.python.util.pex.EntryPointTemplateProvider;
import com.linkedin.gradle.python.util.pex.DefaultPexEntryPointTemplateProvider;
import com.linkedin.gradle.python.util.zipapp.EntryPointTemplateProvider;
import org.apache.commons.io.FileUtils;
import org.gradle.api.DefaultTask;
import org.gradle.api.Project;
Expand Down Expand Up @@ -58,7 +58,7 @@
public class BuildPexTask extends DefaultTask implements FailureReasonProvider {

private Map<String, String> additionalProperties;
private EntryPointTemplateProvider templateProvider = new DefaultEntryPointTemplateProvider();
private EntryPointTemplateProvider templateProvider = new DefaultPexEntryPointTemplateProvider();
private TeeOutputContainer container = new TeeOutputContainer(System.out, System.err);
private List<String> pexOptions = new ArrayList<>();

Expand Down Expand Up @@ -87,7 +87,7 @@ public void buildPex() throws Exception {

deployableExtension.getDeployableBuildDir().mkdirs();

if (pexExtension.isFatPex()) {
if (pexExtension.isFat()) {
new FatPexGenerator(project, pexOptions).buildEntryPoints();
} else {
new ThinPexGenerator(project, pexOptions, templateProvider, additionalProperties).buildEntryPoints();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public void buildWebapp() throws IOException, ClassNotFoundException {
Project project = getProject();
PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(project, PexExtension.class);

if (pexExtension.isFatPex()) {
if (pexExtension.isFat()) {
new FatPexGenerator(project, pexOptions).buildEntryPoint(
PexFileUtil.createFatPexFilename(executable.getName()), entryPoint, null);
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,18 +15,22 @@
*/
package com.linkedin.gradle.python.util.internal.pex;

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.util.EntryPointHelpers;
import com.linkedin.gradle.python.util.PexFileUtil;
import com.linkedin.gradle.python.util.internal.zipapp.ZipappGenerator;
import com.linkedin.gradle.python.util.pip.PipFreezeAction;
import org.gradle.api.Project;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.process.ExecResult;

import java.util.HashMap;
import java.util.List;
import java.util.Map;

public class FatPexGenerator implements PexGenerator {

public class FatPexGenerator implements ZipappGenerator {

private static final Logger logger = Logging.getLogger(FatPexGenerator.class);

Expand All @@ -38,6 +42,12 @@ public FatPexGenerator(Project project, List<String> pexOptions) {
this.pexOptions = pexOptions;
}

@Override
public Map<String, String> buildSubstitutions(PythonExtension extension, String entry) {
// Not used for fat pexes.
return new HashMap<>();
}

@Override
public void buildEntryPoints() {
Map<String, String> dependencies = new PipFreezeAction(project).getDependencies();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,53 +17,54 @@

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.extension.DeployableExtension;
import com.linkedin.gradle.python.extension.PexExtension;
import com.linkedin.gradle.python.util.EntryPointHelpers;
import com.linkedin.gradle.python.util.ExtensionUtils;
import com.linkedin.gradle.python.util.PexFileUtil;
import com.linkedin.gradle.python.util.entrypoint.EntryPointWriter;
import com.linkedin.gradle.python.util.pex.EntryPointTemplateProvider;
import com.linkedin.gradle.python.util.internal.zipapp.DefaultTemplateProviderOptions;
import com.linkedin.gradle.python.util.internal.zipapp.ThinZipappGenerator;
import com.linkedin.gradle.python.util.pip.PipFreezeAction;
import com.linkedin.gradle.python.util.zipapp.EntryPointTemplateProvider;
import org.gradle.api.Project;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;
import org.gradle.process.ExecResult;

import java.io.File;
import java.util.HashMap;
import java.util.List;
import java.util.Map;


public class ThinPexGenerator implements PexGenerator {

private static final Logger logger = Logging.getLogger(ThinPexGenerator.class);

private final Project project;
private final List<String> pexOptions;
private final EntryPointTemplateProvider templateProvider;
private final Map<String, String> extraProperties;
public class ThinPexGenerator extends ThinZipappGenerator {

public ThinPexGenerator(
Project project,
List<String> pexOptions,
EntryPointTemplateProvider templateProvider,
Map<String, String> extraProperties) {
this.project = project;
this.pexOptions = pexOptions;
this.templateProvider = templateProvider;
this.extraProperties = extraProperties == null ? new HashMap<>() : extraProperties;

super(project, pexOptions, templateProvider, extraProperties);
logger = Logging.getLogger(ThinPexGenerator.class);
}

@Override
public Map<String, String> buildSubstitutions(PythonExtension extension, String entry) {
Map<String, String> substitutions = super.buildSubstitutions(extension, entry);
substitutions.put("realPex", PexFileUtil.createThinPexFilename(project.getName()));
return substitutions;
}

@Override
public void buildEntryPoints() throws Exception {
PythonExtension extension = ExtensionUtils.getPythonExtension(project);
PexExtension pexExtension = ExtensionUtils.getPythonComponentExtension(extension, PexExtension.class);
DeployableExtension deployableExtension = ExtensionUtils.getPythonComponentExtension(
extension, DeployableExtension.class);

Map<String, String> dependencies = new PipFreezeAction(project).getDependencies();

PexExecSpecAction action = PexExecSpecAction.withOutEntryPoint(
project, project.getName(), pexOptions, dependencies);
project, project.getName(), options, dependencies);

ExecResult exec = project.exec(action);
new PexExecOutputParser(action, exec).validatePexBuildSuccessfully();
Expand All @@ -73,17 +74,13 @@ public void buildEntryPoints() throws Exception {
String[] split = it.split("=");
String name = split[0].trim();
String entry = split[1].trim();

Map<String, String> propertyMap = new HashMap<>();
propertyMap.putAll(extraProperties);
propertyMap.put("realPex", PexFileUtil.createThinPexFilename(project.getName()));
propertyMap.put("entryPoint", entry);
propertyMap.put("pythonExecutable", extension.getDetails().getSystemPythonInterpreter().getAbsolutePath());
propertyMap.put("toolName", project.getName());
Map<String, String> substitutions = buildSubstitutions(extension, entry);

DefaultTemplateProviderOptions providerOptions = new DefaultTemplateProviderOptions(project, extension, entry);
new EntryPointWriter(project, templateProvider.retrieveTemplate(providerOptions))
.writeEntryPoint(new File(deployableExtension.getDeployableBinDir(), name), propertyMap);
new EntryPointWriter(
project,
templateProvider.retrieveTemplate(providerOptions, pexExtension.isPythonWrapper()))
.writeEntryPoint(new File(deployableExtension.getDeployableBinDir(), name), substitutions);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.gradle.python.util.internal.pex;
package com.linkedin.gradle.python.util.internal.zipapp;

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.util.pex.TemplateProviderOptions;
import com.linkedin.gradle.python.util.zipapp.TemplateProviderOptions;
import org.gradle.api.Project;

/**
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,65 @@
/*
* Copyright 2016 LinkedIn Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.linkedin.gradle.python.util.internal.zipapp;

import com.linkedin.gradle.python.PythonExtension;
import com.linkedin.gradle.python.util.zipapp.EntryPointTemplateProvider;
import org.gradle.api.Project;
import org.gradle.api.logging.Logger;
import org.gradle.api.logging.Logging;

import java.util.HashMap;
import java.util.List;
import java.util.Map;


public class ThinZipappGenerator implements ZipappGenerator {

protected static Logger logger = Logging.getLogger(ThinZipappGenerator.class);

protected final Project project;
protected final List<String> options;
protected final EntryPointTemplateProvider templateProvider;
protected final Map<String, String> extraProperties;

public ThinZipappGenerator(
Project project,
List<String> options,
EntryPointTemplateProvider templateProvider,
Map<String, String> extraProperties) {

this.project = project;
this.options = options;
this.templateProvider = templateProvider;
this.extraProperties = extraProperties == null ? new HashMap<>() : extraProperties;
}

@Override
public Map<String, String> buildSubstitutions(PythonExtension extension, String entry) {
Map<String, String> substitutions = new HashMap<>();
substitutions.putAll(extraProperties);
substitutions.put("entryPoint", entry);
substitutions.put("pythonExecutable", extension.getDetails().getSystemPythonInterpreter().getAbsolutePath());
substitutions.put("toolName", project.getName());
return substitutions;
}

@Override
public void buildEntryPoints() throws Exception {
// Generic zipapps don't build anything, so subclasses should override
// this to build their entry point specific artifacts.
};
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
/*
* Copyright 2016 LinkedIn Corp.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.linkedin.gradle.python.util.internal.zipapp;

import com.linkedin.gradle.python.PythonExtension;
import java.util.Map;


public interface ZipappGenerator {

/**
* When called, create the substitution map for the template generation.
*/
Map<String, String> buildSubstitutions(PythonExtension extension, String entry);

/**
* When called will generate entry point files for a zipapp.
*
* @throws Exception when failing to build entry point
*/
void buildEntryPoints() throws Exception;
}
Loading

0 comments on commit 5f446bd

Please sign in to comment.