diff --git a/.github/workflows/lsp-tests.yml b/.github/workflows/lsp-tests.yml index a724ed42e1..115924d686 100644 --- a/.github/workflows/lsp-tests.yml +++ b/.github/workflows/lsp-tests.yml @@ -69,7 +69,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: "3.10" + python-version: "3.12" - name: Run language server Python tests without PyLint run: ./gradlew core:integrationTest --tests org.lflang.tests.lsp.LspTests.pythonValidationTestSyntaxOnly core:integrationTestCodeCoverageReport - name: Install pylint diff --git a/.github/workflows/py-tests.yml b/.github/workflows/py-tests.yml index 3a6bb1485e..8a29a886bb 100644 --- a/.github/workflows/py-tests.yml +++ b/.github/workflows/py-tests.yml @@ -36,7 +36,7 @@ jobs: - name: Setup Python uses: actions/setup-python@v4 with: - python-version: '3.10' + python-version: '3.12' - name: Install dependencies OS X run: | brew install coreutils diff --git a/core/src/main/java/org/lflang/federated/extensions/PythonExtension.java b/core/src/main/java/org/lflang/federated/extensions/PythonExtension.java index 13c6a46fa2..2b6571d4dd 100644 --- a/core/src/main/java/org/lflang/federated/extensions/PythonExtension.java +++ b/core/src/main/java/org/lflang/federated/extensions/PythonExtension.java @@ -85,6 +85,11 @@ protected String generateSerializationIncludes( return code.getCode(); } + @Override + public String getNetworkBufferType() { + return "PyObject*"; + } + @Override public String generateNetworkSenderBody( VarRef sendingPort, diff --git a/core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java b/core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java index cfb64a0a65..d187dc98f5 100644 --- a/core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CCmakeGenerator.java @@ -69,7 +69,7 @@ public class CCmakeGenerator { private final FileConfig fileConfig; private final List additionalSources; - private final SetUpMainTarget setUpMainTarget; + private SetUpMainTarget setUpMainTarget; private final String installCode; public CCmakeGenerator(FileConfig fileConfig, List additionalSources) { @@ -90,6 +90,15 @@ public CCmakeGenerator( this.installCode = installCode; } + /** + * Set the code generator for the CMake main target. + * + * @param setUpMainTarget + */ + public void setCmakeGenerator(SetUpMainTarget setUpMainTarget) { + this.setUpMainTarget = setUpMainTarget; + } + /** * Generate the contents of a CMakeLists.txt that builds the provided LF C 'sources'. Any error * will be reported in the 'errorReporter'. diff --git a/core/src/main/java/org/lflang/generator/c/CGenerator.java b/core/src/main/java/org/lflang/generator/c/CGenerator.java index 8adaa02eeb..2c21ae7b7f 100644 --- a/core/src/main/java/org/lflang/generator/c/CGenerator.java +++ b/core/src/main/java/org/lflang/generator/c/CGenerator.java @@ -319,7 +319,7 @@ public class CGenerator extends GeneratorBase { private final CTypes types; - private final CCmakeGenerator cmakeGenerator; + protected CCmakeGenerator cmakeGenerator; protected CGenerator( LFGeneratorContext context, diff --git a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java index f486c64a47..71138ec187 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonGenerator.java @@ -65,6 +65,7 @@ import org.lflang.target.Target; import org.lflang.target.property.DockerProperty; import org.lflang.target.property.ProtobufsProperty; +import org.lflang.target.property.PythonVersionProperty; import org.lflang.util.FileUtil; import org.lflang.util.LFCommand; @@ -82,7 +83,7 @@ * * @author Soroush Bateni */ -public class PythonGenerator extends CGenerator { +public class PythonGenerator extends CGenerator implements CCmakeGenerator.SetUpMainTarget { // Used to add statements that come before reactor classes and user code private final CodeBuilder pythonPreamble = new CodeBuilder(); @@ -90,6 +91,9 @@ public class PythonGenerator extends CGenerator { // Used to add module requirements to setup.py (delimited with ,) private final List pythonRequiredModules = new ArrayList<>(); + /** Indicator that we have already generated top-level preambles. */ + private Set generatedTopLevelPreambles = new HashSet(); + private final PythonTypes types; public PythonGenerator(LFGeneratorContext context) { @@ -104,8 +108,9 @@ public PythonGenerator(LFGeneratorContext context) { "lib/python_tag.c", "lib/python_time.c", "lib/pythontarget.c"), - PythonGenerator::setUpMainTarget, + null, // Temporarily, because can't pass this. generateCmakeInstall(context.getFileConfig()))); + cmakeGenerator.setCmakeGenerator(this); } private PythonGenerator( @@ -186,6 +191,7 @@ public String generatePythonCode(String pyModuleName) { "\n", "import os", "import sys", + "print(\"******* Using Python version: %s.%s.%s\" % sys.version_info[:3])", "sys.path.append(os.path.dirname(__file__))", "# List imported names, but do not use pylint's --extension-pkg-allow-list option", "# so that these names will be assumed present without having to compile and install.", @@ -270,7 +276,12 @@ protected String generateTopLevelPreambles(Reactor ignored) { models.add((Model) ASTUtils.toDefinition(this.mainDef.getReactorClass()).eContainer()); } for (Model m : models) { - pythonPreamble.pr(PythonPreambleGenerator.generatePythonPreambles(m.getPreambles())); + // In the generated Python code, unlike C, all reactors go into the same file. + // Therefore, we do not need to generate this if it has already been generated. + if (!generatedTopLevelPreambles.contains(m)) { + generatedTopLevelPreambles.add(m); + pythonPreamble.pr(PythonPreambleGenerator.generatePythonPreambles(m.getPreambles())); + } } return PythonPreambleGenerator.generateCIncludeStatements( targetConfig, targetLanguageIsCpp(), hasModalReactors); @@ -486,9 +497,6 @@ protected void generateUserPreamblesForReactor(Reactor reactor, CodeBuilder src) @Override protected void generateReactorClassHeaders( TypeParameterizedReactor tpr, String headerName, CodeBuilder header, CodeBuilder src) { - header.pr( - PythonPreambleGenerator.generateCIncludeStatements( - targetConfig, targetLanguageIsCpp(), hasModalReactors)); super.generateReactorClassHeaders(tpr, headerName, header, src); } @@ -639,15 +647,27 @@ protected void additionalPostProcessingForModes() { PythonModeGenerator.generateResetReactionsIfNeeded(reactors); } - private static String setUpMainTarget( - boolean hasMain, String executableName, Stream cSources) { + public String getCmakeCode(boolean hasMain, String executableName, Stream cSources) { + // According to https://cmake.org/cmake/help/latest/module/FindPython.html#hints, the following + // should work to select the version of Python given in your virtual environment. + // However, this does not work for me (macOS Sequoia 15.0.1). + // As a consequence, the python-version target property can be used to specify the exact Python + // version. + var pythonVersion = + "3.10.0"; // Allows 3.10 or later. Change to "3.10.0...<3.11.0" to require 3.10 by default. + if (targetConfig.isSet(PythonVersionProperty.INSTANCE)) { + pythonVersion = targetConfig.get(PythonVersionProperty.INSTANCE) + " EXACT"; + } return (""" set(CMAKE_POSITION_INDEPENDENT_CODE ON) add_compile_definitions(_PYTHON_TARGET_ENABLED) add_subdirectory(core) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${CMAKE_SOURCE_DIR}) set(LF_MAIN_TARGET ) - find_package(Python 3.10.0...<3.11.0 REQUIRED COMPONENTS Interpreter Development) + set(Python_FIND_VIRTUALENV FIRST) + set(Python_FIND_STRATEGY LOCATION) + set(Python_FIND_FRAMEWORK LAST) + find_package(Python REQUIRED COMPONENTS Interpreter Development) Python_add_library( ${LF_MAIN_TARGET} MODULE @@ -667,7 +687,8 @@ private static String setUpMainTarget( target_link_libraries(${LF_MAIN_TARGET} PRIVATE ${Python_LIBRARIES}) target_compile_definitions(${LF_MAIN_TARGET} PUBLIC MODULE_NAME=) """) - .replace("", generatePythonModuleName(executableName)); + .replace("", generatePythonModuleName(executableName)) + .replace("", pythonVersion); // The use of fileConfig.name will break federated execution, but that's fine } @@ -677,6 +698,9 @@ private static String generateCmakeInstall(FileConfig fileConfig) { // need to replace '\' with '\\' on Windwos for proper escaping in cmake final var pyMainName = pyMainPath.toString().replace("\\", "\\\\"); return """ + if (NOT DEFINED CMAKE_INSTALL_BINDIR) + set(CMAKE_INSTALL_BINDIR "bin") + endif() if(WIN32) file(GENERATE OUTPUT .bat CONTENT "@echo off diff --git a/core/src/main/java/org/lflang/generator/python/PythonPortGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonPortGenerator.java index d8ece7d206..aa1e9a21c3 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonPortGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonPortGenerator.java @@ -177,7 +177,6 @@ public static String generatePythonListForContainedBank( " }", " /* Release the thread. No Python API allowed beyond this point. */", " PyGILState_Release(gstate);", - " Py_FinalizeEx();", " exit(1);", "}", "for (int i = 0; i < " + generateWidthVariable(reactorName) + "; i++) {", @@ -193,7 +192,6 @@ public static String generatePythonListForContainedBank( " }", " /* Release the thread. No Python API allowed beyond this point. */", " PyGILState_Release(gstate);", - " Py_FinalizeEx();", " exit(1);", " }", "}"); diff --git a/core/src/main/java/org/lflang/generator/python/PythonPreambleGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonPreambleGenerator.java index 8194f78c63..692bf3d747 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonPreambleGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonPreambleGenerator.java @@ -44,8 +44,8 @@ public static String generateCDefineDirectives( public static String generateCIncludeStatements( TargetConfig targetConfig, boolean CCppMode, boolean hasModalReactors) { CodeBuilder code = new CodeBuilder(); - code.pr(CPreambleGenerator.generateIncludeStatements(targetConfig, CCppMode)); code.pr("#include \"pythontarget.h\""); + code.pr(CPreambleGenerator.generateIncludeStatements(targetConfig, CCppMode)); if (hasModalReactors) { code.pr("#include \"include/modal_models/definitions.h\""); } diff --git a/core/src/main/java/org/lflang/generator/python/PythonReactionGenerator.java b/core/src/main/java/org/lflang/generator/python/PythonReactionGenerator.java index d0a1bffac7..c44475f5f8 100644 --- a/core/src/main/java/org/lflang/generator/python/PythonReactionGenerator.java +++ b/core/src/main/java/org/lflang/generator/python/PythonReactionGenerator.java @@ -102,10 +102,16 @@ private static String generateCPythonFunctionCaller( + "." + pythonFunctionName + "\");", + "PyObject *arglist = Py_BuildValue(\"(" + + "O".repeat(pyObjects.size()) + + ")\"" + + pyObjectsJoined + + ");", "PyObject *rValue = PyObject_CallObject(", " self->" + cpythonFunctionName + ", ", - " Py_BuildValue(\"(" + "O".repeat(pyObjects.size()) + ")\"" + pyObjectsJoined + ")", + " arglist", ");", + "Py_DECREF(arglist);", "if (rValue == NULL) {", " lf_print_error(\"FATAL: Calling reaction " + reactorDeclName diff --git a/core/src/main/java/org/lflang/target/Target.java b/core/src/main/java/org/lflang/target/Target.java index c99ed4a0d3..81adb25d3f 100644 --- a/core/src/main/java/org/lflang/target/Target.java +++ b/core/src/main/java/org/lflang/target/Target.java @@ -26,39 +26,7 @@ import java.util.Set; import net.jcip.annotations.Immutable; import org.lflang.lf.TargetDecl; -import org.lflang.target.property.AuthProperty; -import org.lflang.target.property.BuildCommandsProperty; -import org.lflang.target.property.BuildTypeProperty; -import org.lflang.target.property.CargoDependenciesProperty; -import org.lflang.target.property.CargoFeaturesProperty; -import org.lflang.target.property.ClockSyncModeProperty; -import org.lflang.target.property.ClockSyncOptionsProperty; -import org.lflang.target.property.CmakeIncludeProperty; -import org.lflang.target.property.CompileDefinitionsProperty; -import org.lflang.target.property.CompilerProperty; -import org.lflang.target.property.CoordinationOptionsProperty; -import org.lflang.target.property.CoordinationProperty; -import org.lflang.target.property.DockerProperty; -import org.lflang.target.property.ExportDependencyGraphProperty; -import org.lflang.target.property.ExternalRuntimePathProperty; -import org.lflang.target.property.FilesProperty; -import org.lflang.target.property.KeepaliveProperty; -import org.lflang.target.property.NoRuntimeValidationProperty; -import org.lflang.target.property.NoSourceMappingProperty; -import org.lflang.target.property.PlatformProperty; -import org.lflang.target.property.PrintStatisticsProperty; -import org.lflang.target.property.ProtobufsProperty; -import org.lflang.target.property.Ros2DependenciesProperty; -import org.lflang.target.property.Ros2Property; -import org.lflang.target.property.RuntimeVersionProperty; -import org.lflang.target.property.RustIncludeProperty; -import org.lflang.target.property.SchedulerProperty; -import org.lflang.target.property.SingleFileProjectProperty; -import org.lflang.target.property.SingleThreadedProperty; -import org.lflang.target.property.TracePluginProperty; -import org.lflang.target.property.TracingProperty; -import org.lflang.target.property.VerifyProperty; -import org.lflang.target.property.WorkersProperty; +import org.lflang.target.property.*; /** * Enumeration of targets and their associated properties. @@ -634,6 +602,7 @@ public void initialize(TargetConfig config) { KeepaliveProperty.INSTANCE, NoSourceMappingProperty.INSTANCE, ProtobufsProperty.INSTANCE, + PythonVersionProperty.INSTANCE, SchedulerProperty.INSTANCE, SingleThreadedProperty.INSTANCE, TracingProperty.INSTANCE, diff --git a/core/src/main/java/org/lflang/target/property/PythonVersionProperty.java b/core/src/main/java/org/lflang/target/property/PythonVersionProperty.java new file mode 100644 index 0000000000..af8d0f2104 --- /dev/null +++ b/core/src/main/java/org/lflang/target/property/PythonVersionProperty.java @@ -0,0 +1,17 @@ +package org.lflang.target.property; + +/** A specific Python version to use. */ +public final class PythonVersionProperty extends StringProperty { + + /** Singleton target property instance. */ + public static final PythonVersionProperty INSTANCE = new PythonVersionProperty(); + + private PythonVersionProperty() { + super(); + } + + @Override + public String name() { + return "python-version"; + } +} diff --git a/core/src/main/resources/lib/c/reactor-c b/core/src/main/resources/lib/c/reactor-c index 862e0b9617..7acd2f9618 160000 --- a/core/src/main/resources/lib/c/reactor-c +++ b/core/src/main/resources/lib/c/reactor-c @@ -1 +1 @@ -Subproject commit 862e0b9617ca55b4fe5f0b2248c63846a8630d3d +Subproject commit 7acd2f9618d2e983749cdcf66e990c993372d5b8 diff --git a/test/Python/src/serialization/CustomSerializer.lf b/test/Python/src/serialization/CustomSerializer.lf index adcec6a350..ee1f112ed1 100644 --- a/test/Python/src/serialization/CustomSerializer.lf +++ b/test/Python/src/serialization/CustomSerializer.lf @@ -5,7 +5,10 @@ target Python { } preamble {= + # Note that both federates will try to install the pickle_serializer package. One will likely fail, + # but the other will succeed. os.system("pip install ./src/serialization/pickle_serializer/ --user") + import pickle_serializer =} reactor Client { @@ -66,5 +69,5 @@ federated reactor { client = new Client() server = new Server() server.server_message -> client.server_message after 100 ms serializer "pickle_serializer" - client.client_message -> server.client_message serializer "pickle_serializer" + client.client_message -> server.client_message after 100 ms serializer "pickle_serializer" }