Implement venv

.github/workflows/develop.yml

@@ -10,7 +10,7 @@ jobs:
   build:
 
     runs-on: ubuntu-latest
-    container: thistleman/test:latest
+    container: hipas/gridlabd-venv:develop
 
     steps:
     - uses: actions/checkout@v2
@@ -20,7 +20,10 @@ jobs:
       run: |
         chown -R $(id -u):$(id -g) $PWD
     - name: install gridlabd
-      run: ./install.sh -v -t -p
+      run: |
+        autoreconf -isf
+        ./configure
+        make system
     - name: Install openfido
       run: curl -sL https://raw.githubusercontent.com/openfido/cli/main/install.sh | bash
     - name: Validate build

Makefile.am

@@ -297,12 +297,12 @@ install-exec-hook: index python-requirements docs
 	@echo "  $(DESTDIR)$(bindir)/gridlabd"
 	@echo ""
 	@echo "To make this version the default for all users on this system:"
-	@echo "  $(DESTDIR)$(bindir)/gridlabd version set"
+	@echo "  $(DESTDIR)$(bindir)/gridlabd version set $(shell basename $(exec_prefix))"
 	@echo ""
 
 python-requirements: $(python_requirements)
 	@echo "Updating python requirements..."
-	@$(bindir)/python3 -m pip install -q -r $(subst $(subst ,, ), -r ,$(python_requirements))
+	@$(bindir)/pkgenv/bin/python3 -m pip install -q -r $(subst $(subst ,, ), -r ,$(python_requirements))
 
 reconfigure: distclean
 	autoreconf -isf && $(PWD)/configure --quiet
@@ -325,7 +325,7 @@ $(PREFIX)/docker.img: docker/Dockerfile Makefile
 system: python-install install
 	@echo "Setting $$($(top_srcdir)/build-aux/version.sh --install) to the current system version"
 #   @$(DESTDIR)$(bindir)/gridlabd docker disable > /dev/null
-	@$(DESTDIR)$(bindir)/gridlabd version set
+	@$(DESTDIR)$(bindir)/gridlabd version set $(shell basename $(exec_prefix))
 	@if [ ! "$$(gridlabd --version=install)" == "$$($(top_srcdir)/build-aux/version.sh --install)" ]; then \
 		echo 'WARNING : build and install versions differ; run `make reconfigure` to fix this problem'; \
 	fi

build-aux/python3.sh

@@ -1,9 +1,9 @@
 
 #
 # This should generally not be used and is only for the few C++ and header files making simple python calls.
-exec $GLD_BIN/python3 "$@"
+exec $GLD_BIN/pkgenv/bin/python3 "$@"
 
 # Python modules should start without a hashbang, and instead use the below code without the #s. See gridlabd-convert as an example. 
 # ''':'
-# exec "$GLD_BIN/python3" "$0" "$@"
+# exec "$GLD_BIN/pkgenv/bin/python3" "$0" "$@"
 # :' '''

build-aux/setup-Darwin-17.sh

@@ -4,15 +4,15 @@
 # Set version and paths, using these vars will make future maintenance much better. #Automation
     VERSION=${VERSION:-`build-aux/version.sh --name`}
     VERSION_DIR=$VAR/$VERSION
-    PYTHON_DIR=Python.framework/Versions/Current
-    PYTHON_VER=3.9.13
-    PY_EXE=3.9
+	PKG_PYTHON_DIR=/usr/local/bin
+	VENV_PYTHON_DIR=$VERSION_DIR/bin/pkgenv/bin
+    PYTHON_VER=3.10
 
 export PATH=$VERSION_DIR/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
 
 # Check if python version currently in Applications and update owner
-    if [ -e /Applications/"Python $PY_EXE" ] ; then
-        sudo chown -R ${USER:-root} /Applications/"Python $PY_EXE"
+    if [ -e /Applications/"Python $PYTHON_VER" ] ; then
+        sudo chown -R ${USER:-root} /Applications/"Python $PYTHON_VER"
     fi
 
 brew update || /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
@@ -35,9 +35,10 @@ brew doctor
         sudo mkdir etc
     fi
 
-export DYLD_LIBRARY_PATH=$VERSION_DIR/lib:$DYLD_LIBRARY_PATH
-export LD_LIBRARY_PATH=$VERSION_DIR/lib:$LD_LIBRARY_PATH
-export LIBRARY_PATH=$VERSION_DIR/lib:$LIBRARY_PATH
+export DYLD_LIBRARY_PATH=$VERSION_DIR/lib:$VERSION_DIR/include:$DYLD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$VERSION_DIR/lib:$VERSION_DIR/include:$LD_LIBRARY_PATH
+export LIBRARY_PATH=$VERSION_DIR/lib:$VERSION_DIR/include:$LIBRARY_PATH
+
 
 # build tools
 
@@ -53,82 +54,29 @@ export LIBRARY_PATH=$VERSION_DIR/lib:$LIBRARY_PATH
 
 # Install python $PYTHON_VER
 # python3 support needed as of 4.2
-if [ ! -x $VERSION_DIR/bin/python3 -o "$($VERSION_DIR/bin/python3 --version | cut -f3 -d.)" != "Python $PY_EXE" ]; then
+if [ ! -x /usr/local/bin/python3 ] || [ "$(/usr/local/bin/python3 --version | cut -d' ' -f2 | cut -d. -f1-2)" != "$PYTHON_VER" ]; then
 	echo "installing python $PYTHON_VER and ssl module dependencies"
-	cd $VERSION_DIR/src
-
-	curl https://www.python.org/ftp/python/$PYTHON_VER/Python-$PYTHON_VER.tgz | tar xz
-
-# include python ssl module and dependencies, uses ./Configure instead of ./configure due to custom implementation    
-    curl -L http://xrl.us/installperlosx | bash
-    curl https://www.openssl.org/source/old/1.1.1/openssl-1.1.1n.tar.gz | tar xz
-    cd openssl-1.1.1n
-
-# Needed to build python's ssl module
-    ./Configure --prefix=$VERSION_DIR/openssl --openssldir=$VERSION_DIR/ssl --libdir=lib darwin64-x86_64-cc
-    make
-    make install
 
 # needed for SSL module to make proper connections, as openssl does not actually provide the certificates.
+    brew reinstall [email protected]
     brew install ca-certificates
     CERT_DIR="$(echo /usr/local/Cellar/ca-certificates)"
     unset -v latest
     for file in "$CERT_DIR"/* ; do
         [[ $file -nt $latest ]] && latest=$file
     done
     CERT_DIR=$latest
-    cp $CERT_DIR/share/ca-certificates/* $VERSION_DIR/ssl/cert.pem
+    cp $CERT_DIR/share/ca-certificates/* /usr/local/etc/[email protected]/certs/cert.pem
 
-    if [ ! -f $VERSION_DIR/ssl/cert.pem ] ; then
-        echo "The installer was unable to locate certificates for the python build"
-        echo "Please make sure homebrew can install ca-certificates before retrying build."
-        exit 1
-    fi
+    echo "installing python $PYTHON_VER"
+    brew install python@$PYTHON_VER
 
-	# tar xzf Python-$PYTHON_VER.tgz
-	cd $VERSION_DIR/src/Python-$PYTHON_VER
-
-    export MACOSX_DEPLOYMENT_TARGET=$( sw_vers -productVersion | sed 's/..$//' )
-     export PKG_CONFIG_PATH="$VERSION_DIR/lib/pkgconfig:/usr/local/opt/tcl-tk/lib/pkgconfig:$pythonLocation/lib/pkgconfig"
-	./configure --prefix=$VERSION_DIR \
-    --enable-framework=$VERSION_DIR \
-    --with-openssl=$VERSION_DIR/openssl \
-    --with-pydebug \
-    --with-computed-gotos \
-    --with-tcltk-libs="$(pkg-config --libs tcl tk)" \
-    --with-tcltk-includes="$(pkg-config --cflags tcl tk)" \
-    CFLAGS="-I/usr/local/include " \
-    LDFLAGS="-L/usr/local/lib -L/usr/local/opt/zlib/lib" \
-    CPPFLAGS="-I/usr/local/include -I/usr/local/opt/zlib/include"
-
-    # MAKEFLAGS need to be set to an empty string to prevent a bug with framework builds. Multiprocessor python framework builds fail when a -j value is set.
-    export MAKEFLAGS=""
-	make
-	make install
-
-	sudo ln -s $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d $VERSION_DIR/$PYTHON_DIR/bin/python3
-    $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d -m ensurepip --upgrade
-
-	sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d-config $VERSION_DIR/$PYTHON_DIR/bin/python3-config
-	sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/pydoc${PY_EXE} $VERSION_DIR/$PYTHON_DIR/bin/pydoc
-	sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/idle${PY_EXE} $VERSION_DIR/$PYTHON_DIR/bin/idle
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/* $VERSION_DIR/bin
-
-    $VERSION_DIR/bin/python3 -m ensurepip --upgrade
-	$VERSION_DIR/bin/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-    $VERSION_DIR/bin/python3 -m pip install build
-    $VERSION_DIR/bin/python3 -m pip install pyproj
-
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/* $VERSION_DIR/bin
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/include/* $VERSION_DIR/include
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/lib/* $VERSION_DIR/lib
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/share/* $VERSION_DIR/share
 fi
 
 # check for successful python build
-if [ ! -x $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d ]; then
+if [ ! -x /usr/local/bin/python$PYTHON_VER ]; then
     echo "Could not locate python executable in"
-    echo "PYTHON LOCATION: $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d"
+    echo "PYTHON LOCATION: /usr/local/bin/python$PYTHON_VER"
     echo "Exiting build."
     exit 1
 fi
@@ -151,7 +99,6 @@ brew install gdal
 
 # libgeos
     brew install geos
-    cp /usr/local/opt/geos/lib/libgeos* $VERSION_DIR/lib
 
     if test ! -e /usr/local/lib; then
         cd /usr/local

build-aux/setup-Darwin-18.sh

@@ -4,15 +4,15 @@
 # Set version and paths, using these vars will make future maintenance much better. #Automation
     VERSION=${VERSION:-`build-aux/version.sh --name`}
     VERSION_DIR=$VAR/$VERSION
-    PYTHON_DIR=Python.framework/Versions/Current
-    PYTHON_VER=3.9.13
-    PY_EXE=3.9
+	PKG_PYTHON_DIR=/usr/local/bin
+	VENV_PYTHON_DIR=$VERSION_DIR/bin/pkgenv/bin
+    PYTHON_VER=3.10
 
 export PATH=$VERSION_DIR/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin
 
 # Check if python version currently in Applications and update owner
-    if [ -e /Applications/"Python $PY_EXE" ] ; then
-        sudo chown -R ${USER:-root} /Applications/"Python $PY_EXE"
+    if [ -e /Applications/"Python $PYTHON_VER" ] ; then
+        sudo chown -R ${USER:-root} /Applications/"Python $PYTHON_VER"
     fi
 
 brew update || /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)"
@@ -35,12 +35,11 @@ brew doctor
         sudo mkdir etc
     fi
 
+export DYLD_LIBRARY_PATH=$VERSION_DIR/lib:$VERSION_DIR/include:$DYLD_LIBRARY_PATH
+export LD_LIBRARY_PATH=$VERSION_DIR/lib:$VERSION_DIR/include:$LD_LIBRARY_PATH
+export LIBRARY_PATH=$VERSION_DIR/lib:$VERSION_DIR/include:$LIBRARY_PATH
 
 
-export DYLD_LIBRARY_PATH=$VERSION_DIR/lib:$DYLD_LIBRARY_PATH
-export LD_LIBRARY_PATH=$VERSION_DIR/lib:$LD_LIBRARY_PATH
-export LIBRARY_PATH=$VERSION_DIR/lib:$LIBRARY_PATH
-
 # build tools
 
     brew install autoconf automake libtool gnu-sed gawk git
@@ -55,82 +54,29 @@ export LIBRARY_PATH=$VERSION_DIR/lib:$LIBRARY_PATH
 
 # Install python $PYTHON_VER
 # python3 support needed as of 4.2
-if [ ! -x $VERSION_DIR/bin/python3 -o "$($VERSION_DIR/bin/python3 --version | cut -f3 -d.)" != "Python $PY_EXE" ]; then
+if [ ! -x /usr/local/bin/python3 ] || [ "$(/usr/local/bin/python3 --version | cut -d' ' -f2 | cut -d. -f1-2)" != "$PYTHON_VER" ]; then
 	echo "installing python $PYTHON_VER and ssl module dependencies"
-	cd $VERSION_DIR/src
-
-	curl https://www.python.org/ftp/python/$PYTHON_VER/Python-$PYTHON_VER.tgz | tar xz
-
-# include python ssl module and dependencies, uses ./Configure instead of ./configure due to custom implementation    
-    curl -L http://xrl.us/installperlosx | bash
-    curl https://www.openssl.org/source/old/1.1.1/openssl-1.1.1n.tar.gz | tar xz
-    cd openssl-1.1.1n
-
-# Needed to build python's ssl module
-    ./Configure --prefix=$VERSION_DIR/openssl --openssldir=$VERSION_DIR/ssl --libdir=lib darwin64-x86_64-cc
-    make
-    make install
 
 # needed for SSL module to make proper connections, as openssl does not actually provide the certificates.
+    brew reinstall [email protected]
     brew install ca-certificates
     CERT_DIR="$(echo /usr/local/Cellar/ca-certificates)"
     unset -v latest
     for file in "$CERT_DIR"/* ; do
         [[ $file -nt $latest ]] && latest=$file
     done
     CERT_DIR=$latest
-    cp $CERT_DIR/share/ca-certificates/* $VERSION_DIR/ssl/cert.pem
+    cp $CERT_DIR/share/ca-certificates/* /usr/local/etc/[email protected]/certs/cert.pem
 
-    if [ ! -f $VERSION_DIR/ssl/cert.pem ] ; then
-        echo "The installer was unable to locate certificates for the python build"
-        echo "Please make sure homebrew can install ca-certificates before retrying build."
-        exit 1
-    fi
+    echo "installing python $PYTHON_VER"
+    brew install python@$PYTHON_VER
 
-	# tar xzf Python-$PYTHON_VER.tgz
-	cd $VERSION_DIR/src/Python-$PYTHON_VER
-
-    export MACOSX_DEPLOYMENT_TARGET=$( sw_vers -productVersion | sed 's/..$//' )
-     export PKG_CONFIG_PATH="$VERSION_DIR/lib/pkgconfig:/usr/local/opt/tcl-tk/lib/pkgconfig:$pythonLocation/lib/pkgconfig"
-	./configure --prefix=$VERSION_DIR \
-    --enable-framework=$VERSION_DIR \
-    --with-openssl=$VERSION_DIR/openssl \
-    --with-pydebug \
-    --with-computed-gotos \
-    --with-tcltk-libs="$(pkg-config --libs tcl tk)" \
-    --with-tcltk-includes="$(pkg-config --cflags tcl tk)" \
-    CFLAGS="-I/usr/local/include " \
-    LDFLAGS="-L/usr/local/lib -L/usr/local/opt/zlib/lib" \
-    CPPFLAGS="-I/usr/local/include -I/usr/local/opt/zlib/include"
-
-    # MAKEFLAGS need to be set to an empty string to prevent a bug with framework builds. Multiprocessor python framework builds fail when a -j value is set.
-    export MAKEFLAGS=""
-	make
-	make install
-
-	sudo ln -s $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d $VERSION_DIR/$PYTHON_DIR/bin/python3
-    $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d -m ensurepip --upgrade
-
-	sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d-config $VERSION_DIR/$PYTHON_DIR/bin/python3-config
-	sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/pydoc${PY_EXE} $VERSION_DIR/$PYTHON_DIR/bin/pydoc
-	sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/idle${PY_EXE} $VERSION_DIR/$PYTHON_DIR/bin/idle
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/* $VERSION_DIR/bin
-
-    $VERSION_DIR/bin/python3 -m ensurepip --upgrade
-	$VERSION_DIR/bin/python3 -m pip install matplotlib Pillow pandas numpy networkx pytz pysolar PyGithub scikit-learn xlrd boto3
-    $VERSION_DIR/bin/python3 -m pip install build
-    $VERSION_DIR/bin/python3 -m pip install pyproj
-
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/bin/* $VERSION_DIR/bin
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/include/* $VERSION_DIR/include
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/lib/* $VERSION_DIR/lib
-    sudo ln -sf $VERSION_DIR/$PYTHON_DIR/share/* $VERSION_DIR/share
 fi
 
 # check for successful python build
-if [ ! -x $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d ]; then
+if [ ! -x /usr/local/bin/python$PYTHON_VER ]; then
     echo "Could not locate python executable in"
-    echo "PYTHON LOCATION: $VERSION_DIR/$PYTHON_DIR/bin/python${PY_EXE}d"
+    echo "PYTHON LOCATION: /usr/local/bin/python$PYTHON_VER"
     echo "Exiting build."
     exit 1
 fi
@@ -153,7 +99,6 @@ brew install gdal
 
 # libgeos
     brew install geos
-    cp /usr/local/opt/geos/lib/libgeos* $VERSION_DIR/lib
 
     if test ! -e /usr/local/lib; then
         cd /usr/local