compile

#!/usr/bin/env bash
# bin/compile <build-dir> <cache-dir> <env-dir>

### Configure environment

set -o errexit    # always exit on error
set -o pipefail   # don t ignore exit codes when piping output

if [ -n "$BUILDPACK_DEBUG" ]; then
    set -x
fi

unset GIT_DIR     # Avoid GIT_DIR leak from previous build steps

[ "$BUILDPACK_XTRACE" ] && set -o xtrace

### Constants

# This is used by the buildpack stdlib for metrics
# shellcheck disable=SC2034
BPLOG_PREFIX="buildpack.nodejs"

### Configure directories

BUILD_DIR=${1:-}
CACHE_DIR=${2:-}
ENV_DIR=${3:-}
BP_DIR=$(cd "$(dirname "${0:-}")"; cd ..; pwd)

### Load dependencies

# shellcheck source=lib/vendor/stdlib_v7.sh
source "$BP_DIR/lib/vendor/stdlib_v7.sh"
# shellcheck source=lib/output.sh
source "$BP_DIR/lib/output.sh"
# shellcheck source=lib/monitor.sh
source "$BP_DIR/lib/monitor.sh"
# shellcheck source=lib/environment.sh
source "$BP_DIR/lib/environment.sh"
# shellcheck source=lib/npm.sh
source "$BP_DIR/lib/npm.sh"
# shellcheck source=lib/failure.sh
source "$BP_DIR/lib/failure.sh"
# shellcheck source=lib/binaries.sh
source "$BP_DIR/lib/binaries.sh"
# shellcheck source=lib/json.sh
source "$BP_DIR/lib/json.sh"
# shellcheck source=lib/cache.sh
source "$BP_DIR/lib/cache.sh"
# shellcheck source=lib/dependencies.sh
source "$BP_DIR/lib/dependencies.sh"
# shellcheck source=lib/uuid.sh
source "$BP_DIR/lib/uuid.sh"
# shellcheck source=lib/kvstore.sh
source "$BP_DIR/lib/kvstore.sh"
# shellcheck source=lib/metadata.sh
source "$BP_DIR/lib/metadata.sh"
# shellcheck source=lib/features.sh
source "$BP_DIR/lib/features.sh"
# shellcheck source=lib/builddata.sh
source "$BP_DIR/lib/builddata.sh"
# shellcheck source=lib/yarn-2.sh
source "$BP_DIR/lib/yarn-2.sh"
# Load meteor framework related functions
source $BP_DIR/bin/meteor.sh
if test -n "$meteor_version" ; then
  # Most Meteor projects are using React or equivalent and all these
  # dependencies are in dev usually. We had way too much support because of
  # this.
  npm_config_production=${NPM_CONFIG_PRODUCTION:-false}
  export NPM_CONFIG_PRODUCTION=$npm_config_production
fi

export PATH="$BUILD_DIR/.scalingo/node/bin:$BUILD_DIR/.scalingo/yarn/bin":$PATH

export COREPACK_HOME="$BUILD_DIR/.scalingo/corepack"
export COREPACK_ENABLE_DOWNLOAD_PROMPT=0

# 2024-11-20:
# We started getting reports on build timeouts on the platform that were unexpected
# due to the specific versions on Node.js and npm which had been working fine up until
# this date. After some extensive digging, we believe we narrowed down the cause to
# a bug in io_uring that affects the Amazon Linux 2023 release. There is a pending fix
# which will be released around Dec. 9th but, until then, the workaround is set disable
# the use of io_uring in libuv.
#
# See https://github.com/npm/cli/issues/7814#issuecomment-2488626736
# → https://github.com/amazonlinux/amazon-linux-2023/issues/840#issuecomment-2485782075
# → https://lore.kernel.org/io-uring/3d913aef-8c44-4f50-9bdf-7d9051b08941@app.fastmail.com/T/#m57570b5f8f2fc00d5a17cfe18ffeeba9fc23a43d
export UV_USE_IO_URING=${UV_USE_IO_URING:-0}

LOG_FILE=$(mktemp -t node-build-log.XXXXX)
echo "" > "$LOG_FILE"

build_start_time=$(nowms)

### Handle errors

handle_failure() {
  meta_set "node-build-success" "false"
  header "Build failed"
  fail_using_yarn2_with_yarn_production_environment_variable_set "$LOG_FILE"
  fail_yarn_outdated "$LOG_FILE"
  fail_yarn_lockfile_outdated "$LOG_FILE"
  fail_node_install "$LOG_FILE" "$BUILD_DIR"
  fail_yarn_install "$LOG_FILE" "$BUILD_DIR"
  fail_invalid_semver "$LOG_FILE"
  log_other_failures "$LOG_FILE"
  warn_aws_proxy "$BUILD_DIR"
  warn_untracked_dependencies "$LOG_FILE"
  warn_angular_resolution "$LOG_FILE"
  warn_missing_devdeps "$LOG_FILE" "$BUILD_DIR"
  warn_econnreset "$LOG_FILE"
  log_meta_data >> "$BUILDPACK_LOG_FILE"
  failure_message | output "$LOG_FILE"
}
trap 'handle_failure' ERR

### Initalize metadata store
# Create the metadata store
meta_init "$CACHE_DIR"
# Make sure that it's empty and save any previous build's data
meta_setup
# the "build-step" key keeps track of where we are in the execution
# so that we can see where failed builds errored
meta_set "build-step" "init"

### Check initial state

[ -e "$BUILD_DIR/node_modules" ] && PREBUILD=true || PREBUILD=false
[ -f "$BUILD_DIR/yarn.lock" ] && YARN=true || YARN=false
[ -f "$BUILD_DIR/package-lock.json" ] && NPM_LOCK=true || NPM_LOCK=false
[ -f "$BUILD_DIR/pnpm-lock.yaml" ] && PNPM=true || PNPM=false
YARN_2=$(detect_yarn_2 "$YARN" "$BUILD_DIR")

### Save build info
features_init "nodejs" "$BUILD_DIR" "$CACHE_DIR" "$BP_DIR/features"
log_initial_state
generate_uuids

### Failures that should be caught immediately

fail_dot_scalingo "$BUILD_DIR"
fail_dot_scalingo_node "$BUILD_DIR"
fail_invalid_package_json "$BUILD_DIR"
fail_multiple_lockfiles "$BUILD_DIR"
fail_conflicting_package_manager_metadata "$BUILD_DIR"
fail_iojs_unsupported "$BUILD_DIR"
warn_prebuilt_modules "$BUILD_DIR"
warn_missing_package_json "$BUILD_DIR"

### Gather more project information

# we know that package.json is valid json, and can pull more data out
log_project_info "$BUILD_DIR"

### Compile

create_env() {
  export_env_dir "$ENV_DIR"
  create_default_env "$YARN"
  write_profile "$BP_DIR" "$BUILD_DIR"
  write_export "$BP_DIR" "$BUILD_DIR"
}

header "Creating runtime environment" | output "$LOG_FILE"

mkdir -p "$BUILD_DIR/.scalingo/node/"
cd $BUILD_DIR
create_env # can't pipe the whole thing because piping causes subshells, preventing exports
list_node_config | output "$LOG_FILE"
create_build_env


### Configure vendored package manager
export VENDOR_PATH

if [[ "$YARN_2" == "true" ]]; then
  if [[ -f "$BUILD_DIR/.npmrc" ]]; then
    warn "There is an existing .npmrc file that will not be used. All configurations are read from the .yarnrc.yml file." "https://doc.scalingo.com/languages/nodejs/yarn-2"
  fi

  if [[ -f "$BUILD_DIR/.yarnrc" ]]; then
    warn "There is an existing .yarnrc file that will not be used. All configurations are read from the .yarnrc.yml file." "https://doc.scalingo.com/languages/nodejs/yarn-2"
  fi

  # When in Plug'n'Play mode the yarn cache will include dependencies needed at
  # runtime; the cache must live in /app (not /tmp) so that dependencies are
  # persisted. When using zero-install, /app/.yarn/cache will include all
  # dependencies which yarn should reuse.
  if use_yarn_app_cache "$BUILD_DIR"; then
    YARN_CACHE_FOLDER=${YARN_CACHE_FOLDER:-"$BUILD_DIR/.yarn/cache"}
  fi

  # When in zero-install mode, use --immutable-cache to verify that the cache
  # is pristine. Cache mutation is expected in nonzero installs.
  if has_yarn_cache "$BUILD_DIR"; then
    YARN_ENABLE_IMMUTABLE_CACHE=${YARN_ENABLE_IMMUTABLE_CACHE:-true}
  fi

  meta_set "yarn2-immutable-cache" ${YARN_ENABLE_IMMUTABLE_CACHE:-false}

  if node_modules_enabled "$BUILD_DIR"; then
    meta_set "yarn2-pnp-linker" "false"
  else
    meta_set "yarn2-pnp-linker" "true"
  fi

  ### Configure Yarn 2
  # fail for no yarnrc.yml
  fail_missing_yarnrc_yml "$BUILD_DIR"

  # get yarn_path
  VENDOR_PATH=$(get_yarn_path "$BUILD_DIR")

  # if we're installing yarn via corepack we don't need to fail
  # on the yarn path configuration or vendored script
  if [[ "$(read_json "$BUILD_DIR/package.json" ".packageManager")" != yarn* ]]; then
    # fail for no yarnPath in rc
    fail_missing_yarn_path "$BUILD_DIR" "$VENDOR_PATH"
    # fail for missing yarn in .yarn/releases
    fail_missing_yarn_vendor "$BUILD_DIR" "$VENDOR_PATH"
  fi
fi

### Configure package manager cache directories
[ ! "$YARN_CACHE_FOLDER" ] && YARN_CACHE_FOLDER=$(mktemp -d -t yarncache.XXXXX)
[ ! "$NPM_CONFIG_CACHE" ] && NPM_CONFIG_CACHE=$(mktemp -d -t npmcache.XXXXX)
[ ! "$PNPM_CONFIG_CACHE" ] && PNPM_CONFIG_CACHE=$(mktemp -d -t pnpmcache.XXXXX)
export YARN_CACHE_FOLDER NPM_CONFIG_CACHE PNPM_CONFIG_CACHE

install_bins() {
  local node_engine npm_engine yarn_engine node_version package_manager pnpm_engine

  node_engine=$(read_json "$BUILD_DIR/package.json" ".engines.node")
  npm_engine=$(read_json "$BUILD_DIR/package.json" ".engines.npm")
  yarn_engine=$(read_json "$BUILD_DIR/package.json" ".engines.yarn")
  pnpm_engine=$(read_json "$BUILD_DIR/package.json" ".engines.pnpm")
  package_manager=$(read_json "$BUILD_DIR/package.json" ".packageManager")

  if [[ "$YARN_2" == "true" && -n "$yarn_engine" ]]; then
    warn "You don't need to specify Yarn engine. Scalingo will install the latest Yarn 1.x, so that per project version can be used. More information here: https://yarnpkg.com/getting-started/install#global-install"
    unset yarn_engine
  fi

  if [ -n "${meteor_version}" ] ; then
    [ -z "$node_engine" ] && node_engine=$(meteor_node_version "${meteor_version}")
    [ -z "$npm_engine" ]  && npm_engine=$(meteor_npm_version "${meteor_version}")
  fi

  meta_set "node-version-request" "$node_engine"
  meta_set "npm-version-request" "$npm_engine"
  meta_set "yarn-version-request" "$yarn_engine"
  meta_set "pnpm-version-request" "$pnpm_engine"
  meta_set "package-manager-request" "$package_manager"

  echo "engines.node (package.json):   ${node_engine:-unspecified}"
  echo "engines.npm (package.json):    ${npm_engine:-unspecified (use default)}"

  if $YARN || [ -n "$yarn_engine" ]; then
    echo "engines.yarn (package.json):   ${yarn_engine:-unspecified (use default)}"
  fi

  if $PNPM || [ -n "$pnpm_engine" ]; then
    echo "engines.pnpm (package.json):   ${pnpm_engine:-unspecified (use default)}"
  fi

  if [ -n "$package_manager" ]; then
    echo "packageManager (package.json): $(echo "$package_manager" | cut -d "+" -f 1)"
  fi

  echo ""

  warn_node_engine "$node_engine"

  if [ -n "$pnpm_engine" ] && [[ "$package_manager" == pnpm* ]]; then
    warn_multiple_pnpm_version "$package_manager" "$pnpm_engine"
  fi

  if [ -n "$yarn_engine" ] && [[ "$package_manager" == yarn* ]]; then
    warn_multiple_yarn_version "$package_manager" "$yarn_engine"
  fi

  if has_release_script "$BUILD_DIR" && [[ "$package_manager" == yarn* ]]; then
    warn_yarn_release_script_with_package_manager "$package_manager" "$(get_yarn_path "$BUILD_DIR")"
  fi

  meta_set "build-step" "install-nodejs"
  monitor "install-node-binary" install_nodejs "$node_engine" "$BUILD_DIR/.scalingo/node"

  meta_set "build-step" "install-npm"
  monitor "install-npm-binary" install_npm "$npm_engine" "$BUILD_DIR/.scalingo/node" $NPM_LOCK

  node_version="$(node --version)"
  mcount "version.node.$node_version"
  meta_set "node-version" "$node_version"

  if ! has_release_script "$BUILD_DIR" && [[ "$package_manager" == yarn* ]]; then
    meta_set "build-step" "install-yarn-using-corepack"
    monitor "install-yarn-using-corepack" install_yarn_using_corepack_package_manager "$package_manager" "$node_version"
  else
    # Download yarn if there is a yarn.lock file or if the user
    # has specified a version of yarn under "engines". We'll still
    # only install using yarn if there is a yarn.lock file
    if $YARN || [ -n "$yarn_engine" ]; then
      meta_set "build-step" "install-yarn"
      monitor "install-yarn-binary" install_yarn "$BUILD_DIR/.scalingo/yarn" "$yarn_engine"
    fi
  fi

  if $PNPM; then
    meta_set "build-step" "install-pnpm-using-corepack"

    default_pnpm_version="pnpm@latest"
    using_default_pnpm=false

    if [[ "$package_manager" == pnpm* ]]; then
      package_manager="$package_manager"
    elif [ -n "$pnpm_engine" ]; then
      package_manager="pnpm@${pnpm_engine}"
    else
      using_default_pnpm=true
      package_manager="$default_pnpm_version"
    fi

    monitor "install-pnpm-using-corepack" install_pnpm_using_corepack_package_manager "$package_manager" "$node_version" "$PNPM_CONFIG_CACHE"
    if [ "$using_default_pnpm" == true ]; then
      warn_default_pnpm_version_used "$default_pnpm_version"
    fi
  fi

  if $YARN; then
    mcount "version.yarn.$(yarn --version)"
    meta_set "yarn-version" "$(yarn --version)"
  elif $PNPM; then
    mcount "version.pnpm.$(pnpm --version)"
    meta_set "pnpm-version" "$(pnpm --version)"
  else
    mcount "version.npm.$(npm --version)"
    meta_set "npm-version" "$(npm --version)"
  fi

  warn_old_npm

  if [[ $YARN_2 == true ]]; then
    YARN=false
  fi
}

header "Installing binaries" | output "$LOG_FILE"
install_bins | output "$LOG_FILE"

if [[ -z "$USE_NPM_INSTALL" ]]; then
  if [[ "$(should_use_npm_ci "$BUILD_DIR")" == "true" ]]; then
    USE_NPM_INSTALL=false
  else
    USE_NPM_INSTALL=true
  fi
fi

restore_cache() {
  local cache_status cache_directories restore_cache_start_time
  restore_cache_start_time=$(nowms)

  cache_status="$(get_cache_status "$CACHE_DIR")"
  cache_directories="$(get_cache_directories "$BUILD_DIR")"

  if $YARN; then
    if $PREBUILD && [[ "$SKIP_NODE_MODULES_CHECK" == "true" ]]; then
      echo "Keeping existing node_modules because SKIP_NODE_MODULES_CHECK=${SKIP_NODE_MODULES_CHECK}"
    elif $PREBUILD; then
      warn "node_modules checked into source control" "http://doc.scalingo.com/languages/javascript/nodejs#do-not-track-modules-with-git"
      rm -rf "$BUILD_DIR/node_modules"
    fi
  fi

  if [[ "$cache_status" == "disabled" ]]; then
    header "Restoring cache"
    echo "Caching has been disabled because NODE_MODULES_CACHE=${NODE_MODULES_CACHE}"
  elif [[ "$cache_status" == "valid" ]]; then
    header "Restoring cache"
    if [[ "$cache_directories" == "" ]]; then
      restore_default_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$YARN_CACHE_FOLDER" "$NPM_CONFIG_CACHE" "$PNPM_CONFIG_CACHE"
    else
      restore_custom_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$cache_directories"
    fi
  elif [[ "$cache_status" == "new-signature" ]]; then
    header "Restoring cache"
    if [[ "$cache_directories" == "" ]]; then
      echo "Cached directories were not restored due to a change in version of node, npm, yarn or stack"
      echo "Module installation may take longer for this build"
    else
      # If the user has specified custom cache directories, be more explicit
      echo "Invalidating cache due to a change in version of node, npm, yarn or stack"
      echo "Will not restore the following directories for this build:"
      for directory in $cache_directories; do
        echo "  $directory"
      done
    fi
  else
    # No cache exists, be silent
    :
  fi

  mcount "cache.$cache_status"
  meta_set "cache-status" "$cache_status"
  meta_time "restore-cache-time" "$restore_cache_start_time"
}

meta_set "build-step" "restore-cache"
restore_cache | output "$LOG_FILE"

build_dependencies() {
  local cache_status start

  cache_status="$(get_cache_status "$CACHE_DIR")"
  start=$(nowms)
  if [[ "$YARN_2" == "true" ]]; then
    yarn_2_install "$BUILD_DIR"
  elif $YARN; then
    yarn_node_modules "$BUILD_DIR"
  elif $PNPM; then
    pnpm_install "$BUILD_DIR" "$CACHE_DIR"
  elif $PREBUILD; then
    echo "Prebuild detected (node_modules already exists)"
    npm_rebuild "$BUILD_DIR"
  else
    npm_node_modules "$BUILD_DIR"
  fi

  mtime "modules.time.cache.$cache_status" "${start}"

  meta_set "build-step" "build-script"
  header "Build"
  run_build_script "$BUILD_DIR"
}

meta_set "build-step" "install-dependencies"
log_build_scripts "$BUILD_DIR"
run_prebuild_script "$BUILD_DIR" | output "$LOG_FILE"
header "Installing dependencies" | output "$LOG_FILE"
build_dependencies | output "$LOG_FILE"

cache_build() {
  local cache_directories cache_build_start_time
  cache_build_start_time=$(nowms)
  cache_directories="$(get_cache_directories "$BUILD_DIR")"

  clear_cache "$CACHE_DIR"
  if ! ${NODE_MODULES_CACHE:-true}; then
    # we've already warned that caching is disabled in the restore step
    # so be silent here
    :
  elif [[ "$cache_directories" == "" ]]; then
    header "Caching build"
    save_default_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$YARN_CACHE_FOLDER" "$NPM_CONFIG_CACHE" "$PNPM_CONFIG_CACHE"
  else
    header "Caching build"
    save_custom_cache_directories "$BUILD_DIR" "$CACHE_DIR" "$cache_directories"
  fi
  save_signature "$CACHE_DIR"
  meta_time "save-cache-time" "$cache_build_start_time"
}

prune_devdependencies() {
  if $YARN || $YARN_2; then
    yarn_prune_devdependencies "$BUILD_DIR" "$YARN_CACHE_FOLDER" "$BP_DIR"
  elif $PNPM; then
    pnpm_prune_devdependencies "$BUILD_DIR"
  else
    npm_prune_devdependencies "$BUILD_DIR"
  fi
}

if $YARN_2; then
  meta_set "build-step" "save-cache"
  cache_build | output "$LOG_FILE"
  meta_set "build-step" "prune-dependencies"
  header "Pruning devDependencies" | output "$LOG_FILE"
  prune_devdependencies | output "$LOG_FILE"
elif "$YARN" && "$USE_YARN_CACHE" && [[ "$(get_cache_directories "$BUILD_DIR")" == "" ]]; then
  meta_set "build-step" "prune-dependencies"
  header "Pruning devDependencies" | output "$LOG_FILE"
  prune_devdependencies | output "$LOG_FILE"
  meta_set "build-step" "save-cache"
  cache_build | output "$LOG_FILE"
else
  meta_set "build-step" "save-cache"
  cache_build | output "$LOG_FILE"
  meta_set "build-step" "prune-dependencies"
  header "Pruning devDependencies" | output "$LOG_FILE"
  prune_devdependencies | output "$LOG_FILE"
fi

# Build meteor app if required
if test -n "$meteor_version"; then
  header "Meteor app detected: version ${meteor_version}"
  setup_meteor_build_environment
  build_meteor_app $BUILD_DIR $CACHE_DIR
fi

meta_set "build-step" "cleanup-script"
run_cleanup_script "$BUILD_DIR" | output "$LOG_FILE"

summarize_build() {
  if $NODE_VERBOSE; then
    list_dependencies "$BUILD_DIR"
  fi

  mmeasure 'modules.size' "$(measure_size)"
  meta_set "node-modules-size" "$(measure_size)"
}

meta_set "build-step" "summarize"
header "Build succeeded!" | output "$LOG_FILE"
mcount "compile"
summarize_build | output "$LOG_FILE"
meta_set "node-build-success" "true"
meta_time "build-time" "$build_start_time"

if test -n "$meteor_version" ; then
  rm -r "$BUILD_DIR/node_modules"
else
  warn_no_start "$BUILD_DIR"
  warn_unmet_dep "$LOG_FILE"
fi
warn_old_npm_lockfile $NPM_LOCK

meta_set "build-step" "finished"
log_meta_data >> "$BUILDPACK_LOG_FILE"