Studio can build against downgraded channels

.expeditor/release_habitat.pipeline.yml

@@ -13,7 +13,8 @@ expeditor:
         HAB_ORIGIN: "core"
         PIPELINE_HAB_BLDR_URL: "https://bldr.habitat.sh"
         # Necessary to prevent old studios from poisoning builds after core plans refreshes
-        HAB_STUDIO_SECRET_HAB_FEAT_IGNORE_LOCAL: "true"
+        HAB_STUDIO_SECRET_HAB_PREFER_LOCAL_CHEF_DEPS: "true"
+        HAB_STUDIO_SECRET_HAB_REFRESH_CHANNEL: "stable"
 
 steps:
 #######################################################################

.expeditor/scripts/end_to_end/run_e2e_test_core.ps1

@@ -267,8 +267,12 @@ function Get-Leader($Remote, $ServiceGroup) {
     }
 }
 
-function Invoke-Build($PackageName) {
-    hab pkg build test/fixtures/$PackageName --reuse
+function Invoke-Build($PackageName, $RefreshChannel) {
+    $commandArgs = @("--reuse")
+    if($RefreshChannel) {
+        $commandArgs += @("--refresh-channel", $RefreshChannel)
+    }
+    hab pkg build test/fixtures/$PackageName $commandArgs
     if ($IsLinux) {
         # This changes the format of last_build from `var=value` to `$var='value'`
         # so that powershell can parse and source the script
@@ -277,8 +281,8 @@ function Invoke-Build($PackageName) {
     }
 }
 
-Function Invoke-BuildAndInstall($PackageName) {
-    Invoke-Build $PackageName
+Function Invoke-BuildAndInstall($PackageName, $RefreshChannel) {
+    Invoke-Build @PSBoundParameters
     . ./results/last_build.ps1
     hab pkg install ./results/$pkg_artifact
     hab studio run "rm /hab/pkgs/$pkg_ident/hooks"

.expeditor/verify.pipeline.yml

@@ -631,7 +631,6 @@ steps:
       BUILD_PKG_TARGET: "x86_64-linux"
       HAB_BLDR_CHANNEL: "acceptance"
       HAB_INTERNAL_BLDR_CHANNEL: "acceptance"
-      HAB_STUDIO_SECRET_HAB_FEAT_IGNORE_LOCAL: "true"
     command:
       - .expeditor/scripts/verify/build_package.sh components/backline
     expeditor:
@@ -676,7 +675,6 @@ steps:
       BUILD_PKG_TARGET: "x86_64-linux"
       HAB_BLDR_CHANNEL: "acceptance"
       HAB_INTERNAL_BLDR_CHANNEL: "acceptance"
-      HAB_STUDIO_SECRET_HAB_FEAT_IGNORE_LOCAL: "true"
     command:
       - .expeditor/scripts/verify/build_package.sh components/pkg-cfize
     expeditor:
@@ -806,7 +804,6 @@ steps:
       BUILD_PKG_TARGET: "x86_64-linux-kernel2"
       HAB_BLDR_CHANNEL: "acceptance"
       HAB_INTERNAL_BLDR_CHANNEL: "acceptance"
-      HAB_STUDIO_SECRET_HAB_FEAT_IGNORE_LOCAL: "true"
     expeditor:
       executor:
         docker:

components/common/src/command/package/install.rs

@@ -251,10 +251,6 @@ pub enum InstallHookMode {
 /// package to satisfy a dependency, or if we should ignore it, thus
 /// giving the user the opportunity to try installing from another
 /// channel.
-///
-/// Usage of this is currently hidden behind the IGNORE_LOCAL feature
-/// flag, as there is still some question as to the best way to solve
-/// this.
 #[derive(Debug, Eq, PartialEq)]
 pub enum LocalPackageUsage {
     /// Use locally-installed packages if they satisfy the desired
@@ -531,7 +527,7 @@ impl<'a> InstallTask<'a> {
 
             match (latest_local, latest_remote) {
                 (Ok(local), Some(remote)) => {
-                    if local > remote {
+                    if local > remote && !self.ignore_locally_installed_packages() {
                         // Return the latest identifier reported by
                         // the Builder API *unless* there is a newer
                         // version found installed locally.
@@ -546,8 +542,6 @@ impl<'a> InstallTask<'a> {
                 }
                 (Ok(local), None) => {
                     if self.ignore_locally_installed_packages() {
-                        // This is the behavior that is currently
-                        // governed by the IGNORE_LOCAL feature-flag
                         self.recommend_channels(ui, (&ident, target), token).await?;
                         ui.warn(format!("Locally-installed package '{}' would satisfy '{}', \
                                          but we are ignoring that as directed",

components/common/src/lib.rs

@@ -75,7 +75,6 @@ bitflags::bitflags! {
         #[cfg(target_family = "unix")]
         const NATIVE_PACKAGE_SUPPORT     = 0b0000_0010_0000;
         const OFFLINE_INSTALL            = 0b0000_0100_0000;
-        const IGNORE_LOCAL               = 0b0000_1000_0000;
         const TRIGGER_ELECTION           = 0b0010_0000_0000;
         const STRUCTOPT_CLI              = 0b0100_0000_0000;
         const NO_NAMED_PIPE_HEALTH_CHECK = 0b1000_0000_0000;
@@ -90,7 +89,6 @@ lazy_static! {
                  (FeatureFlag::TEST_BOOT_FAIL, "HAB_FEAT_BOOT_FAIL"),
                  (FeatureFlag::REDACT_HTTP, "HAB_FEAT_REDACT_HTTP"),
                  (FeatureFlag::OFFLINE_INSTALL, "HAB_FEAT_OFFLINE_INSTALL"),
-                 (FeatureFlag::IGNORE_LOCAL, "HAB_FEAT_IGNORE_LOCAL"),
                  (FeatureFlag::TRIGGER_ELECTION, "HAB_FEAT_TRIGGER_ELECTION"),
                  (FeatureFlag::STRUCTOPT_CLI, "HAB_FEAT_STRUCTOPT_CLI"),
                  (FeatureFlag::NO_NAMED_PIPE_HEALTH_CHECK, "HAB_FEAT_NO_NAMED_PIPE_HEALTH_CHECK"),

components/hab/src/cli/hab/pkg.rs

@@ -348,6 +348,13 @@ pub struct PkgBuild {
     /// Uses a Dockerized Studio for the build
     #[structopt(name = "DOCKER", short = "D", long = "docker")]
     docker:          bool,
+    /// Channel used to retrieve plan dependencies for Chef supported origins
+    #[structopt(name = "REFRESH_CHANNEL",
+                short = "f",
+                long = "refresh-channel",
+                env = "HAB_REFRESH_CHANNEL",
+                default_value = "stable")]
+    refresh_channel: Option<String>,
 }
 
 /// Bulk Uploads Habitat Artifacts to a Depot from a local directory
@@ -523,8 +530,7 @@ pub struct PkgInstall {
     offline:               bool,
     /// Do not use locally-installed packages when a corresponding package cannot be installed
     /// from Builder
-    #[structopt(long = "ignore-local",
-                hidden = !FEATURE_FLAGS.contains(FeatureFlag::IGNORE_LOCAL))]
+    #[structopt(long = "ignore-local")]
     ignore_local:          bool,
 }
 

components/hab/src/command/pkg/build.rs

@@ -14,7 +14,8 @@ pub async fn start(ui: &mut UI,
                    origins: &[Origin],
                    native_package: bool,
                    reuse: bool,
-                   docker: bool)
+                   docker: bool,
+                   refresh_channel: Option<&str>)
                    -> Result<()> {
     let mut args: Vec<OsString> = Vec::new();
     if let Some(root) = root {
@@ -25,6 +26,10 @@ pub async fn start(ui: &mut UI,
         args.push("-s".into());
         args.push(src.into());
     }
+    if let Some(refresh_channel) = refresh_channel {
+        args.push("-f".into());
+        args.push(refresh_channel.into());
+    }
 
     if !origins.is_empty() {
         let signing_key_names = origins.iter()

components/hab/src/main.rs

@@ -809,6 +809,7 @@ async fn sub_pkg_build(ui: &mut UI, m: &ArgMatches<'_>, feature_flags: FeatureFl
     let plan_context = required_value_of(m, "PLAN_CONTEXT");
     let root = m.value_of("HAB_STUDIO_ROOT");
     let src = m.value_of("SRC_PATH");
+    let refresh_channel = m.value_of("REFRESH_CHANNEL");
 
     let origins = hab_key_origins(m)?;
     if !origins.is_empty() {
@@ -846,7 +847,8 @@ async fn sub_pkg_build(ui: &mut UI, m: &ArgMatches<'_>, feature_flags: FeatureFl
                                &origins,
                                native_package,
                                reuse,
-                               docker).await
+                               docker,
+                               refresh_channel).await
 }
 
 fn sub_pkg_config(m: &ArgMatches<'_>) -> Result<()> {
@@ -1104,12 +1106,11 @@ async fn sub_pkg_install(ui: &mut UI,
             InstallMode::default()
         };
 
-    let local_package_usage =
-        if feature_flags.contains(FeatureFlag::IGNORE_LOCAL) && m.is_present("IGNORE_LOCAL") {
-            LocalPackageUsage::Ignore
-        } else {
-            LocalPackageUsage::default()
-        };
+    let local_package_usage = if m.is_present("IGNORE_LOCAL") {
+        LocalPackageUsage::Ignore
+    } else {
+        LocalPackageUsage::default()
+    };
 
     let install_hook_mode = if m.is_present("IGNORE_INSTALL_HOOK") {
         InstallHookMode::Ignore

components/plan-build-ps1/bin/hab-plan-build.ps1

@@ -85,6 +85,10 @@ if (!(Test-Path Env:\HAB_BLDR_CHANNEL)) {
 if (!(Test-Path Env:\HAB_FALLBACK_CHANNEL)) {
     $env:HAB_FALLBACK_CHANNEL = "stable"
 }
+# Use the refresh channel for dependencies in the core/chef/chef-platform origins
+if (!(Test-Path Env:\HAB_REFRESH_CHANNEL)) {
+    $env:HAB_REFRESH_CHANNEL = "stable"
+}
 # The value of `$env:Path` on initial start of this program
 $script:INITIAL_PATH = "$env:Path"
 # The full target tuple this plan will be built for
@@ -331,58 +335,25 @@ function Set-HabBin {
 
 function Install-Dependency($dependency, $install_args = $null) {
     if (!$env:NO_INSTALL_DEPS) {
-        $cmd = "$HAB_BIN pkg install -u $env:HAB_BLDR_URL --channel $env:HAB_BLDR_CHANNEL $dependency $install_args"
-        if($env:HAB_FEAT_IGNORE_LOCAL -eq "true") { $cmd += " --ignore-local" }
+        $origin = $dependency.Split("/")[0]
+        $channel = $env:HAB_BLDR_CHANNEL
+        $ignoreLocal = ""
+        if ($origin -eq "core" -or $origin -eq "chef" -or $origin -eq "chef-platform") {
+            $channel="$env:HAB_REFRESH_CHANNEL"
+            if (!$env:HAB_PREFER_LOCAL_CHEF_DEPS) {
+                $ignoreLocal="--ignore-local"
+            }
+        }
+        $cmd = "$HAB_BIN pkg install -u $env:HAB_BLDR_URL --channel $channel $dependency $install_args $ignoreLocal"
         Invoke-Expression $cmd
-        if ($LASTEXITCODE -ne 0 -and ($env:HAB_BLDR_CHANNEL -ne $env:HAB_FALLBACK_CHANNEL)) {
+        if ($LASTEXITCODE -ne 0 -and ($channel -ne $env:HAB_FALLBACK_CHANNEL)) {
             Write-BuildLine "Trying to install '$dependency' from '$env:HAB_FALLBACK_CHANNEL'"
-            $cmd = "$HAB_BIN pkg install -u $env:HAB_BLDR_URL --channel $env:HAB_FALLBACK_CHANNEL $dependency $install_args"
-            if($env:HAB_FEAT_IGNORE_LOCAL -eq "true") { $cmd += " --ignore-local" }
+            $cmd = "$HAB_BIN pkg install -u $env:HAB_BLDR_URL --channel $env:HAB_FALLBACK_CHANNEL $dependency $install_args $ignoreLocal"
             Invoke-Expression $cmd
         }
     }
 }
 
-# **Internal** Return the path to the latest release of a package on stdout.
-#
-# ```
-# _latest_installed_package acme/nginx
-# # /hab/pkgs/acme/nginx/1.8.0/20150911120000
-# _latest_installed_package acme/nginx/1.8.0
-# # /hab/pkgs/acme/nginx/1.8.0/20150911120000
-# _latest_installed_package acme/nginx/1.8.0/20150911120000
-# # /hab/pkgs/acme/nginx/1.8.0/20150911120000
-# ```
-#
-# Will return the package found on disk, and $false if a package cannot be found.
-function _latest_installed_package($dependency) {
-    if (!(Test-Path "$HAB_PKG_PATH/$dependency")) {
-        Write-Warning "No installed packages of '$dependency' were found"
-        return $false
-    }
-
-    $result = Get-ChildItem "$HAB_PKG_PATH/$dependency" -Recurse -Include MANIFEST -ErrorAction SilentlyContinue
-    if (!$result) {
-        Write-Warning "Could not find a suitable installed package for '$dependency'"
-        return $false
-    } else {
-        return Split-Path ($result[-1].FullName) -Parent
-    }
-}
-
-function Resolve-Dependency($dependency) {
-    if (!$dependency.Contains("/")) {
-        Write-Warning "Origin required for '$dependency' in plan '$pkg_origin/$pkg_name' (example: acme/$dependency)"
-        return $false
-    }
-
-    if ($dep_path = _latest_installed_package $dependency) {
-        return $dep_path
-    } else {
-        return $false
-    }
-}
-
 # **Internal** Returns (on stdout) the `DEPS` file contents of another locally
 # installed package which contain the set of all direct run dependencies. An
 # empty set could be returned as whitespace and/or newlines.  The lack of a
@@ -658,10 +629,17 @@ function Resolve-ScaffoldingDependencyList {
 
     if($pkg_scaffolding) {
         $pkg_scaffolding = @($pkg_scaffolding)[0]
-        Install-Dependency $pkg_scaffolding
+        $oldEncoding = [Console]::OutputEncoding
+        [Console]::OutputEncoding = [System.Text.Encoding]::UTF8
+        $res = Install-Dependency $pkg_scaffolding | Out-String
+        Write-Host $res
+        [Console]::OutputEncoding = $oldEncoding
+        if($res.Split("`n")[-2] -match "\S+/\S+") {
+            $resolved = $Matches[0]
+        }
         # Add scaffolding package to the list of scaffolding build deps
         $scaff_build_deps += $pkg_scaffolding
-        if($resolved=(Resolve-Dependency $pkg_scaffolding)) {
+        if($resolved) {
             Write-BuildLine "Resolved scaffolding dependency '$pkg_scaffolding' to $resolved"
             $scaff_build_deps_resolved+=($resolved)
             $sdeps=(@(Get-DepsFor $resolved) + @(Get-BuildDepsFor $resolved))
@@ -742,10 +720,17 @@ function Resolve-BuildDependencyList {
     # Build `${pkg_build_deps_resolved[@]}` containing all resolved direct build
     # dependencies.
     foreach($dep in $pkg_build_deps) {
-        Install-Dependency $dep
-        if($resolved=(Resolve-Dependency $dep)) {
+        $oldEncoding = [Console]::OutputEncoding
+        [Console]::OutputEncoding = [System.Text.Encoding]::UTF8
+        $res = Install-Dependency $dep | Out-String
+        Write-Host $res
+        [Console]::OutputEncoding = $oldEncoding
+        if($res.Split("`n")[-2] -match "\S+/\S+") {
+            $resolved = $Matches[0]
+        }
+        if($resolved) {
             Write-BuildLine "Resolved build dependency '$dep' to $resolved"
-            $script:pkg_build_deps_resolved+=($resolved)
+            $script:pkg_build_deps_resolved+=(Resolve-Path "$HAB_PKG_PATH/$resolved").Path
         } else {
             throw "Resolving '$dep' failed, should this be built first?"
         }
@@ -758,10 +743,17 @@ function Resolve-RunDependencyList {
     # Build `${pkg_deps_resolved[@]}` containing all resolved direct run
     # dependencies.
     foreach($dep in $pkg_deps) {
-        Install-Dependency $dep "--ignore-install-hook"
-        if ($resolved=(Resolve-Dependency $dep)) {
+        $oldEncoding = [Console]::OutputEncoding
+        [Console]::OutputEncoding = [System.Text.Encoding]::UTF8
+        $res = Install-Dependency $dep --ignore-install-hook | Out-String
+        Write-Host $res
+        [Console]::OutputEncoding = $oldEncoding
+        if($res.Split("`n")[-2] -match "\S+/\S+") {
+            $resolved = $Matches[0]
+        }
+        if ($resolved) {
             Write-BuildLine "Resolved dependency '$dep' to $resolved"
-            $script:pkg_deps_resolved+=$resolved
+            $script:pkg_deps_resolved+=(Resolve-Path "$HAB_PKG_PATH/$resolved").Path
         } else {
             throw "Resolving '$dep' failed, should this be built first?"
         }