Merge pull request kubernetes#206 from nikhita/goprivate

Set GOPRIVATE to avoid validating unpublished SHAs

artifacts/scripts/util.sh

@@ -847,6 +847,12 @@ update-deps-in-gomod() {
     local dep_count=${#deps_array[@]}
     local base_package=${2}
 
+    # if dependencies exist, dep_packages is a comma separated list of {base_package}/{dep}. Eg: "k8s.io/api,k8s.io/apimachinery"
+    local dep_packages=""
+    if [ "$dep_count" != 0 ]; then
+      dep_packages="$(echo ${1} | tr "," "\n" | sed -e 's/:.*//' -e s/^/"${base_package}\/"/ | paste -sd "," -)"
+    fi
+
     for (( i=0; i<${dep_count}; i++ )); do
         local dep="${deps_array[i]%%:*}"
         local dep_commit=$(cd ../${dep}; gomod-pseudo-version)
@@ -857,7 +863,7 @@ update-deps-in-gomod() {
 
     GO111MODULE=on go mod edit -json | jq -r '.Replace[]? | select(.New.Path | startswith("../")) | "-dropreplace \(.Old.Path)"' | GO111MODULE=on xargs -L 100 go mod edit -fmt
 
-    GO111MODULE=on GOPROXY=https://proxy.golang.org go mod download
+    GO111MODULE=on GOPRIVATE="${dep_packages}" GOPROXY=https://proxy.golang.org go mod download
     GOPROXY="file://${GOPATH}/pkg/mod/cache/download" GO111MODULE=on go mod tidy
 
     git add go.mod go.sum
@@ -890,6 +896,13 @@ checkout-deps-to-kube-commit() {
     local deps=()
     IFS=',' read -a deps <<< "${2}"
     local base_package=${3}
+    local dep_count=${#deps[@]}
+
+    # if dependencies exist, dep_packages is a comma separated list of {base_package}/{dep}. Eg: "k8s.io/api,k8s.io/apimachinery"
+    local dep_packages=""
+    if [ "$dep_count" != 0 ]; then
+      dep_packages="$(echo ${2} | tr "," "\n" | sed -e 's/:.*//' -e s/^/"${base_package}\/"/ | paste -sd "," -)"
+    fi
 
     # get last k8s.io/kubernetes commit on HEAD ...
     local k_last_kube_commit="$(last-kube-commit ${commit_msg_tag} HEAD)"
@@ -902,7 +915,6 @@ checkout-deps-to-kube-commit() {
     # might have been dropped on HEAD).
     local k_last_kube_merge=$(git-find-merge "${k_last_kube_commit}" upstream-branch)
 
-    local dep_count=${#deps[@]}
     for (( i=0; i<${dep_count}; i++ )); do
         local dep="${deps[i]%%:*}"
         local branch="${deps[i]##*:}"
@@ -921,7 +933,7 @@ checkout-deps-to-kube-commit() {
             git checkout -q "${dep_commit}"
 
             echo "Downloading go mod dependencies..."
-            GO111MODULE=on GOPROXY=https://proxy.golang.org go mod download
+            GO111MODULE=on GOPRIVATE="${dep_packages}" GOPROXY=https://proxy.golang.org go mod download
 
             local pseudo_version=$(gomod-pseudo-version)
             local cache_dir="${GOPATH}/pkg/mod/cache/download/${base_package}/${dep}/@v"

cmd/sync-tags/gomod.go

@@ -24,6 +24,7 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"strings"
 	"time"
 
 	gogit "gopkg.in/src-d/go-git.v4"
@@ -36,6 +37,11 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
 	found := map[string]bool{}
 	changed := false
 
+	depPackages, err := depsImportPaths(depsRepo)
+	if err != nil {
+		return changed, err
+	}
+
 	for _, dep := range depsRepo {
 		depPath := filepath.Join("..", dep)
 		dr, err := gogit.PlainOpen(depPath)
@@ -58,7 +64,7 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
 		// in case the pseudoVersion has not changed, running go mod download will help
 		// in avoiding packaging it up if the pseudoVersion has been published already
 		downloadCommand := exec.Command("go", "mod", "download")
-		downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", "GOPROXY=https://proxy.golang.org")
+		downloadCommand.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
 		downloadCommand.Stdout = os.Stdout
 		downloadCommand.Stderr = os.Stderr
 		if err := downloadCommand.Run(); err != nil {
@@ -88,7 +94,7 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
 		}
 
 		downloadCommand2 := exec.Command("go", "mod", "download")
-		downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", "GOPROXY=https://proxy.golang.org")
+		downloadCommand2.Env = append(os.Environ(), "GO111MODULE=on", fmt.Sprintf("GOPRIVATE=%s", depPackages), "GOPROXY=https://proxy.golang.org")
 		downloadCommand2.Stdout = os.Stdout
 		downloadCommand2.Stderr = os.Stderr
 		if err := downloadCommand2.Run(); err != nil {
@@ -116,6 +122,23 @@ func updateGomodWithTaggedDependencies(tag string, depsRepo []string) (bool, err
 	return changed, nil
 }
 
+// depImportPaths returns a comma separated string with each dependencies' import path.
+// Eg. "k8s.io/api,k8s.io/apimachinery,k8s.io/client-go"
+func depsImportPaths(depsRepo []string) (string, error) {
+	dir, err := os.Getwd()
+	if err != nil {
+		return "", fmt.Errorf("unable to get current working directory: %v", err)
+	}
+	d := strings.Split(dir, "/")
+	basePackage := d[len(d)-2]
+
+	var depImportPathList []string
+	for _, dep := range depsRepo {
+		depImportPathList = append(depImportPathList, fmt.Sprintf("%s/%s", basePackage, dep))
+	}
+	return strings.Join(depImportPathList, ","), nil
+}
+
 type ModuleInfo struct {
 	Version string
 	Name    string