Merge branch 'master' into fix/gcp-sa-key-encoding

.ci/ci.yml.tmpl

@@ -387,6 +387,13 @@ jobs:
                   path: mm-initial-pr
               get_params:
                   skip_clone: true
+          # TODO(emilymye): Aggregate or add into create-or-update-pr
+          # to group updates to downstreams
+          - task: downstream-changelog-metadata
+            file: magic-modules/.ci/magic-modules/downstream-changelog-metadata.yml
+            params:
+              GITHUB_TOKEN: ((github-account.password))
+              DOWNSTREAM_REPOS: "{{','.join(vars.downstreams_with_changelogs)}}"
 
           - put: magic-modules
             params:
@@ -408,6 +415,7 @@ jobs:
                 label_file: magic-modules-with-comment/label_file
             get_params:
                 skip_clone: true
+
     - name: terraform-acceptance-tests
       plan:
         - get: magic-modules
@@ -430,6 +438,11 @@ jobs:
       plan:
           - get: mm-approved-prs
             attempts: 2
+          - task: downstream-changelog-metadata
+            file: mm-approved-prs/.ci/magic-modules/downstream-changelog-metadata-mergeprs.yml
+            params:
+              GITHUB_TOKEN: ((github-account.password))
+              DOWNSTREAM_REPOS: "{{','.join(vars.downstreams_with_changelogs)}}"
           - task: ensure-downstreams-merged
             file: mm-approved-prs/.ci/magic-modules/ensure-downstreams-merged.yml
             params:

.ci/magic-modules/downstream-changelog-metadata-mergeprs.yml

@@ -0,0 +1,28 @@
+---
+# This file takes in mm-approved-prs (magic-modules) to get code that it runs
+# and upstream PR.
+# Required information:
+#  - Github API token.
+#  - Upstream PR (magic modules) number
+# It produces no output.
+
+platform: linux
+
+image_resource:
+    type: docker-image
+    source:
+        # This task requires python + pip package 'pygithub'.
+        repository: gcr.io/magic-modules/python
+        tag: '1.0'
+
+inputs:
+    - name: mm-approved-prs
+
+params:
+    GITHUB_TOKEN: ""
+    DOWNSTREAM_REPOS: ""
+
+run:
+    path: mm-approved-prs/.ci/magic-modules/downstream_changelog_metadata.py
+    args:
+        - mm-approved-prs/.git/id

.ci/magic-modules/downstream-changelog-metadata.yml

@@ -0,0 +1,29 @@
+---
+# This file takes in mm-approved-prs (magic-modules) to get code that it runs
+# and upstream PR.
+# Required information:
+#  - Github API token.
+#  - Upstream PR (magic modules) number
+# It produces no output.
+
+platform: linux
+
+image_resource:
+    type: docker-image
+    source:
+        # This task requires python + pip package 'pygithub'.
+        repository: gcr.io/magic-modules/python
+        tag: '1.0'
+
+inputs:
+    - name: magic-modules
+    - name: mm-initial-pr
+
+params:
+    GITHUB_TOKEN: ""
+    DOWNSTREAM_REPOS: ""
+
+run:
+    path: magic-modules/.ci/magic-modules/downstream_changelog_metadata.py
+    args:
+        - mm-initial-pr/.git/id

.ci/magic-modules/downstream_changelog_metadata.py

@@ -0,0 +1,87 @@
+#!/usr/bin/env python
+"""
+Script to edit downstream PRs with CHANGELOG release note and label metadata.
+
+Usage:
+  ./downstream_changelog_info.py path/to/.git/.id
+  python /downstream_changelog_info.py
+
+Note that release_note/labels are authoritative - if empty or not set in the MM
+upstream PR, release notes will be removed from downstreams and labels
+unset.
+"""
+from pyutils import strutils, downstreams
+from github import Github
+import os
+import sys
+import argparse
+
+CHANGELOG_LABEL_PREFIX = "changelog: "
+
+def downstream_changelog_info(gh, upstream_pr_num, changelog_repos):
+  """Edit downstream PRs with CHANGELOG info.
+
+  Args:
+    gh: github.Github client
+    upstream_pr_num: Upstream PR number
+    changelog_repos: List of repo names to downstream changelog metadata for
+  """
+  # Parse CHANGELOG info from upstream
+  upstream_pr = gh.get_repo(downstreams.UPSTREAM_REPO)\
+                  .get_pull(upstream_pr_num)
+  release_note = strutils.get_release_note(upstream_pr.body)
+  labels_to_add = strutils.find_prefixed_labels(
+    [l.name for l in upstream_pr.labels],
+    CHANGELOG_LABEL_PREFIX)
+
+  print "Applying changelog info to downstreams for upstream PR %d:" % (
+    upstream_pr.number)
+  print "Release Note: \"%s\"" % release_note
+  print "Labels: [%s]" % labels_to_add
+
+  parsed_urls = downstreams.get_parsed_downstream_urls(gh, upstream_pr_num)
+  for repo_name, pulls in parsed_urls:
+    if repo_name not in changelog_repos:
+      print "[DEBUG] skipping repo %s with no CHANGELOG" % repo_name
+      continue
+
+    ghrepo = gh.get_repo(repo_name)
+    for _r, prnum in pulls:
+      pr = ghrepo.get_pull(int(prnum))
+      set_changelog_info(pr, release_note, labels_to_add)
+
+def set_changelog_info(gh_pull, release_note, labels_to_add):
+  """Set release note and labels on a downstream PR in Github.
+
+  Args:
+    gh_pull: A github.PullRequest.PullRequest handle
+    release_note: String of release note text to set
+    labels_to_add: List of strings. Changelog-related labels to add/replace.
+  """
+  print "Setting changelog info for downstream PR %s" % gh_pull.html_url
+  edited_body = strutils.set_release_note(release_note, gh_pull.body)
+  gh_pull.edit(body=edited_body)
+
+  # Get all non-changelog-related labels
+  labels_to_set = []
+  for l in gh_pull.get_labels():
+    if not l.name.startswith(CHANGELOG_LABEL_PREFIX):
+      labels_to_set.append(l.name)
+  labels_to_set += labels_to_add
+  gh_pull.set_labels(*labels_to_set)
+
+if __name__ == '__main__':
+  downstream_urls = os.environ.get('DOWNSTREAM_REPOS').split(',')
+  if len(downstream_urls) == 0:
+    print "Skipping, no downstreams repos given to downstream changelog info for"
+    sys.exit(0)
+
+  gh = Github(os.environ.get('GITHUB_TOKEN'))
+
+  assert len(sys.argv) == 2, "expected id filename as argument"
+  with open(sys.argv[1]) as f:
+    pr_num = int(f.read())
+
+    # TODO(emilymye): Replace this no-op print statement with code after
+    # verifying w/ pipeline.
+    downstream_changelog_info(gh, pr_num, downstream_urls)

.ci/magic-modules/ensure_downstreams_merged.py

@@ -1,36 +1,22 @@
 #!/usr/bin/env python
+"""
+This script takes the name of a file containing an upstream PR number
+and returns an error if not all of its downstreams have been merged.
 
-import get_downstream_prs
-from github import Github
+Required env vars:
+  GH_TOKEN: Github token
+"""
 import os
-import re
-import operator
-import itertools
 import sys
-
-
-def get_unmerged_prs(g, dependencies):
-  parsed_dependencies = [re.match(r'https://github.com/([\w-]+/[\w-]+)/pull/(\d+)', d).groups()
-          for d in dependencies]
-  parsed_dependencies.sort(key=operator.itemgetter(0))
-  unmerged_dependencies = []
-  # group those dependencies by repo - e.g. [("terraform-provider-google", ["123", "456"]), ...]
-  for r, pulls in itertools.groupby(parsed_dependencies, key=operator.itemgetter(0)):
-    repo = g.get_repo(r)
-    for pull in pulls:
-      # check whether the PR is merged - if it is, add it to the list.
-      pr = repo.get_pull(int(pull[1]))
-      if not pr.is_merged() and not pr.state == "closed":
-        unmerged_dependencies.append(pull)
-  return unmerged_dependencies
-
+from github import Github
+from pyutils import downstreams
 
 if __name__ == '__main__':
-  g = Github(os.environ.get('GH_TOKEN'))
-  assert len(sys.argv) == 2
-  id_filename = sys.argv[1]
-  unmerged = get_unmerged_prs(
-          g, get_downstream_prs.get_github_dependencies(
-              g, int(open(id_filename).read())))
+  assert len(sys.argv) == 2, "expected id filename as argument"
+  with open(sys.argv[1]) as f:
+    pr_num = int(f.read())
+
+  client = Github(os.environ.get('GH_TOKEN'))
+  unmerged = downstreams.find_unmerged_downstreams(client, pr_num)
   if unmerged:
     raise ValueError("some PRs are unmerged", unmerged)

.ci/magic-modules/get_downstream_prs.py

@@ -1,23 +1,14 @@
 #!/usr/bin/env python
-import functools
 import os
-import re
 import sys
 from github import Github
-
-def append_github_dependencies_to_list(lst, comment_body):
-  list_of_urls = re.findall(r'^depends: (https://github.com/.*)', comment_body, re.MULTILINE)
-  return lst + list_of_urls
-
-def get_github_dependencies(g, pr_number):
-  pull_request = g.get_repo('GoogleCloudPlatform/magic-modules').get_pull(pr_number)
-  comment_bodies = [c.body for c in pull_request.get_issue_comments()]
-  # "reduce" is "foldl" - apply this function to the result of the previous function and
-  # the next value in the iterable.
-  return functools.reduce(append_github_dependencies_to_list, comment_bodies, [])
+from pyutils import downstreams
 
 if __name__ == '__main__':
-  g = Github(os.environ.get('GH_TOKEN'))
-  assert len(sys.argv) == 2
-  for downstream_pr in get_github_dependencies(g, int(sys.argv[1])):
-    print downstream_pr
+  assert len(sys.argv) == 2, "expected a Github PR ID as argument"
+  upstream_pr = int(sys.argv[1])
+
+  downstream_urls = downstreams.get_downstream_urls(
+    Github(os.environ.get('GH_TOKEN')), upstream_pr)
+  for url in downstream_urls:
+    print url

.ci/magic-modules/get_merged_patches.py

@@ -1,25 +1,45 @@
 #!/usr/bin/env python
-import get_downstream_prs
-import itertools
-import re
-import operator
 import os
 import urllib
-
 from github import Github
+from pyutils import downstreams
+
+def get_merged_patches(gh):
+  """Download all merged patches for open upstream PRs.
+
+  Args:
+    gh: Github client to make calls to Github with.
+  """
+  open_pulls = gh.get_repo('GoogleCloudPlatform/magic-modules')\
+                 .get_pulls(state='open')
+  for open_pr in open_pulls:
+    print 'Downloading patches for upstream PR %d...' % open_pr.number
+    parsed_urls = downstreams.get_parsed_downstream_urls(gh, open_pr.number)
+    for repo_name, pulls in parsed_urls:
+      repo = gh.get_repo(repo_name)
+      for r, pr_num in pulls:
+          print 'Check to see if %s/%s is merged and should be downloaded\n' % (
+            r, pr_num)
+          downstream_pr = repo.get_pull(int(pr_num))
+          if downstream_pr.is_merged():
+            download_patch(r, downstream_pr)
+
+def download_patch(repo, pr):
+  """Download merged downstream PR patch.
+
+  Args:
+    pr: Github Pull request to download patch for
+  """
+  download_location = os.path.join('./patches', repo_name, '%d.patch' % pr.id)
+  print download_location
+  # Skip already downloaded patches
+  if os.path.exists(download_location):
+    return
+
+  if not os.path.exists(os.path.dirname(download_location)):
+      os.makedirs(os.path.dirname(download_location))
+  urllib.urlretrieve(pr.patch_url, download_location)
 
 if __name__ == '__main__':
-  g = Github(os.environ.get('GH_TOKEN'))
-  open_pulls = g.get_repo('GoogleCloudPlatform/magic-modules').get_pulls(state='open')
-  depends = [item for sublist in [get_downstream_prs.get_github_dependencies(g, open_pull.number) for open_pull in open_pulls] for item in sublist]
-  parsed_dependencies = [re.match(r'https://github.com/([\w-]+/[\w-]+)/pull/(\d+)', d).groups() for d in depends]
-  for r, pulls in itertools.groupby(parsed_dependencies, key=operator.itemgetter(0)):
-    repo = g.get_repo(r)
-    for pull in pulls:
-      pr = repo.get_pull(int(pull[1]))
-      print 'Checking %s to see if it should be downloaded.' % (pr,)
-      if pr.is_merged():
-        download_location = os.path.join('./patches', pull[0], pull[1] + '.patch')
-        if not os.path.exists(os.path.dirname(download_location)):
-          os.makedirs(os.path.dirname(download_location))
-        urllib.urlretrieve(pr.patch_url, download_location)
+  gh = Github(os.environ.get('GH_TOKEN'))
+  get_merged_patches(gh)

.ci/magic-modules/pyutils/README.md

@@ -0,0 +1,50 @@
+# Magic Modules CI Utils
+
+This directory manages all Python utils that the Magician uses to take upstream Magic Module PRs and generate and manage PRs in various downstream repos.
+
+What this shouldn't contain:
+
+- Python scripts called directly by Concourse jobs.
+- Non-Python code
+
+## Tests
+
+Currently we use the standard [unittest](https://docs.python.org/3/library/unittest.html) library. Because CI development is mostly done locally on your developer machine before being directly deployed, these tests are run manually.
+
+This section reviews running/writing tests for someone fairly new to Python/unittest, so some of this information is just from unittest docs.
+
+### Running tests
+
+Set a test environment variable to make calls to Github:
+```
+export TEST_GITHUB_TOKEN=...
+```
+
+Otherwise, tests calling Github will be ignored (or likely be rate-limited).
+```
+cd pyutils
+
+python -m unittest discover -p "*_test.py"
+python ./changelog_utils_test.py
+```
+
+Read [unittest](https://docs.python.org/3/library/unittest.html#command-line-interface) docs to see how to run tests at finer granularity.
+
+*NOTE*: Don't forget to delete .pyc files if you feel like tests aren't reflecting your changes!
+
+### Writing Tests:
+
+This is mostly a very shallow review of unittest, but your test should inherit from the `unittest.TestCase` class in some way (i.e. we haven't had the need to write our own TestCase-inheriting Test class but feel free to in the future if needed).
+
+```
+class MyModuleTest(unittest.TestCase):
+```
+
+Make sure to include the following at the bottom of your test file, so it defaults to running the tests in this file if run as a normal Python script.
+```
+if __name__ == '__main__':
+  unittest.main()
+```
+
+
+