7730: gh data sourcing and screenshot generation

www/.gitignore

@@ -31,3 +31,6 @@ node_modules
 /yarn.lock
 src/cache
 /package-lock.json
+
+# Env variables
+.env.development

www/README.md

@@ -6,3 +6,17 @@ Run locally with
 
 - `yarn install`
 - `yarn run develop`
+
+## Working with the starter showcase
+
+To develop on the starter showcase, you'll need to supply a GitHub personal access token.
+
+1. Create a personal access token in your GitHub [Developer settings](https://github.com/settings/tokens).
+2. In the new token's settings, grant that token the "public_repo" scope.
+3. Create a file in the root of `www` called `.env.development`, and add the token to that file like so:
+
+```
+GITHUB_API_TOKEN=YOUR_TOKEN_HERE
+```
+
+The `.env.development` file is ignored by git. Your token should never be committed.

www/gatsby-config.js

@@ -28,21 +28,6 @@ module.exports = {
         path: `${__dirname}/../packages/`,
       },
     },
-    {
-      resolve: `gatsby-source-filesystem`,
-      options: {
-        name: `StarterShowcaseImages`,
-        path: `${__dirname}/src/data/StarterShowcase/generatedScreenshots`,
-      },
-    },
-    //   need to have the img processing first? https://github.com/gatsbyjs/gatsby/issues/5196
-    {
-      resolve: `gatsby-source-filesystem`,
-      options: {
-        name: `StarterShowcaseData`,
-        path: `${__dirname}/src/data/StarterShowcase/startersData`,
-      },
-    },
     {
       resolve: `gatsby-plugin-typography`,
       options: {
@@ -183,7 +168,12 @@ module.exports = {
         endpoint: `https://gatsbyjs.us17.list-manage.com/subscribe/post?u=1dc33f19eb115f7ebe4afe5ee&id=f366064ba7`,
       },
     },
-    `gatsby-transformer-screenshot`,
+    {
+      resolve: `gatsby-transformer-screenshot`,
+      options: {
+        nodeTypes: [`StartersYaml`],
+      },
+    },
     `gatsby-plugin-subfont`,
     // {
     // resolve: `gatsby-plugin-guess-js`,

www/gatsby-node.js

@@ -6,6 +6,28 @@ const fs = require(`fs-extra`)
 const slash = require(`slash`)
 const slugify = require(`slugify`)
 const url = require(`url`)
+const getpkgjson = require(`get-package-json-from-github`)
+const parseGHUrl = require("parse-github-url")
+const { GraphQLClient } = require("graphql-request")
+
+require(`dotenv`).config({
+  path: `.env.${process.env.NODE_ENV}`,
+})
+
+if (process.env.NODE_ENV === "production" && !process.env.GITHUB_API_TOKEN) {
+  throw new Error(
+    `A GitHub token is required to build the site. Check the README.`
+  )
+}
+
+// used to gather repo data on starters
+const githubApiClient = process.env.GITHUB_API_TOKEN
+  ? new GraphQLClient(`https://api.github.com/graphql`, {
+      headers: {
+        authorization: `Bearer ${process.env.GITHUB_API_TOKEN}`,
+      },
+    })
+  : null
 
 const localPackages = `../packages`
 const localPackagesArr = []
@@ -111,10 +133,6 @@ exports.createPages = ({ graphql, actions }) => {
                 fields {
                   slug
                   package
-                  starterShowcase {
-                    slug
-                    stub
-                  }
                 }
                 frontmatter {
                   title
@@ -153,6 +171,21 @@ exports.createPages = ({ graphql, actions }) => {
               }
             }
           }
+          allStartersYaml {
+            edges {
+              node {
+                id
+                fields {
+                  starterShowcase {
+                    slug
+                    stub
+                  }
+                }
+                url
+                repo
+              }
+            }
+          }
           allNpmPackage {
             edges {
               node {
@@ -236,27 +269,30 @@ exports.createPages = ({ graphql, actions }) => {
         })
       })
 
-      // Create starters.
-      const starters = _.filter(result.data.allMarkdownRemark.edges, edge => {
+      // Create starter pages.
+      const starters = _.filter(result.data.allStartersYaml.edges, edge => {
         const slug = _.get(edge, `node.fields.starterShowcase.slug`)
-        if (!slug) return null
-        else return edge
+        if (!slug) {
+          return null
+        } else {
+          return edge
+        }
       })
+
       const starterTemplate = path.resolve(
         `src/templates/template-starter-showcase.js`
       )
 
       starters.forEach((edge, index) => {
         createPage({
-          path: `/starters${edge.node.fields.starterShowcase.slug}`, // required
+          path: `/starters${edge.node.fields.starterShowcase.slug}`,
           component: slash(starterTemplate),
           context: {
             slug: edge.node.fields.starterShowcase.slug,
             stub: edge.node.fields.starterShowcase.stub,
           },
         })
       })
-      // END Create starters.
 
       // Create contributor pages.
       result.data.allAuthorYaml.edges.forEach(edge => {
@@ -387,13 +423,6 @@ exports.onCreateNode = ({ node, actions, getNode, getNodes }) => {
       })
       createNodeField({ node, name: `package`, value: true })
     }
-    if (
-      // starter showcase
-      fileNode.sourceInstanceName === `StarterShowcaseData` &&
-      parsedFilePath.name !== `README`
-    ) {
-      createNodesForStarterShowcase({ node, getNode, getNodes, actions })
-    } // end starter showcase
     if (slug) {
       createNodeField({ node, name: `anchor`, value: slugToAnchor(slug) })
       createNodeField({ node, name: `slug`, value: slug })
@@ -408,6 +437,105 @@ exports.onCreateNode = ({ node, actions, getNode, getNodes }) => {
     const cleaned = parsed.hostname + parsed.pathname
     slug = `/showcase/${slugify(cleaned)}`
     createNodeField({ node, name: `slug`, value: slug })
+  } else if (node.internal.type === `StartersYaml` && node.repo) {
+    // To develop on the starter showcase, you'll need a GitHub
+    // personal access token. Check the `www` README for details.
+    // Default fields are to avoid graphql errors.
+    const { owner, name: repoStub } = parseGHUrl(node.repo)
+    const defaultFields = {
+      slug: ``,
+      stub: ``,
+      name: ``,
+      description: ``,
+      stars: 0,
+      lastUpdated: ``,
+      owner: ``,
+      githubFullName: ``,
+      allDependencies: [],
+      gatsbyDependencies: [],
+      miscDependencies: [],
+    }
+
+    if (!process.env.GITHUB_API_TOKEN) {
+      return createNodeField({
+        node,
+        name: `starterShowcase`,
+        value: {
+          ...defaultFields,
+        },
+      })
+    }
+
+    Promise.all([
+      getpkgjson(node.repo),
+      githubApiClient.request(`
+          query {
+            repository(owner:"${owner}", name:"${repoStub}") {
+              name
+              stargazers {
+                totalCount
+              }
+              createdAt
+              updatedAt
+              owner {
+                login
+              }
+              nameWithOwner
+            }
+          }
+        `),
+    ])
+      .then(results => {
+        const [pkgjson, githubData] = results
+        const {
+          stargazers: { totalCount: stars },
+          updatedAt: lastUpdated,
+          owner: { login: owner },
+          name,
+          nameWithOwner: githubFullName,
+        } = githubData.repository
+
+        const { dependencies = [], devDependencies = [] } = pkgjson
+        const allDependencies = Object.entries(dependencies).concat(
+          Object.entries(devDependencies)
+        )
+        const starterShowcaseFields = {
+          slug: `/${repoStub}/`,
+          stub: repoStub,
+          name,
+          description: pkgjson.description,
+          stars,
+          lastUpdated,
+          owner,
+          githubFullName,
+          allDependencies,
+          gatsbyDependencies: allDependencies
+            .filter(
+              ([key, _]) => ![`gatsby-cli`, `gatsby-link`].includes(key) // remove stuff everyone has
+            )
+            .filter(([key, _]) => key.includes(`gatsby`)),
+          miscDependencies: allDependencies.filter(
+            ([key, _]) => !key.includes(`gatsby`)
+          ),
+        }
+        createNodeField({
+          node,
+          name: `starterShowcase`,
+          value: starterShowcaseFields,
+        })
+      })
+      .catch(err => {
+        console.log(
+          `\nError getting repo data. Your GitHub token may be invalid`
+        )
+        return createNodeField({
+          node,
+          name: `starterShowcase`,
+          value: {
+            ...defaultFields,
+          },
+        })
+      })
   }
 
   // Community/Creators Pages
@@ -440,59 +568,6 @@ exports.onPostBuild = () => {
   )
 }
 
-// Starter Showcase related code
-const { createFilePath } = require(`gatsby-source-filesystem`)
-const gitFolder = `./src/data/StarterShowcase/generatedGithubData`
-function createNodesForStarterShowcase({ node, getNode, getNodes, actions }) {
-  const { createNodeField, createParentChildLink } = actions
-  if (node.internal.type === `MarkdownRemark`) {
-    const slug = createFilePath({
-      node,
-      getNode,
-      basePath: `startersData`,
-    })
-    // preprocessing
-    const stub = slug.replace(/\//gi, ``)
-    var fromPath = path.join(gitFolder, `${stub}.json`)
-    var data = fs.readFileSync(fromPath, `utf8`)
-    const ghdata = JSON.parse(data)
-    if (ghdata.repository && ghdata.repository.url)
-      ghdata.repository = ghdata.repository.url // flatten a potential object into a string. weird quirk.
-    const { repoMetadata, dependencies = [], devDependencies = [] } = ghdata
-    const allDependencies = Object.entries(dependencies).concat(
-      Object.entries(devDependencies)
-    )
-    // make an object to stick into a Field
-    const starterShowcaseFields = {
-      slug,
-      stub,
-      date: new Date(node.frontmatter.date),
-      githubData: ghdata,
-      // nice-to-have destructures of githubData
-      description: ghdata.description,
-      stars: repoMetadata.stargazers_count,
-      lastUpdated: repoMetadata.created_at,
-      owner: repoMetadata.owner,
-      githubFullName: repoMetadata.full_name,
-      allDependencies,
-      gatsbyDependencies: allDependencies
-        .filter(
-          ([key, _]) => ![`gatsby-cli`, `gatsby-link`].includes(key) // remove stuff everyone has
-        )
-        .filter(([key, _]) => key.includes(`gatsby`)),
-      miscDependencies: allDependencies.filter(
-        ([key, _]) => !key.includes(`gatsby`)
-      ),
-    }
-    createNodeField({
-      node,
-      name: `starterShowcase`,
-      value: starterShowcaseFields,
-    })
-  }
-}
-// End Starter Showcase related code
-
 // limited logging for debug purposes
 let limitlogcount = 0
 function log(max) {

www/package.json

@@ -5,6 +5,7 @@
   "author": "Kyle Mathews <[email protected]>",
   "dependencies": {
     "bluebird": "^3.5.1",
+    "dotenv": "^6.0.0",
     "email-validator": "^1.1.1",
     "fuse.js": "^3.2.0",
     "gatsby": "^2.0.7",
@@ -53,6 +54,7 @@
     "mitt": "^1.1.3",
     "mousetrap": "^1.6.1",
     "parse-filepath": "^1.0.2",
+    "parse-github-url": "^1.0.2",
     "prismjs": "^1.14.0",
     "qs": "^6.5.2",
     "query-string": "^6.1.0",