PCC-842, PCC-814, PCC-824 Added: Drupal import command.

packages/cli/package.json

@@ -39,23 +39,27 @@
   "dependencies": {
     "@pantheon-systems/pcc-sdk-core": "latest",
     "axios": "^1.6.0",
+    "bluebird": "^3.7.2",
     "boxen": "^7.1.1",
     "chalk": "^5.3.0",
     "dayjs": "^1.11.9",
     "fs-extra": "^11.1.1",
     "get-port": "^7.0.0",
-    "google-auth-library": "^8.9.0",
+    "google-auth-library": "^9.4.0",
+    "googleapis": "^129.0.0",
     "inquirer": "^8.2.6",
     "nunjucks": "^3.2.4",
     "octokit": "^2.1.0",
     "open": "^9.1.0",
     "ora": "^6.3.1",
     "package-json": "^8.1.1",
+    "query-string": "^8.1.0",
     "server-destroy": "^1.0.1",
     "yargs": "^17.7.2"
   },
   "devDependencies": {
     "@babel/preset-env": "7.21.5",
+    "@types/bluebird": "^3.5.42",
     "@types/fs-extra": "^11.0.1",
     "@types/inquirer": "^9.0.3",
     "@types/jest": "29.5.1",

packages/cli/src/cli/commands/import.ts

@@ -0,0 +1,172 @@
+import { randomUUID } from "crypto";
+import { exit } from "process";
+import axios, { AxiosError } from "axios";
+import Promise from "bluebird";
+import chalk from "chalk";
+import type { GaxiosResponse } from "gaxios";
+import { OAuth2Client } from "google-auth-library";
+import { drive_v3, google } from "googleapis";
+import queryString from "query-string";
+import AddOnApiHelper from "../../lib/addonApiHelper";
+import { getLocalAuthDetails } from "../../lib/localStorage";
+import { Logger } from "../../lib/logger";
+import { errorHandler } from "../exceptions";
+import login from "./login";
+
+type ImportParams = {
+  baseUrl: string;
+  siteId: string;
+};
+
+async function getDrupalPosts(url: string) {
+  try {
+    console.log(`Importing from ${url}`);
+    const result = (await axios.get(url)).data;
+
+    return {
+      nextURL: result.links?.next?.href,
+      posts: result.data,
+      includedData: result.included,
+    };
+  } catch (e: any) {
+    console.log(e, e.message);
+    throw e;
+  }
+}
+
+export const importFromDrupal = errorHandler<ImportParams>(
+  async ({ baseUrl, siteId }: ImportParams) => {
+    const logger = new Logger();
+
+    if (baseUrl) {
+      try {
+        new URL(baseUrl);
+      } catch (_err) {
+        logger.error(
+          chalk.red(
+            `ERROR: Value provided for \`baseUrl\` is not a valid URL. `,
+          ),
+        );
+        exit(1);
+      }
+    }
+
+    await login(["https://www.googleapis.com/auth/drive.file"]);
+    let authDetails = await getLocalAuthDetails();
+    const oauth2Client = new OAuth2Client();
+    oauth2Client.setCredentials(authDetails!);
+    const drive = google.drive({
+      version: "v3",
+      auth: oauth2Client,
+    });
+
+    const folderRes = (await drive.files
+      .create({
+        fields: "id,name",
+        requestBody: {
+          name: `PCC Import from Drupal on ${new Date().toLocaleDateString()} unique id: ${randomUUID()}`,
+          mimeType: "application/vnd.google-apps.folder",
+        },
+      })
+      .catch(console.error)) as GaxiosResponse<drive_v3.Schema$File>;
+
+    // Get results.
+    let page = 0;
+    let { url, query } = queryString.parseUrl(baseUrl);
+    query.include = "field_author,field_topics";
+    let allPosts: any[] = [];
+    let allIncludedData: any[] = [];
+    let nextURL = queryString.stringifyUrl({ url, query });
+
+    do {
+      const drupalData = await getDrupalPosts(nextURL);
+      nextURL = drupalData.nextURL;
+
+      if (drupalData.posts?.length) {
+        allPosts.push(...drupalData.posts);
+      }
+
+      if (drupalData.includedData?.length) {
+        allIncludedData.push(...drupalData.includedData);
+      }
+    } while (nextURL != null && ++page < 1000);
+
+    logger.log(chalk.green(`Retrieved ${allPosts.length} posts after ${page} pages`));
+
+    // Ensure that these metadata fields exist.
+    await AddOnApiHelper.addSiteMetadataField(
+      siteId,
+      "blog",
+      "drupalId",
+      "string",
+    );
+    await AddOnApiHelper.addSiteMetadataField(
+      siteId,
+      "blog",
+      "author",
+      "string",
+    );
+
+    await Promise.map(
+      allPosts,
+      async (post) => {
+        if (post?.attributes?.body == null) {
+          console.log("Skipping post", Object.keys(post));
+          return;
+        }
+
+        // Create the google doc.
+        const authorName = allIncludedData.find(
+          (x) => x.id === post.relationships.field_author.data.id,
+        )?.attributes?.title;
+        const res = (await drive.files.create({
+          requestBody: {
+            // Name from the article.
+            name: post.attributes.title,
+            mimeType: "application/vnd.google-apps.document",
+            parents: [folderRes.data.id!],
+          },
+          media: {
+            mimeType: "text/html",
+            body: post.attributes.body.processed,
+          },
+        })) as GaxiosResponse<drive_v3.Schema$File>;
+
+        // Add it to the PCC site.
+        console.log("Get document", res.data.id);
+        await AddOnApiHelper.getDocument(res.data.id!, true);
+
+        try {
+          await AddOnApiHelper.updateDocument(
+            res.data.id!,
+            siteId,
+            post.attributes.title,
+            post.relationships.field_topics?.data
+              ?.map(
+                (topic: any) =>
+                  allIncludedData.find((x: any) => x.id === topic.id)
+                    ?.attributes?.name,
+              )
+              .filter((x: string | undefined) => x != null) || [],
+            {
+              author: authorName,
+              drupalId: post.id,
+            },
+          );
+        } catch (e: any) {
+          console.error(e.response?.data);
+          throw e;
+        }
+      },
+      {
+        concurrency: 20,
+      },
+    );
+
+    logger.log(
+      chalk.green(
+        `Successfully imported ${allPosts.length} documents into ${folderRes.data.name}`,
+      ),
+    );
+  },
+);

packages/cli/src/cli/commands/login.ts

@@ -20,17 +20,26 @@ nunjucks.configure({ autoescape: true });
 
 const OAUTH_SCOPES = ["https://www.googleapis.com/auth/userinfo.email"];
 
-function login(): Promise<void> {
+function login(extraScopes: string[] = []): Promise<void> {
   return new Promise(
     // eslint-disable-next-line no-async-promise-executor -- Handling promise rejection in the executor
     async (resolve, reject) => {
       const spinner = ora("Logging you in...").start();
       try {
         const authData = await getLocalAuthDetails();
         if (authData) {
-          const jwtPayload = parseJwt(authData.id_token as string);
-          spinner.succeed(`You are already logged in as ${jwtPayload.email}.`);
-          return;
+          let scopes = authData.scope?.split(" ");
+
+          if (
+            !extraScopes?.length ||
+            extraScopes.find((x) => scopes?.includes(x))
+          ) {
+            const jwtPayload = parseJwt(authData.id_token as string);
+            spinner.succeed(
+              `You are already logged in as ${jwtPayload.email}.`,
+            );
+            return resolve();
+          }
         }
 
         const oAuth2Client = new OAuth2Client({
@@ -41,7 +50,7 @@ function login(): Promise<void> {
         // Generate the url that will be used for the consent dialog.
         const authorizeUrl = oAuth2Client.generateAuthUrl({
           access_type: "offline",
-          scope: OAUTH_SCOPES,
+          scope: [...OAUTH_SCOPES, ...extraScopes],
         });
 
         const server = http.createServer(async (req, res) => {
@@ -92,4 +101,4 @@ function login(): Promise<void> {
     },
   );
 }
-export default errorHandler<void>(login);
+export default errorHandler<string[]>(login);

packages/cli/src/cli/index.ts

@@ -3,6 +3,7 @@ import yargs from "yargs";
 import { hideBin } from "yargs/helpers";
 import checkUpdate from "../lib/checkUpdate";
 import { generatePreviewLink } from "./commands/documents";
+import { importFromDrupal } from "./commands/import";
 import init from "./commands/init";
 import login from "./commands/login";
 import logout from "./commands/logout";
@@ -353,6 +354,41 @@ yargs(hideBin(process.argv))
         );
     },
   )
+  .command(
+    "import",
+    "Imports posts from a Drupal JSON API endpoint into PCC",
+    (yargs) => {
+      yargs
+        .strictCommands()
+        .demandCommand()
+        .command(
+          "drupal <baseUrl> <siteId>",
+          "Imports all articles from a Drupal JSON API endpoint into a new Google Drive folder and connects them to a target PCC site",
+          (yargs) => {
+            yargs
+              .strictCommands()
+              .positional("baseUrl", {
+                describe:
+                  'URL of drupal json API endpoint such as "https://example.com/jsonapi/node/blog".',
+                type: "string",
+              })
+              .positional("siteId", {
+                describe: "Id of site to import articles into.",
+                type: "string",
+              })
+              .demandOption(["baseUrl", "siteId"]);
+          },
+          async (args) =>
+            await importFromDrupal({
+              baseUrl: args.baseUrl as string,
+              siteId: args.siteId as string,
+            }),
+        );
+    },
+    async () => {
+      // noop
+    },
+  )
   .command(
     "whoami",
     "Print information about yourself.",

packages/cli/src/lib/addonApiHelper.ts

@@ -49,14 +49,81 @@
     return authDetails.id_token as string;
   }
 
-  static async getDocument(documentId: string): Promise<Article> {
+  static async getDocument(
+    documentId: string,
+    insertIfMissing: boolean = false,
+  ): Promise<Article> {
     const idToken = await this.getIdToken();
 
-    const resp = await axios.get(`${DOCUMENT_ENDPOINT}/${documentId}`, {
-      headers: {
-        Authorization: `Bearer ${idToken}`,
+    const resp = await axios.get(
+      `${DOCUMENT_ENDPOINT}/${documentId}?insertIfMissing=${insertIfMissing.toString()}`,
+      {
+        headers: {
+          Authorization: `Bearer ${idToken}`,
+        },
       },
+    );
+
+    return resp.data as Article;
+  }
+
+  static async addSiteMetadataField(
+    siteId: string,
+    contentType: string,
+    fieldTitle: string,
+    fieldType: string,
+  ): Promise<void> {
+    const idToken = await this.getIdToken();
+
+    await axios.post(
+      `${SITE_ENDPOINT}/${siteId}/metadata`,
+      {
+        contentType,
+        field: {
+          title: fieldTitle,
+          type: fieldType,
+        },
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${idToken}`,
+          "Content-Type": "application/json",
+        },
+      },
+    );
+  }
+
+  static async updateDocument(
+    documentId: string,
+    siteId: string,
+    title: string,
+    tags: string[],
+    metadataFields: any,
+  ): Promise<Article> {
+    const idToken = await this.getIdToken();
+
+    console.log("update document", {
+      documentId,
+      siteId,
+      title,
+      tags,
+      metadataFields,
     });
+    const resp = await axios.patch(
+      `${DOCUMENT_ENDPOINT}/${documentId}`,
+      {
+        siteId,
+        tags,
+        title,
+        metadataFields,
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${idToken}`,
+          "Content-Type": "application/json",
+        },
+      },
+    );
 
     return resp.data as Article;
   }