-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #160 from pantheon-systems/PCC-842-create-script-w…
…hich-transforms-drupal-json-api-export-to-pcc-powered-google-documents PCC-842, PCC-814, PCC-824 Added: Drupal import command.
- Loading branch information
Showing
8 changed files
with
5,273 additions
and
6,852 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,237 @@ | ||
| import { randomUUID } from "crypto"; | ||
| import { exit } from "process"; | ||
| import axios, { AxiosError } from "axios"; | ||
| import Promise from "bluebird"; | ||
| import chalk from "chalk"; | ||
| import type { GaxiosResponse } from "gaxios"; | ||
| import { OAuth2Client } from "google-auth-library"; | ||
| import { drive_v3, google } from "googleapis"; | ||
| import queryString from "query-string"; | ||
| import AddOnApiHelper from "../../lib/addonApiHelper"; | ||
| import { getLocalAuthDetails } from "../../lib/localStorage"; | ||
| import { Logger } from "../../lib/logger"; | ||
| import { errorHandler } from "../exceptions"; | ||
| import login from "./login"; | ||
|
|
||
| type ImportParams = { | ||
| baseUrl: string; | ||
| siteId: string; | ||
| verbose: boolean; | ||
| }; | ||
|
|
||
| interface DrupalPost { | ||
| id: string; | ||
| attributes?: { | ||
| body?: { | ||
| processed: string; | ||
| }; | ||
| title: string; | ||
| }; | ||
| relationships: { | ||
| field_author: { | ||
| data: { | ||
| id: string; | ||
| }; | ||
| }; | ||
| field_topics?: { | ||
| data: [ | ||
| { | ||
| id: string; | ||
| }, | ||
| ]; | ||
| }; | ||
| }; | ||
| } | ||
|
|
||
| interface DrupalTopic { | ||
| id: string; | ||
| attributes?: { | ||
| name: string; | ||
| }; | ||
| } | ||
|
|
||
| interface DrupalIncludedData { | ||
| id: string; | ||
| attributes?: { | ||
| name: string; | ||
| title: string; | ||
| }; | ||
| } | ||
|
|
||
| async function getDrupalPosts(url: string) { | ||
| try { | ||
| console.log(`Importing from ${url}`); | ||
| const result = (await axios.get(url)).data; | ||
|
|
||
| return { | ||
| nextURL: result.links?.next?.href, | ||
| posts: result.data, | ||
| includedData: result.included, | ||
| }; | ||
| } catch (e) { | ||
| console.error(e); | ||
| throw e; | ||
| } | ||
| } | ||
|
|
||
| export const importFromDrupal = errorHandler<ImportParams>( | ||
| async ({ baseUrl, siteId, verbose }: ImportParams) => { | ||
| const logger = new Logger(); | ||
|
|
||
| if (baseUrl) { | ||
| try { | ||
| new URL(baseUrl); | ||
| } catch (_err) { | ||
| logger.error( | ||
| chalk.red( | ||
| `ERROR: Value provided for \`baseUrl\` is not a valid URL. `, | ||
| ), | ||
| ); | ||
| exit(1); | ||
| } | ||
| } | ||
|
|
||
| await login(["https://www.googleapis.com/auth/drive.file"]); | ||
| const authDetails = await getLocalAuthDetails(); | ||
|
|
||
| if (!authDetails) { | ||
| logger.error(chalk.red(`ERROR: Failed to retrieve login details. `)); | ||
| exit(1); | ||
| } | ||
|
|
||
| const oauth2Client = new OAuth2Client(); | ||
| oauth2Client.setCredentials(authDetails); | ||
| const drive = google.drive({ | ||
| version: "v3", | ||
| auth: oauth2Client, | ||
| }); | ||
|
|
||
| const folderRes = (await drive.files | ||
| .create({ | ||
| fields: "id,name", | ||
| requestBody: { | ||
| name: `PCC Import from Drupal on ${new Date().toLocaleDateString()} unique id: ${randomUUID()}`, | ||
| mimeType: "application/vnd.google-apps.folder", | ||
| }, | ||
| }) | ||
| .catch(console.error)) as GaxiosResponse<drive_v3.Schema$File>; | ||
|
|
||
| const folderId = folderRes.data.id; | ||
|
|
||
| if (folderId == null) { | ||
| logger.error( | ||
| chalk.red( | ||
| `Failed to create parent folder which we would have imported posts into`, | ||
| ), | ||
| ); | ||
| exit(1); | ||
| } | ||
|
|
||
| // Get results. | ||
| let page = 0; | ||
| const { url, query } = queryString.parseUrl(baseUrl); | ||
| query.include = "field_author,field_topics"; | ||
| const allPosts: DrupalPost[] = []; | ||
| const allIncludedData: DrupalIncludedData[] = []; | ||
| let nextURL = queryString.stringifyUrl({ url, query }); | ||
|
|
||
| do { | ||
| const drupalData = await getDrupalPosts(nextURL); | ||
| nextURL = drupalData.nextURL; | ||
|
|
||
| if (drupalData.posts?.length) { | ||
| allPosts.push(...drupalData.posts); | ||
| } | ||
|
|
||
| if (drupalData.includedData?.length) { | ||
| allIncludedData.push(...drupalData.includedData); | ||
| } | ||
| } while (nextURL != null && ++page < 1000); | ||
|
|
||
| logger.log( | ||
| chalk.green(`Retrieved ${allPosts.length} posts after ${page} pages`), | ||
| ); | ||
|
|
||
| // Ensure that these metadata fields exist. | ||
| await AddOnApiHelper.addSiteMetadataField( | ||
| siteId, | ||
| "blog", | ||
| "drupalId", | ||
| "string", | ||
| ); | ||
| await AddOnApiHelper.addSiteMetadataField( | ||
| siteId, | ||
| "blog", | ||
| "author", | ||
| "string", | ||
| ); | ||
|
|
||
| await Promise.map( | ||
| allPosts, | ||
| async (post) => { | ||
| if (post?.attributes?.body == null) { | ||
| console.log("Skipping post", Object.keys(post)); | ||
| return; | ||
| } | ||
|
|
||
| // Create the google doc. | ||
| const authorName: string | undefined = allIncludedData.find( | ||
| (x) => x.id === post.relationships.field_author.data.id, | ||
| )?.attributes?.title; | ||
|
|
||
| const res = (await drive.files.create({ | ||
| requestBody: { | ||
| // Name from the article. | ||
| name: post.attributes.title, | ||
| mimeType: "application/vnd.google-apps.document", | ||
| parents: [folderId], | ||
| }, | ||
| media: { | ||
| mimeType: "text/html", | ||
| body: post.attributes.body.processed, | ||
| }, | ||
| })) as GaxiosResponse<drive_v3.Schema$File>; | ||
| const fileId = res.data.id; | ||
|
|
||
| if (!fileId) { | ||
| throw new Error(`Failed to create file for ${post.attributes.title}`); | ||
| } | ||
|
|
||
| // Add it to the PCC site. | ||
| await AddOnApiHelper.getDocument(fileId, true); | ||
|
|
||
| try { | ||
| await AddOnApiHelper.updateDocument( | ||
| fileId, | ||
| siteId, | ||
| post.attributes.title, | ||
| post.relationships.field_topics?.data | ||
| ?.map( | ||
| (topic: DrupalTopic) => | ||
| allIncludedData.find((x) => x.id === topic.id)?.attributes | ||
| ?.name, | ||
| ) | ||
| .filter((x: string | undefined): x is string => x != null) || [], | ||
| { | ||
| author: authorName, | ||
| drupalId: post.id, | ||
| }, | ||
| verbose, | ||
| ); | ||
| } catch (e) { | ||
| console.error(e instanceof AxiosError ? e.response?.data : e); | ||
| throw e; | ||
| } | ||
| }, | ||
| { | ||
| concurrency: 20, | ||
| }, | ||
| ); | ||
|
|
||
| logger.log( | ||
| chalk.green( | ||
| `Successfully imported ${allPosts.length} documents into ${folderRes.data.name}`, | ||
| ), | ||
| ); | ||
| }, | ||
| ); |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.