From 5f04ae7cd9f85468423a144fe408b86402d395f6 Mon Sep 17 00:00:00 2001 From: g-div Date: Thu, 7 Dec 2017 15:25:33 +0100 Subject: [PATCH] feat(importer): add disqus importer --- README.md | 8 +++ package.json | 4 +- src/importer.js | 95 ++++++++++++++++++++++++++++++++++ test/disqus.xml | 133 ++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 239 insertions(+), 1 deletion(-) create mode 100644 src/importer.js create mode 100644 test/disqus.xml diff --git a/README.md b/README.md index bf74f89..cb3b3eb 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,14 @@ In order to be able to edit your config file and your SQL database files, you ma docker run -p 3000:3000 -v $(pwd):/usr/src/app -d gka/schnack ``` +### Import comments from disqus + +You can import comments from your [disqus XML export](https://help.disqus.com/customer/portal/articles/472149-comments-export) as following: + +``` +npm run import -- disqus.xml +``` + ### Who is behind Schnack? Schnack is [yet another](https://github.com/gka/canvid/) happy collaboration between [Webkid](https://webkid.io/) and [Gregor Aisch](https://www.vis4.net). diff --git a/package.json b/package.json index b54aae2..41f59d1 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,8 @@ "rollup-plugin-node-resolve": "^3.0.0", "rollup-plugin-string": "^2.0.2", "rollup-plugin-template": "^1.0.10", - "rollup-plugin-uglify": "^2.0.1" + "rollup-plugin-uglify": "^2.0.1", + "to-markdown": "^3.1.0", + "xml2js": "^0.4.19" } } diff --git a/src/importer.js b/src/importer.js new file mode 100644 index 0000000..aa3f434 --- /dev/null +++ b/src/importer.js @@ -0,0 +1,95 @@ +const fs = require('fs'); +const path = require('path'); +const sqlite = require('sqlite'); +const queries = require('./db/queries'); +const xml2js = require('xml2js'); +const toMarkdown = require('to-markdown'); +const dbPromise = sqlite.open('./comments.db', { Promise }); + +const filename = process.argv.slice(2, 3).pop(); +if (!filename) { + console.error('Pass the filepath to your XML file as argument'); + process.exit(1); +} + +function run() { + const parser = new xml2js.Parser({explicitArray: false}); + const xmlFilePath = path.resolve(__dirname, '..', filename); + fs.readFile(xmlFilePath, (err, data) => + parser.parseString(data, async function (err, result) { + if (err) console.error(`Error parsing the XML file '${xmlFilePath}':`, err); + const db = await dbPromise; + + // for each post save it and store the id + // for each post that is a reply replace reply_to with the stored id + const threads = result.disqus.thread; + const posts = result.disqus.post.map(comment => importComment(threads, comment)); + + for (post of posts) { + const newComment = await saveComment(post); + post.new_id = newComment; + } + + for (post of posts) { + const replies = posts.filter(p => p.comment[3] === post.id); // replies to current post + if (replies) { + for (reply of replies) { + const { id, new_id } = post; + const res = await db.run(`UPDATE comment SET reply_to = ? WHERE reply_to = ?`, [new_id, id]); + } + } + } + }) + ); +}; + +async function saveComment(post) { + const db = await dbPromise; + const { comment, author } = post; + + if (!author.username) { + author.username = 'Anonymous Guest'; + } + const user = [ + 'disqus', + author.username, + author.name, + author.username, + 0 + ]; + + try { + await db.run(queries.create_user, user); + const newUser = await db.get(queries.find_user, ['disqus', author.username]); + if (newUser.id) comment.unshift(newUser.id); // push user_id to the front + const res = await db.run(`INSERT INTO comment + (user_id, slug, comment, reply_to, created_at, approved, rejected) + VALUES (?,?,?,?,?,?,0)`, comment); + return res.lastID; + } catch (err) { + console.error(`Error saving the comment for the slug ${comment[0]}:`, err); + } +}; + +function importComment(threads, comment) { + const { author } = comment; + const thread = threads.filter(thread => thread.$['dsq:id'] === comment.thread.$['dsq:id'])[0].id + const reply_to = comment.parent ? comment.parent.$['dsq:id'] : null; + const message = toMarkdown(comment.message.trim()); + const timestamp = comment.createdAt; + const approved = (comment.isDeleted === "true" || comment.isSpam === "true") ? 0 : 1; + + return { + comment: [ + thread, + message, + reply_to, + timestamp, + approved + ], + id: comment.$['dsq:id'], + author + }; +}; + +run(); diff --git a/test/disqus.xml b/test/disqus.xml new file mode 100644 index 0000000..088f231 --- /dev/null +++ b/test/disqus.xml @@ -0,0 +1,133 @@ + + + + webkid + General + true + + + foo + webkid + + http://localhost:3000/foo/ + Welcome + + 2014-09-26T20:11:04Z + + info@webkid.io + webkid + false + webkid + + 127.0.0.1 + false + false + + + bar + webkid + + http://localhost:3000/bar/ + Bar! + + 2014-09-26T20:11:04Z + + info@webkid.io + webkid + false + webkid + + 127.0.0.1 + false + false + + + + + Hey, wow!

// Amazing post.

Check example.

]]> + + 2014-11-01T15:06:24Z + false + false + + moriz@webkid.io + Moritz + false + moritz + + 127.0.0.1 + + + + + + Hi Moritz,

thanks for reading!

]]> + + 2014-11-01T16:59:29Z + false + false + + christopher@webkid.io + Christopher + false + christopher + + 127.0.0.1 + + + + + + + Hi Christopher!

Thanks!

]]> + + 2014-11-13T09:03:26Z + false + false + + moriz@webkid.io + Moritz + false + moritz + + 127.0.0.1 + + + + + + + Bar is the best.

]]> + + 2014-11-01T15:06:24Z + false + false + + moriz@webkid.io + Moritz + false + moritz + + 127.0.0.1 + + + + + + Nice! :)

]]> + + 2014-11-13T10:40:03Z + false + false + + christopher@webkid.io + Christopher + false + christopher + + 127.0.0.1 + + + +