Skip to content

Commit

Permalink
feat(importer): add disqus importer
Browse files Browse the repository at this point in the history
  • Loading branch information
@g-div
g-div committed Dec 7, 2017
1 parent 8ec85e0 commit 5f04ae7
Show file tree
Hide file tree
Showing 4 changed files with 239 additions and 1 deletion.
8 changes: 8 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,14 @@ In order to be able to edit your config file and your SQL database files, you ma
docker run -p 3000:3000 -v $(pwd):/usr/src/app -d gka/schnack
```

### Import comments from disqus

You can import comments from your [disqus XML export](https://help.disqus.com/customer/portal/articles/472149-comments-export) as following:

```
npm run import -- disqus.xml
```

### Who is behind Schnack?

Schnack is [yet another](https://github.com/gka/canvid/) happy collaboration between [Webkid](https://webkid.io/) and [Gregor Aisch](https://www.vis4.net).
Expand Down
4 changes: 3 additions & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,8 @@
"rollup-plugin-node-resolve": "^3.0.0",
"rollup-plugin-string": "^2.0.2",
"rollup-plugin-template": "^1.0.10",
"rollup-plugin-uglify": "^2.0.1"
"rollup-plugin-uglify": "^2.0.1",
"to-markdown": "^3.1.0",
"xml2js": "^0.4.19"
}
}
95 changes: 95 additions & 0 deletions src/importer.js
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
const fs = require('fs');
const path = require('path');
const sqlite = require('sqlite');
const queries = require('./db/queries');
const xml2js = require('xml2js');
const toMarkdown = require('to-markdown');
const dbPromise = sqlite.open('./comments.db', { Promise });

const filename = process.argv.slice(2, 3).pop();
if (!filename) {
console.error('Pass the filepath to your XML file as argument');
process.exit(1);
}

function run() {
const parser = new xml2js.Parser({explicitArray: false});
const xmlFilePath = path.resolve(__dirname, '..', filename);
fs.readFile(xmlFilePath, (err, data) =>
parser.parseString(data, async function (err, result) {
if (err) console.error(`Error parsing the XML file '${xmlFilePath}':`, err);
const db = await dbPromise;

// for each post save it and store the id
// for each post that is a reply replace reply_to with the stored id
const threads = result.disqus.thread;
const posts = result.disqus.post.map(comment => importComment(threads, comment));

for (post of posts) {
const newComment = await saveComment(post);
post.new_id = newComment;
}

for (post of posts) {
const replies = posts.filter(p => p.comment[3] === post.id); // replies to current post
if (replies) {
for (reply of replies) {
const { id, new_id } = post;
const res = await db.run(`UPDATE comment SET reply_to = ? WHERE reply_to = ?`, [new_id, id]);
}
}
}
})
);
};

async function saveComment(post) {
const db = await dbPromise;
const { comment, author } = post;

if (!author.username) {
author.username = 'Anonymous Guest';
}
const user = [
'disqus',
author.username,
author.name,
author.username,
0
];

try {
await db.run(queries.create_user, user);
const newUser = await db.get(queries.find_user, ['disqus', author.username]);
if (newUser.id) comment.unshift(newUser.id); // push user_id to the front
const res = await db.run(`INSERT INTO comment
(user_id, slug, comment, reply_to, created_at, approved, rejected)
VALUES (?,?,?,?,?,?,0)`, comment);
return res.lastID;
} catch (err) {
console.error(`Error saving the comment for the slug ${comment[0]}:`, err);
}
};

function importComment(threads, comment) {
const { author } = comment;
const thread = threads.filter(thread => thread.$['dsq:id'] === comment.thread.$['dsq:id'])[0].id
const reply_to = comment.parent ? comment.parent.$['dsq:id'] : null;
const message = toMarkdown(comment.message.trim());
const timestamp = comment.createdAt;
const approved = (comment.isDeleted === "true" || comment.isSpam === "true") ? 0 : 1;

return {
comment: [
thread,
message,
reply_to,
timestamp,
approved
],
id: comment.$['dsq:id'],
author
};
};

run();
133 changes: 133 additions & 0 deletions test/disqus.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
<?xml version="1.0" encoding="utf-8"?>
<disqus xmlns="http://disqus.com" xmlns:dsq="http://disqus.com/disqus-internals" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://disqus.com/api/schemas/1.0/disqus.xsd http://disqus.com/api/schemas/1.0/disqus-internals.xsd">
<category dsq:id="3286472">
<forum>webkid</forum>
<title>General</title>
<isDefault>true</isDefault>
</category>
<thread dsq:id="3057176120">
<id>foo</id>
<forum>webkid</forum>
<category dsq:id="3286472" />
<link>http://localhost:3000/foo/</link>
<title>Welcome</title>
<message />
<createdAt>2014-09-26T20:11:04Z</createdAt>
<author>
<email>[email protected]</email>
<name>webkid</name>
<isAnonymous>false</isAnonymous>
<username>webkid</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<isClosed>false</isClosed>
<isDeleted>false</isDeleted>
</thread>
<thread dsq:id="3057176121">
<id>bar</id>
<forum>webkid</forum>
<category dsq:id="3286472" />
<link>http://localhost:3000/bar/</link>
<title>Bar!</title>
<message />
<createdAt>2014-09-26T20:11:04Z</createdAt>
<author>
<email>[email protected]</email>
<name>webkid</name>
<isAnonymous>false</isAnonymous>
<username>webkid</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<isClosed>false</isClosed>
<isDeleted>false</isDeleted>
</thread>
<post dsq:id="1664694813">
<id />
<message>
<![CDATA[<p>Hey, wow!</p><p>// Amazing post.</p><p>Check <a href="http://example.example" rel="nofollow noopener" title="example">example</a>.</p>]]>
</message>
<createdAt>2014-11-01T15:06:24Z</createdAt>
<isDeleted>false</isDeleted>
<isSpam>false</isSpam>
<author>
<email>[email protected]</email>
<name>Moritz</name>
<isAnonymous>false</isAnonymous>
<username>moritz</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<thread dsq:id="3057176120" />
</post>
<post dsq:id="1664831530">
<id />
<message>
<![CDATA[<p>Hi Moritz,</p><p>thanks for reading! </p>]]>
</message>
<createdAt>2014-11-01T16:59:29Z</createdAt>
<isDeleted>false</isDeleted>
<isSpam>false</isSpam>
<author>
<email>[email protected]</email>
<name>Christopher</name>
<isAnonymous>false</isAnonymous>
<username>christopher</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<thread dsq:id="3057176120" />
<parent dsq:id="1664694813" />
</post>
<post dsq:id="1690922142">
<id />
<message>
<![CDATA[<p>Hi Christopher!</p><p>Thanks! </p>]]>
</message>
<createdAt>2014-11-13T09:03:26Z</createdAt>
<isDeleted>false</isDeleted>
<isSpam>false</isSpam>
<author>
<email>[email protected]</email>
<name>Moritz</name>
<isAnonymous>false</isAnonymous>
<username>moritz</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<thread dsq:id="3057176120" />
<parent dsq:id="1664831530" />
</post>
<post dsq:id="1664694824">
<id />
<message>
<![CDATA[<p>Bar is the best.</p>]]>
</message>
<createdAt>2014-11-01T15:06:24Z</createdAt>
<isDeleted>false</isDeleted>
<isSpam>false</isSpam>
<author>
<email>[email protected]</email>
<name>Moritz</name>
<isAnonymous>false</isAnonymous>
<username>moritz</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<thread dsq:id="3057176121" />
</post>
<post dsq:id="1690992879">
<id />
<message>
<![CDATA[<p>Nice! :)</p>]]>
</message>
<createdAt>2014-11-13T10:40:03Z</createdAt>
<isDeleted>false</isDeleted>
<isSpam>false</isSpam>
<author>
<email>[email protected]</email>
<name>Christopher</name>
<isAnonymous>false</isAnonymous>
<username>christopher</username>
</author>
<ipAddress>127.0.0.1</ipAddress>
<thread dsq:id="3057176120" />
<parent dsq:id="1690922142" />
</post>
</disqus>

0 comments on commit 5f04ae7

Please sign in to comment.