-
Notifications
You must be signed in to change notification settings - Fork 186
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Script to automatically update tlds.js
- Loading branch information
Nick Frasser
committed
Nov 24, 2021
1 parent
1ae5f69
commit 6ba92e4
Showing
4 changed files
with
84 additions
and
25 deletions.
There are no files selected for viewing
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,65 @@ | ||
| const http = require('https'); // or 'https' for https:// URLs | ||
| const fs = require('fs'); | ||
| const punycode = require('punycode/'); | ||
|
|
||
| const tldsListUrl = 'https://data.iana.org/TLD/tlds-alpha-by-domain.txt'; | ||
| const tldsjs = 'packages/linkifyjs/src/core/tlds.js'; | ||
| let tldsListContents = ''; | ||
|
|
||
| http.get(tldsListUrl, (response) => { | ||
| console.log(`Downloading ${tldsListUrl}...`); | ||
| response.on('data', (chunk) => { tldsListContents += chunk; }); | ||
| response.on('end', () => { | ||
| console.log(`Downloaded. Re-generating ${tldsjs}...`); | ||
|
|
||
| // NOTE: punycode versions of IDNs (e.g., `XN--...`) do not get included | ||
| // in the TLDs list because these will not be as commonly used without | ||
| // the http prefix anyway and linkify will already force-encode those. | ||
| let tlds = []; | ||
| let utlds = []; | ||
|
|
||
| // NOTE: vermögensberater vermögensberatung are special cases because | ||
| // they're the only ones that contain a mix of ASCII and non-ASCII | ||
| // characters. | ||
| const specialTlds = ['XN--VERMGENSBERATER-CTB', 'XN--VERMGENSBERATUNG-PWB']; | ||
| const specialUtlds = specialTlds.map(tld => punycode.toUnicode(tld.toLowerCase())); | ||
|
|
||
| for (const line of tldsListContents.split('\n').map(line => line.trim())) { | ||
| if (!line || line[0] === '#' || specialTlds.includes(line)) { continue; } | ||
| if (/^XN--/.test(line)) { | ||
| utlds.push(punycode.toUnicode(line.toLowerCase())); | ||
| } else { | ||
| tlds.push(line.toLowerCase()); | ||
| } | ||
| } | ||
| tlds = tlds.concat(specialUtlds).sort(); | ||
| utlds = utlds.sort(); | ||
|
|
||
| const jsFile = fs.openSync(tldsjs, 'w'); | ||
| fs.writeSync(jsFile, '// THIS FILE IS AUTOMATICALLY GENERATED DO NOT EDIT DIRECTLY\n'); | ||
| fs.writeSync(jsFile, `// ${tldsListUrl}\n`); | ||
|
|
||
| // Write TLDs | ||
| fs.writeSync(jsFile, 'export const tlds = \''); | ||
| let firstWrite = false; | ||
| for (const tld of tlds) { | ||
| if (firstWrite) { fs.writeSync(jsFile, ' \\\n'); } | ||
| else { firstWrite = true; } | ||
| fs.writeSync(jsFile, tld); | ||
| } | ||
| fs.writeSync(jsFile, '\'.split(\' \');\n'); | ||
| fs.writeSync(jsFile, '// Internationalized domain names containing non-ASCII\n'); | ||
| fs.writeSync(jsFile, 'export const utlds = \''); | ||
|
|
||
| firstWrite = false; | ||
| for (const utld of utlds) { | ||
| if (firstWrite) { fs.writeSync(jsFile, ' \\\n'); } | ||
| else { firstWrite = true; } | ||
| fs.writeSync(jsFile, utld); | ||
| } | ||
| fs.writeSync(jsFile, '\'.split(\' \');\n'); | ||
| fs.closeSync(jsFile); | ||
|
|
||
| console.log('Done'); | ||
| }); | ||
| }); |