improve/expand documentation

README.md

@@ -6,7 +6,7 @@
 [![Coverage Status](https://coveralls.io/repos/github/siddharthvp/mwn/badge.svg?branch=master)](https://coveralls.io/github/siddharthvp/mwn?branch=master)
 [![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg?style=flat-square)](http://makeapullrequest.com)
 
-**Quick links: [Getting Started](#user-content-getting-started) — [Docs](https://mwn.toolforge.org/) — [GitHub](https://github.com/siddharthvp/mwn) — [NPM](https://www.npmjs.com/package/mwn) — [API Documentation](https://mwn.toolforge.org/docs/api/classes/mwn.html)**
+**Quick links: [Getting Started](https://mwn.toolforge.org/docs/setting-up-mwn) — [GitHub](https://github.com/siddharthvp/mwn) — [NPM](https://www.npmjs.com/package/mwn) — [User Documentation](https://mwn.toolforge.org/) — [API Documentation](https://mwn.toolforge.org/docs/api/classes/mwn.html)**
 
 **Mwn** is a modern and comprehensive MediaWiki bot framework for Node.js, originally adapted from [mwbot](https://github.com/Fannon/mwbot).
 
@@ -155,6 +155,8 @@ Exclusion compliance is _not_ enabled by default.
 
 ### Getting started
 
+**Note: More complete [documentation is on Toolforge](https://mwn.toolforge.org/).**
+
 Importing mwn:
 
 In JavaScript:

src/bot.ts

@@ -30,7 +30,7 @@
  *
  */
 
-// Node internal module
+// Node internal modules
 import * as fs from 'fs';
 import * as path from 'path';
 import * as crypto from 'crypto';
@@ -87,6 +87,10 @@ import {
 } from './api_response_types';
 
 export { MwnDate, MwnTitle, MwnPage, MwnFile, MwnCategory, MwnWikitext, MwnUser, MwnStream, ApiPage, ApiRevision };
+// Export, if only for the sake of getting generated documentation
+export * from './api_response_types';
+export type { PageViewData, PageViewOptions, AuthorshipData } from './page';
+export type { TemplateConfig, Template, MwnWikitextStatic } from './wikitext';
 
 export interface MwnOptions {
 	silent?: boolean;

src/wikitext.ts

@@ -74,19 +74,20 @@ export interface TemplateConfig {
 // by Evad37 (cc-by-sa-3.0/GFDL)
 // TODO: expand from evad37/xfdcloser
 /**
- * Represents the wikitext of template transclusion. Used by #parseTemplates.
- * @prop {string} name Name of the template
- * @prop {string} wikitext Full wikitext of the transclusion
- * @prop {Object[]} parameters Parameters used in the translcusion, in order, of form:
-		{
-			name: {string|number} parameter name, or position for unnamed parameters,
-			value: {string} Wikitext passed to the parameter (whitespace trimmed),
-			wikitext: {string} Full wikitext (including leading pipe, parameter name/equals sign (if applicable), value, and any whitespace)
-		}
+ * Represents the wikitext of template transclusion. Used by {@link parseTemplates}.
  */
 export class Template {
+	/**
+	 * Full wikitext of the transclusion
+	 */
 	wikitext: string;
+	/**
+	 * Parameters used in the transclusion
+	 */
 	parameters: Array<Parameter>;
+	/**
+	 * Name of the template
+	 */
 	name: string | number;
 
 	/**
@@ -115,9 +116,21 @@ export class Template {
 	}
 }
 
+/**
+ * Represents a template parameter
+ */
 export class Parameter {
+	/**
+	 * parameter name, or position for unnamed parameters
+	 */
 	name: string | number;
+	/**
+	 * Wikitext passed to the parameter (whitespace trimmed)
+	 */
 	value: string;
+	/**
+	 * Full wikitext (including leading pipe, parameter name/equals sign (if applicable), value, and any whitespace)
+	 */
 	wikitext: string;
 
 	constructor(name: string | number, val: string, wikitext: string) {

website/docs/10-bulk-processing.md

@@ -0,0 +1,66 @@
+# Bulk processing
+
+### continuedQuery / continuedQueryGen
+See [Handling query continuation](/docs/handling-query-continuation) for more details.
+
+continuedQuery returns a promised resolved with the array of all individual API response.
+
+Use of `continuedQueryGen` is recommended since continuedQuery will fetch the results of all the API calls before it begins to do anything with the results. `continuedQueryGen` gets the result of each API call and processes them one at a time.
+
+### massQuery / massQueryGen
+MediaWiki sets a limit of 500 (50 for non-bots) on the number of pages that can be queried in a single API call. To query more than that, `massQuery` or `massQueryGen` can be used. This splits the page list into batches of 500 and sends individual queries and returns a promise resolved with the array of all individual API call responses.
+
+Example: get the protection status of a large number of pages:
+
+```js
+bot.massQuery({
+	action: 'query',
+	format: 'json',
+	prop: 'info',
+	titles: ['Page1', 'Page2', 'Page1300'], // array of page names
+	inprop: 'protection'
+}) // 2nd parameter is taken as 'titles' by default
+	.then((jsons) => {
+		// jsons is the array of individual JSON responses.
+	});
+```
+
+Any errors in the individual API calls will not cause the entire massQuery to fail, but the data at the array index corresponding to that API call will be error object.
+
+massQueryGen is the generator equivalent that yields each API response as when they're received.
+
+### Batch operations
+
+Perform asynchronous tasks (involving API usage) over a number of pages (or other arbitrary items). `batchOperation` uses a default concurrency of 5. Customise this according to how expensive the API operation is. Higher concurrency limits could lead to more frequent API errors.
+
+-   `batchOperation(pageList, workerFunction, concurrency, maxRetries)`: The `workerFunction` must return a promise.
+
+```js
+bot.batchOperation(
+	pageList,
+	(page, idx) => {
+		// do something with each page
+		// the index of the page in pageList is available as the 2nd argument
+		// return a promise in the end
+	},
+	/* concurrency */ 5,
+	/* retries */ 2
+);
+```
+
+-   `bot.seriesBatchOperation(pageList, workerFunction, sleepDuration, retries)` can be used for serial operations, with a sleep duration between each task (default 5 seconds).
+
+```js
+bot.seriesBatchOperation(
+	pageList,
+	(page, idx) => {
+		// do something with each page
+		// the index of the page in pageList is available as the 2nd argument
+		// return a promise in the end
+	},
+	5000,
+	2
+); // set the sleep duration in milliseconds as the third parameter, max number of retries for each action is set as the 4th parameter
+```
+
+Note that `seriesBatchOperation` with delay=0 is same as `batchOperation` with concurrency=1.

website/docs/11-integration-with-other-apis.md

@@ -0,0 +1,53 @@
+# Integration with other APIs
+
+Apart from the [MediaWiki API](https://www.mediawiki.org/wiki/API:Main_page), Mwn integrates with a few other APIs:
+
+### ORES
+See https://ores.wikimedia.org/ for details.
+Get ORES scores for revisions:
+```js
+await bot.oresQueryRevisions(
+	'https://ores.wikimedia.org/',		// ORES endpoint URL
+	['articlequality', 'drafttopic'],	// ORES modes       
+	['76923582', '2387429']				// Revision IDs
+);
+```
+
+
+### EventStreams
+See https://wikitech.wikimedia.org/wiki/Event_Platform/EventStreams
+```js
+const stream = bot.stream(['recentchange']);
+stream.addListener(
+	// Consider event only if this function returns true
+	function eventFilter(data) {
+		return data.wiki === 'enwiki';
+	},
+
+	// Run this function for every filtered event
+	async function worker(data) {
+		// do something with data
+	}
+);
+```
+
+### PageViews
+See https://wikitech.wikimedia.org/wiki/Analytics/AQS/Pageviews
+```js
+const page = new bot.page('Deaths in 2020');
+const pageViewData = await page.pageViews({
+	// See https://mwn.toolforge.org/docs/api/interfaces/pageviewoptions.html for available options
+});
+```
+The [PageViewOptions](https://mwn.toolforge.org/docs/api/interfaces/pageviewoptions.html) argument is optional. Return type is Promise<<a href="https://mwn.toolforge.org/docs/api/interfaces/pageviewdata.html">PageViewData</a>[]>.
+
+### WikiWho
+See https://wikiwho.wmflabs.org/
+
+Fetch the list of top contributors to an article. Available for limited number of Wikipedias.
+
+```js
+const page = new bot.page('Lorem ipsum');
+const contributorData = await page.queryAuthors();
+```
+Return type is Promise<<a href="https://mwn.toolforge.org/docs/api/interfaces/authorshipdata.html)">AuthorshipData</a>>.

website/docs/12-logging.md

@@ -0,0 +1,14 @@
+# Logging
+
+Mwn provides a convenient coloured logging utility, based on [semlog](https://npmjs.com/package/semlog).
+
+```js
+const log = mwn.log;
+
+log('[I] Informational message');
+log('[S] Success message');
+log('[W] Warning message');
+log('[E] Error message');
+```
+
+Based on the character within `[]`, colouration happens automatically. 

website/docs/13-dates.md

@@ -0,0 +1,45 @@
+# Dates
+
+Mwn provides a rich wrapper around the native [Date](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date) interface.
+
+The constructor support the common MediaWiki datetime formats:
+```js
+const date1 = new bot.date('13:45, 2 April 2020 (UTC)'); // This won't parse with JS native Date!
+const date2 = new bot.date('20210304134567'); // MW database timestamp format.
+```
+
+in addition to everything that native JS Date supports:
+```js
+const date1 = new bot.date();
+const date2 = new bot.date('3 December 2020');
+```
+
+Note that it inherits the weirdities of JS Date - even "NY 12345" gets parsed as valid date, so per [MDN recommendation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/Date#timestamp_string) you should not parse dates as strings of unknown format.
+
+All methods on native date are inherited, for instance:
+```js
+date.getUTCDate();
+date.toISOString();
+```
+
+
+But in addition, you can get names of the months and days of the week (in English):
+```js
+date.getMonthName();
+date.getUTCMonthName();
+date.getDayName();
+date.getUTCDayName(); 
+```
+
+Add and subtract dates. This mutates the original date as well as returns the mutated object to allow chaining. The supported units are seconds, minutes, hours, days, weeks, months and years.
+```js
+date.add(1, 'hour');
+date.add(4, 'hours');
+date.subtract(5, 'hours').subtract(30, 'minutes');
+```
+
+Check if a date is before or after another date (which can be either an Mwn date or a normal Date object).
+```js
+date.isBefore(new Date()); // boolean
+date.isAfter(new bot.date());
+```

website/docs/6-emergency-shutoff.md

@@ -4,19 +4,27 @@ Mwn exploits Node's asynchronous event loop to efficiently implement emergency s
 
 ```js
 bot.enableEmergencyShutoff({
-	page: 'User:ExampleBot/shutoff', // The name of the page to check
-	intervalDuration: 5000, // check shutoff page every 5 seconds
+	// The name of the page to check
+	page: 'User:ExampleBot/shutoff',
+
+	// check shutoff page every 5 seconds
+	intervalDuration: 5000,
+
+	// function to determine whether the bot should continue to run or not
 	condition: function (pagetext) {
-		// function to determine whether the bot should continue to run or not
+		// Example implementation: if some one changes the text to something
+		// other than "running", let's decide to stop!
 		if (pagetext !== 'running') {
-			// Example implementation: if some one changes the text to something
-			return false; // other than "running", let's decide to stop!
+			return false; 
 		} else return true;
 	},
+
+	// function to trigger when shutoff is activated
 	onShutoff: function (pagetext) {
-		// function to trigger when shutoff is activated
-		process.exit(); // let's just exit, though we could also terminate
-	} // any open connections, close files, etc.
+		// let's just exit, though we could also terminate
+		// any open connections, close files, etc.
+		process.exit(); 
+	}
 });
 ```