Skip to content

Commit

Permalink
move queryAuthors() to page object; fix its types
Browse files Browse the repository at this point in the history
Also added a test.
  • Loading branch information
siddharthvp committed May 16, 2021
1 parent ec1f782 commit 15b7b18
Show file tree
Hide file tree
Showing 7 changed files with 128 additions and 100 deletions.
3 changes: 3 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -1,5 +1,8 @@
Only breaking changes, deprecations and the like are documented in this change log.

#### 0.11.0
- mwn#queryAuthors() now requires `getSiteInfo()` to have run first. Also, it is deprecated in favour of using the `queryAuthors()` method on a page object.

#### 0.10.0

- `loginGetToken()` is now deprecated in favour of `login()` which will now fetch tokens as well.
Expand Down
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "mwn",
"version": "0.10.4",
"version": "0.11.0",
"description": "JavaScript & TypeScript MediaWiki bot framework for Node.js",
"main": "./build/bot.js",
"types": "./build/bot.d.ts",
Expand Down
76 changes: 2 additions & 74 deletions src/bot.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1917,84 +1917,12 @@ export class mwn {
* This API has a throttling of 2000 requests a day.
* Supported for EN, DE, ES, EU, TR Wikipedias only
* @see https://api.wikiwho.net/
* @deprecated Use queryAuthors on the page object directly instead
*/
async queryAuthors(
title: string,
): Promise<{ totalBytes: number; users: { id: number; name: string; bytes: number; percent: number }[] }> {
let langcodematch = this.options.apiUrl.match(/([^/]*?)\.wikipedia\.org/);
if (!langcodematch || !langcodematch[1]) {
throw new Error('WikiWho API is not supported for bot API URL. Re-check.');
}

let json;
try {
json = await this.rawRequest({
url: `https://api.wikiwho.net/${
langcodematch[1]
}/api/v1.0.0-beta/latest_rev_content/${encodeURIComponent(title)}/?editor=true`,
}).then((response) => response.data);
} catch (err) {
throw new Error(err && err.response && err.response.data && err.response.data.Error);
}

const tokens = Object.values(json.revisions[0])[0].tokens;

let data = {
totalBytes: 0,
users: [],
};
let userdata: {
[editor: string]: {
name?: number;
bytes: number;
percent?: number;
};
} = {};

for (let token of tokens) {
data.totalBytes += token.str.length;
let editor = token['editor'];
if (!userdata[editor]) {
userdata[editor] = { bytes: 0 };
}
userdata[editor].bytes += token.str.length;
if (editor.startsWith('0|')) {
// IP
userdata[editor].name = editor.slice(2);
}
}

Object.entries(userdata).map(([userid, { bytes }]) => {
userdata[userid].percent = bytes / data.totalBytes;
if (userdata[userid].percent < 0.02) {
delete userdata[userid];
}
});

await this.request({
action: 'query',
list: 'users',
ususerids: Object.keys(userdata).filter((us) => !us.startsWith('0|')), // don't lookup IPs
}).then((json) => {
json.query.users.forEach((us: any) => {
userdata[String(us.userid)].name = us.name;
});
});

data.users = Object.entries(userdata)
.map(([userid, { bytes, name, percent }]) => {
return {
id: userid,
name: name,
bytes: bytes,
percent: percent,
};
})
.sort((a, b) => {
return a.bytes < b.bytes ? 1 : -1;
});

return data;
return new this.page(title).queryAuthors();
}

/**
Expand Down
120 changes: 104 additions & 16 deletions src/page.ts
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,16 @@ export interface PageViewData {
views: number;
}

export interface AuthorshipData {
totalBytes: number;
users: Array<{
id: number;
name: string;
bytes: number;
percent: number;
}>;
}

export interface ApiPage {
pageid: number;
ns: number;
Expand Down Expand Up @@ -153,6 +163,7 @@ export interface MwnPage extends MwnTitle {
customOptions?: ApiQueryLogEventsParams,
): AsyncGenerator<LogEvent>;
pageViews(options?: PageViewOptions): Promise<PageViewData[]>;
queryAuthors(): Promise<AuthorshipData>;
edit(transform: (rev: { content: string; timestamp: string }) => string | ApiEditPageParams): Promise<any>;
save(text: string, summary?: string, options?: ApiEditPageParams): Promise<any>;
newSection(header: string, message: string, additionalParams?: ApiEditPageParams): Promise<any>;
Expand Down Expand Up @@ -600,22 +611,9 @@ export default function (bot: mwn): MwnPageStatic {

return bot
.rawRequest({
url:
'https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article' +
'/' +
project +
'/' +
access +
'/' +
agent +
'/' +
encodeURIComponent(this.toString()) +
'/' +
granularity +
'/' +
startString +
'/' +
endString,
url: `https://wikimedia.org/api/rest_v1/metrics/pageviews/per-article/${project}/${access}/${agent}/${encodeURIComponent(
this.toString(),
)}/${granularity}/${startString}/${endString}`,
headers: {
'User-Agent': bot.options.userAgent,
},
Expand All @@ -625,6 +623,96 @@ export default function (bot: mwn): MwnPageStatic {
});
}

/**
* Query the top contributors to the article using the WikiWho API.
* This API has a throttling of 2000 requests a day.
* Supported for EN, DE, ES, EU, TR Wikipedias only
* @see https://api.wikiwho.net/
*/
async queryAuthors(): Promise<AuthorshipData> {
let langcodematch = bot.options.apiUrl.match(/([^/]*?)\.wikipedia\.org/);
if (!langcodematch || !langcodematch[1]) {
throw new Error('WikiWho API is not supported for bot API URL. Re-check.');
}

let json;
try {
json = await bot
.rawRequest({
url: `https://api.wikiwho.net/${
langcodematch[1]
}/api/v1.0.0-beta/latest_rev_content/${encodeURIComponent(this.toString())}/?editor=true`,
headers: {
'User-Agent': bot.options.userAgent,
},
})
.then((response) => response.data);
} catch (err) {
throw new Error(err && err.response && err.response.data && err.response.data.Error);
}

const tokens = Object.values(json.revisions[0])[0].tokens;

let data: AuthorshipData = {
totalBytes: 0,
users: [],
};
let userdata: {
[editor: string]: {
name?: string;
bytes: number;
percent?: number;
};
} = {};

for (let token of tokens) {
data.totalBytes += token.str.length;
let editor = token['editor'];
if (!userdata[editor]) {
userdata[editor] = { bytes: 0 };
}
userdata[editor].bytes += token.str.length;
if (editor.startsWith('0|')) {
// IP
userdata[editor].name = editor.slice(2);
}
}

Object.entries(userdata).map(([userid, { bytes }]) => {
userdata[userid].percent = bytes / data.totalBytes;
if (userdata[userid].percent < 0.02) {
delete userdata[userid];
}
});

await bot
.request({
action: 'query',
list: 'users',
ususerids: Object.keys(userdata).filter((us) => !us.startsWith('0|')), // don't lookup IPs
})
.then((json) => {
json.query.users.forEach((us: any) => {
userdata[us.userid].name = us.name;
});
});

data.users = Object.entries(userdata)
.map(([userid, { bytes, name, percent }]) => {
return {
id: Number(userid),
name: name,
bytes: bytes,
percent: percent,
};
})
.sort((a, b) => {
return a.bytes < b.bytes ? 1 : -1;
});

return data;
}

/**** Post operations *****/
// Defined in bot.js

Expand Down
10 changes: 3 additions & 7 deletions src/static_utils.ts
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ export function link(target: string | MwnTitle, displaytext?: string): string {
*/
export function template(title: string | MwnTitle, parameters: { [parameter: string]: string } = {}): string {
if (typeof title !== 'string') {
// title object provided
if (title.namespace === 10) {
title = title.getMainText(); // skip namespace name for templates
} else if (title.namespace === 0) {
Expand All @@ -41,13 +42,8 @@ export function template(title: string | MwnTitle, parameters: { [parameter: str
'{{' +
title +
Object.entries(parameters)
.map(([key, val]) => {
if (!val) {
// skip parameter if no value provided
return '';
}
return '|' + key + '=' + val;
})
.filter(([, value]) => !!value) // ignore params with no value
.map(([name, value]) => `|${name}=${value}`)
.join('') +
'}}'
);
Expand Down
13 changes: 13 additions & 0 deletions tests/suppl.bot.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -102,4 +102,17 @@ describe('supplementary functions', function () {
expect(views[1]).to.be.an('object').with.property('timestamp').that.equals('2021020100');
});

it('wikiwho', async function () {
this.timeout(10000);
await bot.getSiteInfo();
let page = new bot.page('Dairy in India');
let data = await page.queryAuthors();
expect(data).to.be.an('object').with.property('totalBytes').that.is.a('number');
expect(data).to.have.property('users').that.is.instanceOf(Array).of.length.greaterThan(1);
expect(data.users[0]).to.be.an('object').with.property('id').that.is.a('number');
expect(data.users[0]).to.have.property('name').that.is.a('string');
expect(data.users[0]).to.have.property('percent').that.is.a('number');
expect(data.users[0]).to.have.property('bytes').that.is.a('number');
});

});

0 comments on commit 15b7b18

Please sign in to comment.