Skip to content

Commit

Permalink
Adds whether there are types for a package to an index
Browse files Browse the repository at this point in the history
  • Loading branch information
orta committed Jun 14, 2019
1 parent cb207be commit 456188d
Show file tree
Hide file tree
Showing 7 changed files with 149 additions and 75 deletions.
148 changes: 74 additions & 74 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -39,94 +39,94 @@ For every single NPM package, we create a record in the Algolia index. The resul

```json5
{
"name": "babel-core",
"concatenatedName": "babelcore",
"downloadsLast30Days": 10978749,
"downloadsRatio": 0.08310651682685861,
"humanDownloadsLast30Days": "11m",
"jsDelivrHits": 11684192,
"popular": true,
"version": "6.26.0",
"versions": {
name: 'babel-core',
concatenatedName: 'babelcore',
downloadsLast30Days: 10978749,
downloadsRatio: 0.08310651682685861,
humanDownloadsLast30Days: '11m',
jsDelivrHits: 11684192,
popular: true,
version: '6.26.0',
versions: {
// [...]
"7.0.0-beta.3": "2017-10-15T13:12:35.166Z"
'7.0.0-beta.3': '2017-10-15T13:12:35.166Z',
},
"tags": {
"latest": "6.26.0",
"old": "5.8.38",
"next": "7.0.0-beta.3"
tags: {
latest: '6.26.0',
old: '5.8.38',
next: '7.0.0-beta.3',
},
"description": "Babel compiler core.",
"dependencies": {
"babel-code-frame": "^6.26.0"
description: 'Babel compiler core.',
dependencies: {
'babel-code-frame': '^6.26.0',
// [...]
},
"devDependencies": {
"babel-helper-fixtures": "^6.26.0"
devDependencies: {
'babel-helper-fixtures': '^6.26.0',
// [...]
},
"repository": {
"url": "https://github.com/babel/babel/tree/master/packages/babel-core",
"host": "github.com",
"user": "babel",
"project": "babel",
"path": "/tree/master/packages/babel-core",
"branch": "master"
repository: {
url: 'https://github.com/babel/babel/tree/master/packages/babel-core',
host: 'github.com',
user: 'babel',
project: 'babel',
path: '/tree/master/packages/babel-core',
branch: 'master',
},
"readme":
"# babel-core\n\n> Babel compiler core.\n\n\n [... truncated at 200kb]",
"owner": {
readme: '# babel-core\n\n> Babel compiler core.\n\n\n [... truncated at 200kb]',
owner: {
// either GitHub owner or npm owner
"name": "babel",
"avatar": "https://github.com/babel.png",
"link": "https://github.com/babel"
name: 'babel',
avatar: 'https://github.com/babel.png',
link: 'https://github.com/babel',
},
"deprecated": false,
"badPackage": false,
"homepage": "https://babeljs.io/",
"license": "MIT",
"keywords": [
"6to5",
"babel",
"classes",
"const",
"es6",
"harmony",
"let",
"modules",
"transpile",
"transpiler",
"var",
"babel-core",
"compiler"
deprecated: false,
badPackage: false,
homepage: 'https://babeljs.io/',
license: 'MIT',
keywords: [
'6to5',
'babel',
'classes',
'const',
'es6',
'harmony',
'let',
'modules',
'transpile',
'transpiler',
'var',
'babel-core',
'compiler',
],
"created": 1424009748555,
"modified": 1508833762239,
"lastPublisher": {
"name": "hzoo",
"email": "[email protected]",
"avatar": "https://gravatar.com/avatar/851fb4fa7ca479bce1ae0cdf80d6e042",
"link": "https://www.npmjs.com/~hzoo"
created: 1424009748555,
modified: 1508833762239,
lastPublisher: {
name: 'hzoo',
email: '[email protected]',
avatar: 'https://gravatar.com/avatar/851fb4fa7ca479bce1ae0cdf80d6e042',
link: 'https://www.npmjs.com/~hzoo',
},
"owners": [
owners: [
{
"email": "[email protected]",
"name": "thejameskyle",
"avatar": "https://gravatar.com/avatar/8a00efb48d632ae449794c094f7d5c38",
"link": "https://www.npmjs.com/~thejameskyle"
}
email: '[email protected]',
name: 'thejameskyle',
avatar: 'https://gravatar.com/avatar/8a00efb48d632ae449794c094f7d5c38',
link: 'https://www.npmjs.com/~thejameskyle',
},
// [...]
],
"lastCrawl": "2017-10-24T08:29:24.672Z",
"dependents": 3321,
"humanDependents": "3.3k",
"changelogFilename": null, // if babel-core had a changelog, it would be the raw GitHub url here
"objectID": "babel-core",
"_searchInternal": {
"popularName": "babel-core",
"downloadsMagnitude": 8,
"jsDelivrPopularity": 5
}
lastCrawl: '2017-10-24T08:29:24.672Z',
dependents: 3321,
ts: undefined,
humanDependents: '3.3k',
changelogFilename: null, // if babel-core had a changelog, it would be the raw GitHub url here
objectID: 'babel-core',
_searchInternal: {
popularName: 'babel-core',
downloadsMagnitude: 8,
jsDelivrPopularity: 5,
},
}
```

Expand Down Expand Up @@ -182,7 +182,7 @@ apiKey=... yarn start

### Restart

To restart from a particular point (or from the begining):
To restart from a particular point (or from the beginning):

```sh
seq=0 apiKey=... yarn start
Expand Down
2 changes: 2 additions & 0 deletions src/__tests__/__snapshots__/config.test.js.snap
Original file line number Diff line number Diff line change
Expand Up @@ -179,9 +179,11 @@ Object {
"maxObjSize": 450000,
"npmDownloadsEndpoint": "https://api.npmjs.org/downloads",
"npmRegistryEndpoint": "https://replicate.npmjs.com/registry",
"npmRootEndpoint": "https://api.npmjs.org/",
"popularDownloadsRatio": 0.005,
"replicateConcurrency": 10,
"seq": null,
"timeToRedoBootstrap": 604800000,
"unpkgRoot": "https://unpkg.com/",
}
`;
2 changes: 2 additions & 0 deletions src/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,9 @@ import ms from 'ms';
const defaultConfig = {
npmRegistryEndpoint: 'https://replicate.npmjs.com/registry',
npmDownloadsEndpoint: 'https://api.npmjs.org/downloads',
npmRootEndpoint: 'https://api.npmjs.org/',
jsDelivrHitsEndpoint: 'https://data.jsdelivr.com/v1/stats/packages/month/all',
unpkgRoot: 'https://unpkg.com/',
maxObjSize: 450000,
popularDownloadsRatio: 0.005,
appId: 'OFCNCOG2CU',
Expand Down
7 changes: 7 additions & 0 deletions src/npm.js
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ const logWarning = ({ error, type, packagesStr }) => {
);
};

export function validatePackageExists(pkgName) {
return got(`${c.npmRootEndpoint}/${pkgName}`, {
json: true,
method: 'HEAD',
}).then(response => response.statusCode === 200);
}

export async function getDownloads(pkgs) {
// npm has a weird API to get downloads via GET params, so we split pkgs into chunks
// and do multiple requests to avoid weird cases when concurrency is high
Expand Down
5 changes: 4 additions & 1 deletion src/saveDocs.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import log from './log.js';
import { getDownloads, getDependents } from './npm.js';
import { getChangelogs } from './changelog.js';
import { getHits } from './jsDelivr';
import { getTSSupport } from './typescriptSupport';

export default function saveDocs({ docs, index }) {
const rawPkgs = docs
Expand All @@ -26,13 +27,15 @@ function addMetaData(pkgs) {
getDependents(pkgs),
getChangelogs(pkgs),
getHits(pkgs),
]).then(([downloads, dependents, changelogs, hits]) =>
getTSSupport(pkgs),
]).then(([downloads, dependents, changelogs, hits, ts]) =>
pkgs.map((pkg, index) => ({
...pkg,
...downloads[index],
...dependents[index],
...changelogs[index],
...hits[index],
...ts[index],
_searchInternal: {
...pkg._searchInternal,
...downloads[index]._searchInternal,
Expand Down
46 changes: 46 additions & 0 deletions src/typescriptSupport.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
// @ts-check

import { validatePackageExists } from './npm.js';
import { fileExistsInUnpkg } from './unpkg.js';

/**
* Basically either
* - { ts: undefined } for no existing TypeScript support
* - { ts: "@types/module" } - for definitely typed support
* - { ts: "included" } - for types shipped with the module
* */
export async function getTypeScriptSupportString(pkg) {
// The cheap and simple (+ recommended by TS) way
// of adding a types section to your package.json
if (pkg.types) {
return { ts: 'included' };
}

// Older, but still works way of defining your types
if (pkg.typings) {
return { ts: 'included' };
}

// The 2nd most likely is definitely typed
const defTypeName = `@types/${pkg.name}`;
const defTyped = await validatePackageExists(defTypeName);
if (defTyped) {
return { ts: defTypeName };
}

// Check if main's JS file can be resolved to a d.ts file instead
const main = pkg.main || 'index.js';
if (main.endsWith('.js')) {
const dtsMain = main.replace(/js$/, 'd.ts');
const resolved = await fileExistsInUnpkg(pkg.name, pkg.version, dtsMain);
if (resolved) {
return { ts: 'included' };
}
}

return { ts: undefined };
}

export function getTSSupport(pkgs) {
return Promise.all(pkgs.map(getTypeScriptSupportString));
}
14 changes: 14 additions & 0 deletions src/unpkg.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// @ts-check
import c from './config.js';
import got from 'got';

// make a head request to a route like:
// https://unpkg.com/[email protected]/_LazyWrapper.js
// to validate the existence of a particular file
export function fileExistsInUnpkg(pkg, version, path) {
const uri = `${c.unpkgRoot}/${pkg}@${version}/${path}`;
return got(uri, {
json: true,
method: 'HEAD',
}).then(response => response.statusCode === 200);
}

0 comments on commit 456188d

Please sign in to comment.