diff --git a/modules/middleware/fetchPackage.js b/modules/middleware/fetchPackage.js index 3bea9c2..a021f25 100644 --- a/modules/middleware/fetchPackage.js +++ b/modules/middleware/fetchPackage.js @@ -3,7 +3,7 @@ import semver from 'semver'; import addLeadingSlash from '../utils/addLeadingSlash'; import createPackageURL from '../utils/createPackageURL'; import createSearch from '../utils/createSearch'; -import getNpmPackageInfo from '../utils/getNpmPackageInfo'; +import { getPackageInfo as getNpmPackageInfo } from '../utils/npm'; function tagRedirect(req, res) { const version = req.packageInfo['dist-tags'][req.packageVersion]; diff --git a/modules/middleware/findFile.js b/modules/middleware/findFile.js index a632017..95871dd 100644 --- a/modules/middleware/findFile.js +++ b/modules/middleware/findFile.js @@ -3,7 +3,7 @@ import path from 'path'; import addLeadingSlash from '../utils/addLeadingSlash'; import createPackageURL from '../utils/createPackageURL'; import createSearch from '../utils/createSearch'; -import fetchNpmPackage from '../utils/fetchNpmPackage'; +import { fetchPackage as fetchNpmPackage } from '../utils/npm'; import getIntegrity from '../utils/getIntegrity'; import getContentType from '../utils/getContentType'; diff --git a/modules/utils/fetchNpmPackage.js b/modules/utils/fetchNpmPackage.js deleted file mode 100644 index 2a1913a..0000000 --- a/modules/utils/fetchNpmPackage.js +++ /dev/null @@ -1,43 +0,0 @@ -import url from 'url'; -import https from 'https'; -import gunzip from 'gunzip-maybe'; -import tar from 'tar-stream'; - -import debug from './debug'; -import bufferStream from './bufferStream'; -import agent from './registryAgent'; - -export default function fetchNpmPackage(packageConfig) { - return new Promise((resolve, reject) => { - const tarballURL = packageConfig.dist.tarball; - - debug('Fetching package for %s from %s', packageConfig.name, tarballURL); - - const { hostname, pathname } = url.parse(tarballURL); - const options = { - agent: agent, - hostname: hostname, - path: pathname - }; - - https - .get(options, res => { - if (res.statusCode === 200) { - resolve(res.pipe(gunzip()).pipe(tar.extract())); - } else { - bufferStream(res).then(data => { - const spec = `${packageConfig.name}@${packageConfig.version}`; - const content = data.toString('utf-8'); - const error = new Error( - `Failed to fetch tarball for ${spec}\nstatus: ${ - res.statusCode - }\ndata: ${content}` - ); - - reject(error); - }); - } - }) - .on('error', reject); - }); -} diff --git a/modules/utils/fetchNpmPackageInfo.js b/modules/utils/fetchNpmPackageInfo.js deleted file mode 100644 index 4288932..0000000 --- a/modules/utils/fetchNpmPackageInfo.js +++ /dev/null @@ -1,57 +0,0 @@ -import url from 'url'; -import https from 'https'; - -import debug from './debug'; -import bufferStream from './bufferStream'; -import agent from './registryAgent'; - -const npmRegistryURL = - process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org'; - -function parseJSON(res) { - return bufferStream(res).then(JSON.parse); -} - -export default function fetchNpmPackageInfo(packageName) { - return new Promise((resolve, reject) => { - const encodedPackageName = - packageName.charAt(0) === '@' - ? `@${encodeURIComponent(packageName.substring(1))}` - : encodeURIComponent(packageName); - - const infoURL = `${npmRegistryURL}/${encodedPackageName}`; - - debug('Fetching package info for %s from %s', packageName, infoURL); - - const { hostname, pathname } = url.parse(infoURL); - const options = { - agent: agent, - hostname: hostname, - path: pathname, - headers: { - Accept: 'application/json' - } - }; - - https - .get(options, res => { - if (res.statusCode === 200) { - resolve(parseJSON(res)); - } else if (res.statusCode === 404) { - resolve(null); - } else { - bufferStream(res).then(data => { - const content = data.toString('utf-8'); - const error = new Error( - `Failed to fetch info for ${packageName}\nstatus: ${ - res.statusCode - }\ndata: ${content}` - ); - - reject(error); - }); - } - }) - .on('error', reject); - }); -} diff --git a/modules/utils/getNpmPackageInfo.js b/modules/utils/getNpmPackageInfo.js deleted file mode 100644 index 487782b..0000000 --- a/modules/utils/getNpmPackageInfo.js +++ /dev/null @@ -1,43 +0,0 @@ -import LRUCache from 'lru-cache'; - -import fetchNpmPackageInfo from './fetchNpmPackageInfo'; - -const maxMegabytes = 40; // Cap the cache at 40 MB -const maxLength = maxMegabytes * 1024 * 1024; -const oneSecond = 1000; -const oneMinute = 60 * oneSecond; - -const cache = new LRUCache({ - max: maxLength, - maxAge: oneMinute, - length: Buffer.byteLength -}); - -const notFound = ''; - -export default function getNpmPackageInfo(packageName) { - return new Promise((resolve, reject) => { - const key = `npmPackageInfo-${packageName}`; - const value = cache.get(key); - - if (value != null) { - resolve(value === notFound ? null : JSON.parse(value)); - } else { - fetchNpmPackageInfo(packageName).then(info => { - if (info == null) { - // Cache 404s for 5 minutes. This prevents us from making - // unnecessary requests to the registry for bad package names. - // In the worst case, a brand new package's info will be - // available within 5 minutes. - cache.set(key, notFound, oneMinute * 5); - resolve(null); - } else { - // Cache valid package info for 1 minute. In the worst case, - // new versions won't be available for 1 minute. - cache.set(key, JSON.stringify(info), oneMinute); - resolve(info); - } - }, reject); - } - }); -} diff --git a/modules/utils/npm.js b/modules/utils/npm.js new file mode 100644 index 0000000..9d59bc0 --- /dev/null +++ b/modules/utils/npm.js @@ -0,0 +1,129 @@ +import url from 'url'; +import https from 'https'; +import gunzip from 'gunzip-maybe'; +import tar from 'tar-stream'; +import LRUCache from 'lru-cache'; + +import debug from './debug'; +import bufferStream from './bufferStream'; + +const npmRegistryURL = + process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org'; + +const agent = new https.Agent({ + keepAlive: true +}); + +function parseJSON(res) { + return bufferStream(res).then(JSON.parse); +} + +export function fetchPackageInfo(packageName) { + return new Promise((accept, reject) => { + const encodedPackageName = + packageName.charAt(0) === '@' + ? `@${encodeURIComponent(packageName.substring(1))}` + : encodeURIComponent(packageName); + + const infoURL = `${npmRegistryURL}/${encodedPackageName}`; + + debug('Fetching package info for %s from %s', packageName, infoURL); + + const { hostname, pathname } = url.parse(infoURL); + const options = { + agent: agent, + hostname: hostname, + path: pathname, + headers: { + Accept: 'application/json' + } + }; + + https + .get(options, async res => { + if (res.statusCode === 200) { + accept(parseJSON(res)); + } else if (res.statusCode === 404) { + accept(null); + } else { + const data = await bufferStream(res); + const content = data.toString('utf-8'); + const error = new Error( + `Failed to fetch info for ${packageName}\nstatus: ${res.statusCode}\ndata: ${content}` + ); + + reject(error); + } + }) + .on('error', reject); + }); +} + +export function fetchPackage(packageConfig) { + return new Promise((accept, reject) => { + const tarballURL = packageConfig.dist.tarball; + + debug('Fetching package for %s from %s', packageConfig.name, tarballURL); + + const { hostname, pathname } = url.parse(tarballURL); + const options = { + agent: agent, + hostname: hostname, + path: pathname + }; + + https + .get(options, async res => { + if (res.statusCode === 200) { + accept(res.pipe(gunzip()).pipe(tar.extract())); + } else { + const data = await bufferStream(res); + const spec = `${packageConfig.name}@${packageConfig.version}`; + const content = data.toString('utf-8'); + const error = new Error( + `Failed to fetch tarball for ${spec}\nstatus: ${res.statusCode}\ndata: ${content}` + ); + + reject(error); + } + }) + .on('error', reject); + }); +} + +const oneMegabyte = 1024 * 1024; +const oneSecond = 1000; +const oneMinute = oneSecond * 60; + +const cache = new LRUCache({ + max: oneMegabyte * 40, + length: Buffer.byteLength, + maxAge: oneSecond +}); + +const notFound = ''; + +export async function getPackageInfo(packageName) { + const key = `npmPackageInfo-${packageName}`; + const value = cache.get(key); + + if (value != null) { + return value === notFound ? null : JSON.parse(value); + } + + const info = await fetchPackageInfo(packageName); + + if (info == null) { + // Cache 404s for 5 minutes. This prevents us from making + // unnecessary requests to the registry for bad package names. + // In the worst case, a brand new package's info will be + // available within 5 minutes. + cache.set(key, notFound, oneMinute * 5); + return null; + } + + // Cache valid package info for 1 minute. In the worst case, + // new versions won't be available for 1 minute. + cache.set(key, JSON.stringify(info), oneMinute); + return info; +} diff --git a/modules/utils/registryAgent.js b/modules/utils/registryAgent.js deleted file mode 100644 index efe37b4..0000000 --- a/modules/utils/registryAgent.js +++ /dev/null @@ -1,7 +0,0 @@ -import https from 'https'; - -const agent = new https.Agent({ - keepAlive: true -}); - -export default agent;