diff --git a/server/middleware/fetchPackage.js b/server/middleware/fetchPackage.js index 9f75c54..bc56638 100644 --- a/server/middleware/fetchPackage.js +++ b/server/middleware/fetchPackage.js @@ -3,7 +3,7 @@ const semver = require("semver"); const addLeadingSlash = require("../utils/addLeadingSlash"); const createPackageURL = require("../utils/createPackageURL"); const createSearch = require("../utils/createSearch"); -const getPackageInfo = require("../utils/getPackageInfo"); +const getNpmPackageInfo = require("../utils/getNpmPackageInfo"); const incrementCounter = require("../utils/incrementCounter"); function tagRedirect(req, res) { @@ -113,7 +113,7 @@ function filenameRedirect(req, res) { * exact filename if the request omits the filename. */ function fetchPackage(req, res, next) { - getPackageInfo(req.packageName).then( + getNpmPackageInfo(req.packageName).then( packageInfo => { if (packageInfo == null || packageInfo.versions == null) { return res diff --git a/server/middleware/findFile.js b/server/middleware/findFile.js index fa79be4..9221eba 100644 --- a/server/middleware/findFile.js +++ b/server/middleware/findFile.js @@ -3,7 +3,7 @@ const path = require("path"); const addLeadingSlash = require("../utils/addLeadingSlash"); const createPackageURL = require("../utils/createPackageURL"); const createSearch = require("../utils/createSearch"); -const fetchArchive = require("../utils/fetchArchive"); +const fetchNpmPackage = require("../utils/fetchNpmPackage"); const getIntegrity = require("../utils/getIntegrity"); const getContentType = require("../utils/getContentType"); @@ -92,9 +92,7 @@ function searchEntries(tarballStream, entryName, wantsHTML) { const chunks = []; - stream.on("data", chunk => chunks.push(chunk)); - - stream.on("end", () => { + stream.on("data", chunk => chunks.push(chunk)).on("end", () => { const content = Buffer.concat(chunks); // Set some extra properties for files that we will @@ -124,7 +122,7 @@ const trailingSlash = /\/$/; * Redirect to the "index" file if a directory was requested. */ function findFile(req, res, next) { - fetchArchive(req.packageConfig).then(tarballStream => { + fetchNpmPackage(req.packageConfig).then(tarballStream => { const entryName = req.filename .replace(trailingSlash, "") .replace(leadingSlash, ""); diff --git a/server/utils/bufferStream.js b/server/utils/bufferStream.js new file mode 100644 index 0000000..8a709fd --- /dev/null +++ b/server/utils/bufferStream.js @@ -0,0 +1,12 @@ +function bufferStream(stream) { + return new Promise((resolve, reject) => { + const chunks = []; + + stream + .on("error", reject) + .on("data", chunk => chunks.push(chunk)) + .on("end", () => resolve(Buffer.concat(chunks))); + }); +} + +module.exports = bufferStream; diff --git a/server/utils/fetchArchive.js b/server/utils/fetchNpmPackage.js similarity index 52% rename from server/utils/fetchArchive.js rename to server/utils/fetchNpmPackage.js index ab02bcb..d485d20 100644 --- a/server/utils/fetchArchive.js +++ b/server/utils/fetchNpmPackage.js @@ -3,17 +3,21 @@ const https = require("https"); const gunzip = require("gunzip-maybe"); const tar = require("tar-stream"); -const agent = new https.Agent({ - keepAlive: true -}); +const agent = require("./registryAgent"); -function fetchArchive(packageConfig) { +function fetchNpmPackage(packageConfig) { return new Promise((resolve, reject) => { - const tarballURL = url.parse(packageConfig.dist.tarball); + const tarballURL = packageConfig.dist.tarball; + + console.log( + `info: Fetching package for ${packageConfig.name} from ${tarballURL}` + ); + + const { hostname, pathname } = url.parse(tarballURL); const options = { - hostname: tarballURL.hostname, - path: tarballURL.pathname, - agent: agent + agent: agent, + hostname: hostname, + path: pathname }; https @@ -28,4 +32,4 @@ function fetchArchive(packageConfig) { }); } -module.exports = fetchArchive; +module.exports = fetchNpmPackage; diff --git a/server/utils/fetchNpmPackageInfo.js b/server/utils/fetchNpmPackageInfo.js new file mode 100644 index 0000000..e2ff3ec --- /dev/null +++ b/server/utils/fetchNpmPackageInfo.js @@ -0,0 +1,47 @@ +const url = require("url"); +const https = require("https"); + +const config = require("../config"); +const bufferStream = require("./bufferStream"); +const agent = require("./registryAgent"); + +function parseJSON(res) { + return bufferStream(res).then(JSON.parse); +} + +function fetchNpmPackageInfo(packageName) { + return new Promise((resolve, reject) => { + const encodedPackageName = + packageName.charAt(0) === "@" + ? `@${encodeURIComponent(packageName.substring(1))}` + : encodeURIComponent(packageName); + + const infoURL = `${config.registryURL}/${encodedPackageName}`; + + console.log( + `info: Fetching package info for ${packageName} from ${infoURL}` + ); + + const { hostname, pathname } = url.parse(infoURL); + const options = { + agent: agent, + hostname: hostname, + path: pathname, + headers: { + Accept: "application/json" + } + }; + + https + .get(options, res => { + if (res.statusCode === 200) { + resolve(parseJSON(res)); + } else { + reject(res); + } + }) + .on("error", reject); + }); +} + +module.exports = fetchNpmPackageInfo; diff --git a/server/utils/fetchPackageInfo.js b/server/utils/fetchPackageInfo.js deleted file mode 100644 index 8914653..0000000 --- a/server/utils/fetchPackageInfo.js +++ /dev/null @@ -1,24 +0,0 @@ -const fetch = require("isomorphic-fetch"); - -const config = require("../config"); - -function fetchPackageInfo(packageName) { - let encodedPackageName; - if (packageName.charAt(0) === "@") { - encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`; - } else { - encodedPackageName = encodeURIComponent(packageName); - } - - const url = `${config.registryURL}/${encodedPackageName}`; - - console.log(`info: Fetching package info from ${url}`); - - return fetch(url, { - headers: { - Accept: "application/json" - } - }).then(res => (res.status === 404 ? null : res.json())); -} - -module.exports = fetchPackageInfo; diff --git a/server/utils/getPackageInfo.js b/server/utils/getNpmPackageInfo.js similarity index 82% rename from server/utils/getPackageInfo.js rename to server/utils/getNpmPackageInfo.js index c4ca4a8..825041a 100644 --- a/server/utils/getPackageInfo.js +++ b/server/utils/getNpmPackageInfo.js @@ -1,18 +1,18 @@ const createCache = require("./createCache"); const createMutex = require("./createMutex"); -const fetchPackageInfo = require("./fetchPackageInfo"); +const fetchNpmPackageInfo = require("./fetchNpmPackageInfo"); const cache = createCache("packageInfo"); const notFound = "PackageNotFound"; -const fetchMutex = createMutex((packageName, callback) => { +const mutex = createMutex((packageName, callback) => { cache.get(packageName, (error, value) => { if (error) { callback(error); } else if (value != null) { callback(null, value === notFound ? null : value); } else { - fetchPackageInfo(packageName).then( + fetchNpmPackageInfo(packageName).then( value => { if (value == null) { // Cache 404s for 5 minutes. This prevents us from making @@ -40,9 +40,9 @@ const fetchMutex = createMutex((packageName, callback) => { }); }); -function getPackageInfo(packageName) { +function getNpmPackageInfo(packageName) { return new Promise((resolve, reject) => { - fetchMutex(packageName, (error, value) => { + mutex(packageName, (error, value) => { if (error) { reject(error); } else { @@ -52,4 +52,4 @@ function getPackageInfo(packageName) { }); } -module.exports = getPackageInfo; +module.exports = getNpmPackageInfo; diff --git a/server/utils/registryAgent.js b/server/utils/registryAgent.js new file mode 100644 index 0000000..779dc52 --- /dev/null +++ b/server/utils/registryAgent.js @@ -0,0 +1,7 @@ +const https = require("https"); + +const agent = new https.Agent({ + keepAlive: true +}); + +module.exports = agent;