Add fetch* utils
This commit is contained in:
Michael Jackson
parent
ba14f9197f
commit
26ba4698e3
|
|
@ -52,8 +52,42 @@ function semverRedirect(req, res) {
|
|||
* version if the request targets a tag or uses a semver version.
|
||||
*/
|
||||
function fetchPackage(req, res, next) {
|
||||
getPackageInfo(req.packageName, (error, packageInfo) => {
|
||||
if (error) {
|
||||
getPackageInfo(req.packageName).then(
|
||||
packageInfo => {
|
||||
if (packageInfo == null || packageInfo.versions == null) {
|
||||
return res
|
||||
.status(404)
|
||||
.type("text")
|
||||
.send(`Cannot find package "${req.packageName}"`);
|
||||
}
|
||||
|
||||
req.packageInfo = packageInfo;
|
||||
|
||||
if (req.packageVersion in req.packageInfo.versions) {
|
||||
// A valid request for a package we haven't downloaded yet.
|
||||
req.packageConfig = req.packageInfo.versions[req.packageVersion];
|
||||
|
||||
getPackage(req.packageConfig).then(
|
||||
outputDir => {
|
||||
req.packageDir = outputDir;
|
||||
next();
|
||||
},
|
||||
error => {
|
||||
console.error(error);
|
||||
|
||||
res
|
||||
.status(500)
|
||||
.type("text")
|
||||
.send(`Cannot fetch package ${req.packageSpec}`);
|
||||
}
|
||||
);
|
||||
} else if (req.packageVersion in req.packageInfo["dist-tags"]) {
|
||||
tagRedirect(req, res);
|
||||
} else {
|
||||
semverRedirect(req, res);
|
||||
}
|
||||
},
|
||||
error => {
|
||||
console.error(error);
|
||||
|
||||
return res
|
||||
|
|
@ -61,39 +95,7 @@ function fetchPackage(req, res, next) {
|
|||
.type("text")
|
||||
.send(`Cannot get info for package "${req.packageName}"`);
|
||||
}
|
||||
|
||||
if (packageInfo == null || packageInfo.versions == null) {
|
||||
return res
|
||||
.status(404)
|
||||
.type("text")
|
||||
.send(`Cannot find package "${req.packageName}"`);
|
||||
}
|
||||
|
||||
req.packageInfo = packageInfo;
|
||||
|
||||
if (req.packageVersion in req.packageInfo.versions) {
|
||||
// A valid request for a package we haven't downloaded yet.
|
||||
req.packageConfig = req.packageInfo.versions[req.packageVersion];
|
||||
|
||||
getPackage(req.packageConfig, (error, outputDir) => {
|
||||
if (error) {
|
||||
console.error(error);
|
||||
|
||||
res
|
||||
.status(500)
|
||||
.type("text")
|
||||
.send(`Cannot fetch package ${req.packageSpec}`);
|
||||
} else {
|
||||
req.packageDir = outputDir;
|
||||
next();
|
||||
}
|
||||
});
|
||||
} else if (req.packageVersion in req.packageInfo["dist-tags"]) {
|
||||
tagRedirect(req, res);
|
||||
} else {
|
||||
semverRedirect(req, res);
|
||||
}
|
||||
});
|
||||
);
|
||||
}
|
||||
|
||||
module.exports = fetchPackage;
|
||||
|
|
|
|||
|
|
@ -1,7 +1,20 @@
|
|||
function createMutex(doWork) {
|
||||
const invariant = require("invariant");
|
||||
|
||||
function defaultCreateKey(payload) {
|
||||
return payload;
|
||||
}
|
||||
|
||||
function createMutex(doWork, createKey = defaultCreateKey) {
|
||||
const mutex = Object.create(null);
|
||||
|
||||
return (key, payload, callback) => {
|
||||
return (payload, callback) => {
|
||||
const key = createKey(payload);
|
||||
|
||||
invariant(
|
||||
typeof key === "string",
|
||||
"Mutex needs a string key; please provide a createKey function that returns a string"
|
||||
);
|
||||
|
||||
if (mutex[key]) {
|
||||
mutex[key].push(callback);
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,9 @@
|
|||
const path = require("path");
|
||||
const tmpdir = require("os-tmpdir");
|
||||
|
||||
function createTempPath(name, version) {
|
||||
const hyphenName = name.replace(/\//g, "-");
|
||||
return path.join(tmpdir(), `unpkg-${hyphenName}-${version}`);
|
||||
}
|
||||
|
||||
module.exports = createTempPath;
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
require("isomorphic-fetch");
|
||||
const fs = require("fs");
|
||||
const mkdirp = require("mkdirp");
|
||||
const gunzip = require("gunzip-maybe");
|
||||
const tar = require("tar-fs");
|
||||
|
||||
const createTempPath = require("./createTempPath");
|
||||
|
||||
function stripNamePrefix(headers) {
|
||||
// Most packages have header names that look like "package/index.js"
|
||||
// so we shorten that to just "index.js" here. A few packages use a
|
||||
// prefix other than "package/". e.g. the firebase package uses the
|
||||
// "firebase_npm/" prefix. So we just strip the first dir name.
|
||||
headers.name = headers.name.replace(/^[^/]+\//, "");
|
||||
return headers;
|
||||
}
|
||||
|
||||
function ignoreLinks(file, headers) {
|
||||
return headers.type === "link" || headers.type === "symlink";
|
||||
}
|
||||
|
||||
function extractResponse(response, outputDir) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const extract = tar.extract(outputDir, {
|
||||
readable: true, // All dirs/files should be readable.
|
||||
map: stripNamePrefix,
|
||||
ignore: ignoreLinks
|
||||
});
|
||||
|
||||
response.body
|
||||
.pipe(gunzip())
|
||||
.pipe(extract)
|
||||
.on("finish", resolve)
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function fetchPackage(packageConfig) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const tarballURL = packageConfig.dist.tarball;
|
||||
const outputDir = createTempPath(packageConfig.name, packageConfig.version);
|
||||
|
||||
console.log(`info: Fetching ${tarballURL} and extracting to ${outputDir}`);
|
||||
|
||||
fs.access(outputDir, error => {
|
||||
if (error) {
|
||||
if (error.code === "ENOENT" || error.code === "ENOTDIR") {
|
||||
// ENOENT or ENOTDIR are to be expected when we haven't yet
|
||||
// fetched a package for the first time. Carry on!
|
||||
mkdirp(outputDir, error => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve(
|
||||
fetch(tarballURL)
|
||||
.then(res => extractResponse(res, outputDir))
|
||||
.then(() => outputDir)
|
||||
);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
reject(error);
|
||||
}
|
||||
} else {
|
||||
// Best case: we already have this package cached on disk!
|
||||
resolve(outputDir);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = fetchPackage;
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
require("isomorphic-fetch");
|
||||
|
||||
const config = require("../config");
|
||||
|
||||
function fetchPackageInfo(packageName) {
|
||||
return new Promise(resolve => {
|
||||
let encodedPackageName;
|
||||
if (packageName.charAt(0) === "@") {
|
||||
encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`;
|
||||
} else {
|
||||
encodedPackageName = encodeURIComponent(packageName);
|
||||
}
|
||||
|
||||
const url = `${config.registryURL}/${encodedPackageName}`;
|
||||
|
||||
console.log(`info: Fetching package info from ${url}`);
|
||||
|
||||
resolve(
|
||||
fetch(url, {
|
||||
headers: {
|
||||
Accept: "application/json"
|
||||
}
|
||||
}).then(res => (res.status === 404 ? null : res.json()))
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = fetchPackageInfo;
|
||||
|
|
@ -1,88 +1,26 @@
|
|||
require("isomorphic-fetch");
|
||||
const fs = require("fs");
|
||||
const path = require("path");
|
||||
const tmpdir = require("os-tmpdir");
|
||||
const gunzip = require("gunzip-maybe");
|
||||
const mkdirp = require("mkdirp");
|
||||
const tar = require("tar-fs");
|
||||
|
||||
const createMutex = require("./createMutex");
|
||||
const fetchPackage = require("./fetchPackage");
|
||||
|
||||
function createTempPath(name, version) {
|
||||
const normalName = name.replace(/\//g, "-");
|
||||
return path.join(tmpdir(), `unpkg-${normalName}-${version}`);
|
||||
}
|
||||
|
||||
function stripNamePrefix(headers) {
|
||||
// Most packages have header names that look like "package/index.js"
|
||||
// so we shorten that to just "index.js" here. A few packages use a
|
||||
// prefix other than "package/". e.g. the firebase package uses the
|
||||
// "firebase_npm/" prefix. So we just strip the first dir name.
|
||||
headers.name = headers.name.replace(/^[^/]+\//, "");
|
||||
return headers;
|
||||
}
|
||||
|
||||
function ignoreLinks(file, headers) {
|
||||
return headers.type === "link" || headers.type === "symlink";
|
||||
}
|
||||
|
||||
function extractResponse(response, outputDir) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const extract = tar.extract(outputDir, {
|
||||
readable: true, // All dirs/files should be readable.
|
||||
map: stripNamePrefix,
|
||||
ignore: ignoreLinks
|
||||
});
|
||||
|
||||
response.body
|
||||
.pipe(gunzip())
|
||||
.pipe(extract)
|
||||
.on("finish", resolve)
|
||||
.on("error", reject);
|
||||
});
|
||||
}
|
||||
|
||||
function fetchAndExtract(tarballURL, outputDir) {
|
||||
console.log(`info: Fetching ${tarballURL} and extracting to ${outputDir}`);
|
||||
|
||||
return fetch(tarballURL).then(response => {
|
||||
return extractResponse(response, outputDir);
|
||||
});
|
||||
}
|
||||
|
||||
const fetchMutex = createMutex((payload, callback) => {
|
||||
const { tarballURL, outputDir } = payload;
|
||||
|
||||
fs.access(outputDir, error => {
|
||||
if (error) {
|
||||
if (error.code === "ENOENT" || error.code === "ENOTDIR") {
|
||||
// ENOENT or ENOTDIR are to be expected when we haven't yet
|
||||
// fetched a package for the first time. Carry on!
|
||||
mkdirp(outputDir, error => {
|
||||
if (error) {
|
||||
callback(error);
|
||||
} else {
|
||||
fetchAndExtract(tarballURL, outputDir).then(() => {
|
||||
callback();
|
||||
}, callback);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
callback(error);
|
||||
}
|
||||
} else {
|
||||
// Best case: we already have this package cached on disk!
|
||||
callback();
|
||||
const fetchMutex = createMutex((packageConfig, callback) => {
|
||||
fetchPackage(packageConfig).then(
|
||||
outputDir => {
|
||||
callback(null, outputDir);
|
||||
},
|
||||
error => {
|
||||
callback(error);
|
||||
}
|
||||
});
|
||||
});
|
||||
);
|
||||
}, packageConfig => packageConfig.dist.tarball);
|
||||
|
||||
function getPackage(packageConfig, callback) {
|
||||
const tarballURL = packageConfig.dist.tarball;
|
||||
const outputDir = createTempPath(packageConfig.name, packageConfig.version);
|
||||
|
||||
fetchMutex(tarballURL, { tarballURL, outputDir }, error => {
|
||||
callback(error, outputDir);
|
||||
function getPackage(packageConfig) {
|
||||
return new Promise((resolve, reject) => {
|
||||
fetchMutex(packageConfig, (error, value) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve(value);
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,34 +1,9 @@
|
|||
require("isomorphic-fetch");
|
||||
|
||||
const config = require("../config");
|
||||
|
||||
const createCache = require("./createCache");
|
||||
const createMutex = require("./createMutex");
|
||||
const fetchPackageInfo = require("./fetchPackageInfo");
|
||||
|
||||
const packageInfoCache = createCache("packageInfo");
|
||||
|
||||
function fetchPackageInfo(packageName) {
|
||||
console.log(`info: Fetching package info for ${packageName}`);
|
||||
|
||||
let encodedPackageName;
|
||||
if (packageName.charAt(0) === "@") {
|
||||
encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`;
|
||||
} else {
|
||||
encodedPackageName = encodeURIComponent(packageName);
|
||||
}
|
||||
|
||||
const url = `${config.registryURL}/${encodedPackageName}`;
|
||||
|
||||
return fetch(url, {
|
||||
headers: {
|
||||
Accept: "application/json"
|
||||
}
|
||||
}).then(res => {
|
||||
return res.status === 404 ? null : res.json();
|
||||
});
|
||||
}
|
||||
|
||||
const PackageNotFound = "PackageNotFound";
|
||||
const packageNotFound = "PackageNotFound";
|
||||
|
||||
// This mutex prevents multiple concurrent requests to
|
||||
// the registry for the same package info.
|
||||
|
|
@ -40,7 +15,7 @@ const fetchMutex = createMutex((packageName, callback) => {
|
|||
// unnecessary requests to the registry for bad package names.
|
||||
// In the worst case, a brand new package's info will be
|
||||
// available within 5 minutes.
|
||||
packageInfoCache.set(packageName, PackageNotFound, 300, () => {
|
||||
packageInfoCache.set(packageName, packageNotFound, 300, () => {
|
||||
callback(null, value);
|
||||
});
|
||||
} else {
|
||||
|
|
@ -59,13 +34,23 @@ const fetchMutex = createMutex((packageName, callback) => {
|
|||
);
|
||||
});
|
||||
|
||||
function getPackageInfo(packageName, callback) {
|
||||
packageInfoCache.get(packageName, (error, value) => {
|
||||
if (error || value != null) {
|
||||
callback(error, value === PackageNotFound ? null : value);
|
||||
} else {
|
||||
fetchMutex(packageName, packageName, callback);
|
||||
}
|
||||
function getPackageInfo(packageName) {
|
||||
return new Promise((resolve, reject) => {
|
||||
packageInfoCache.get(packageName, (error, value) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else if (value != null) {
|
||||
resolve(value === packageNotFound ? null : value);
|
||||
} else {
|
||||
fetchMutex(packageName, (error, value) => {
|
||||
if (error) {
|
||||
reject(error);
|
||||
} else {
|
||||
resolve(value);
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue