186526/unpkg
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Don't cache packages on the filesystem

Browse Source 
Should help with transient errors reported in #86, #104, and #110
This commit is contained in:
Michael Jackson
2018-07-04 16:30:11 -07:00
parent 700bb109a1
commit 5969ecc6ef
23 changed files with 570 additions and 702 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
server/middleware/fetchPackage.js
View File

@ -1,10 +1,13 @@
const semver = require("semver");
const addLeadingSlash = require("../utils/addLeadingSlash");
const createPackageURL = require("../utils/createPackageURL");
const getPackageInfo = require("../utils/getPackageInfo");
const getPackage = require("../utils/getPackage");
const incrementCounter = require("../utils/incrementCounter");
function tagRedirect(req, res) {
const version = req.packageInfo["dist-tags"][req.packageVersion];
// Cache tag redirects for 1 minute.
res
.set({
@ -13,12 +16,7 @@ function tagRedirect(req, res) {
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageInfo["dist-tags"][req.packageVersion],
req.filename,
req.search
)
createPackageURL(req.packageName, version, req.filename, req.search)
);
}
@ -47,9 +45,71 @@ function semverRedirect(req, res) {
}
}
function filenameRedirect(req, res) {
let filename;
if (req.query.module != null) {
// See https://github.com/rollup/rollup/wiki/pkg.module
filename =
req.packageConfig.module ||
req.packageConfig["jsnext:main"] ||
"/index.js";
} else if (
req.query.main &&
req.packageConfig[req.query.main] &&
typeof req.packageConfig[req.query.main] === "string"
) {
// Deprecated, see #63
filename = req.packageConfig[req.query.main];
// Count which packages are using this so we can warn them when we
// remove this functionality.
incrementCounter(
"package-json-custom-main",
req.packageSpec + "?main=" + req.query.main,
1
);
} else if (
req.packageConfig.unpkg &&
typeof req.packageConfig.unpkg === "string"
) {
filename = req.packageConfig.unpkg;
} else if (
req.packageConfig.browser &&
typeof req.packageConfig.browser === "string"
) {
// Deprecated, see #63
filename = req.packageConfig.browser;
// Count which packages are using this so we can warn them when we
// remove this functionality.
incrementCounter("package-json-browser-fallback", req.packageSpec, 1);
} else {
filename = req.packageConfig.main || "/index.js";
}
// Redirect to the exact filename so relative imports
// and URLs resolve correctly.
// TODO: increase the max-age?
res
.set({
"Cache-Control": "public,max-age=60",
"Cache-Tag": "redirect,filename-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageVersion,
addLeadingSlash(filename),
createSearch(req.query)
)
);
}
/**
* Fetch the package metadata and tarball from npm. Redirect to the exact
* version if the request targets a tag or uses a semver version.
* version if the request targets a tag or uses a semver version, or to the
* exact filename if the request omits the filename.
*/
function fetchPackage(req, res, next) {
getPackageInfo(req.packageName).then(
@ -62,30 +122,21 @@ function fetchPackage(req, res, next) {
}
req.packageInfo = packageInfo;
req.packageConfig = req.packageInfo.versions[req.packageVersion];
if (req.packageVersion in req.packageInfo.versions) {
// A valid request for a package we haven't downloaded yet.
req.packageConfig = req.packageInfo.versions[req.packageVersion];
getPackage(req.packageConfig).then(
outputDir => {
req.packageDir = outputDir;
next();
},
error => {
console.error(error);
res
.status(500)
.type("text")
.send(`Cannot fetch package ${req.packageSpec}`);
}
);
} else if (req.packageVersion in req.packageInfo["dist-tags"]) {
tagRedirect(req, res);
} else {
semverRedirect(req, res);
if (!req.packageConfig) {
if (req.packageVersion in req.packageInfo["dist-tags"]) {
return tagRedirect(req, res);
} else {
return semverRedirect(req, res);
}
}
if (!req.filename) {
return filenameRedirect(req, res);
}
next();
},
error => {
console.error(error);

302
server/middleware/findFile.js
View File

@ -1,166 +1,178 @@
const fs = require("fs");
const path = require("path");
const fetch = require("isomorphic-fetch");
const gunzip = require("gunzip-maybe");
const tar = require("tar-stream");
const addLeadingSlash = require("../utils/addLeadingSlash");
const createPackageURL = require("../utils/createPackageURL");
const createSearch = require("../utils/createSearch");
const incrementCounter = require("../utils/incrementCounter");
const fetchArchive = require("../utils/fetchArchive");
const getIntegrity = require("../utils/getIntegrity");
const getContentType = require("../utils/getContentType");
/**
* File extensions to look for when automatically resolving.
*/
const resolveExtensions = ["", ".js", ".json"];
function indexRedirect(req, res, entry) {
// Redirect to the index file so relative imports
// resolve correctly.
// TODO: increase the max-age?
res
.set({
"Cache-Control": "public,max-age=60",
"Cache-Tag": "redirect,index-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageVersion,
addLeadingSlash(entry.name),
createSearch(req.query)
)
);
}
/**
* Resolves a path like "lib/file" into "lib/file.js" or "lib/file.json"
* depending on which one is available, similar to require('lib/file').
*/
function resolveFile(base, useIndex, callback) {
resolveExtensions.reduceRight((next, ext) => {
const file = base + ext;
function stripLeadingSegment(name) {
return name.replace(/^[^\/]+\/?/, "");
}
return () => {
fs.stat(file, (error, stats) => {
if (error) {
if (error.code === "ENOENT" || error.code === "ENOTDIR") {
next();
} else {
callback(error);
}
} else if (useIndex && stats.isDirectory()) {
resolveFile(
path.join(file, "index"),
false,
(error, indexFile, indexStats) => {
if (error) {
callback(error);
} else if (indexFile) {
callback(null, indexFile, indexStats);
} else {
next();
}
}
);
} else {
callback(null, file, stats);
function searchEntries(tarballStream, entryName, wantsHTML) {
return new Promise((resolve, reject) => {
const entries = {};
let foundEntry = null;
if (entryName === "") {
foundEntry = entries[""] = { name: "", type: "directory" };
}
tarballStream
.on("error", reject)
.on("finish", () => resolve({ entries, foundEntry }))
.on("entry", (header, stream, next) => {
const entry = {
// Most packages have header names that look like `package/index.js`
// so we shorten that to just `index.js` here. A few packages use a
// prefix other than `package/`. e.g. the firebase package uses the
// `firebase_npm/` prefix. So we just strip the first dir name.
name: stripLeadingSegment(header.name),
type: header.type
};
// We are only interested in files that match the entryName.
if (entry.type !== "file" || entry.name.indexOf(entryName) !== 0) {
stream.resume();
stream.on("end", next);
return;
}
entries[entry.name] = entry;
// Dynamically create "directory" entries for all directories
// that are in this file's path. Some tarballs omit these entries
// for some reason, so this is the brute force method.
let dirname = path.dirname(entry.name);
while (dirname !== ".") {
const directoryEntry = { name: dirname, type: "directory" };
if (!entries[dirname]) {
entries[dirname] = directoryEntry;
if (directoryEntry.name === entryName) {
foundEntry = directoryEntry;
}
}
dirname = path.dirname(dirname);
}
// Set the foundEntry variable if this entry name
// matches exactly or if it's an index.html file
// and the client wants HTML.
if (
entry.name === entryName ||
(wantsHTML && entry.name === entryName + "/index.html")
) {
foundEntry = entry;
}
const chunks = [];
stream.on("data", chunk => chunks.push(chunk));
stream.on("end", () => {
const content = Buffer.concat(chunks);
// Set some extra properties for files that we will
// need to serve them and for ?meta listings.
entry.contentType = getContentType(entry.name);
entry.integrity = getIntegrity(content);
entry.lastModified = header.mtime.toUTCString();
entry.size = content.length;
// Set the content only for the foundEntry and
// discard the buffer for all others.
if (entry === foundEntry) {
entry.content = content;
}
next();
});
});
};
}, callback)();
});
}
function getBasename(file) {
return path.basename(file, path.extname(file));
}
const leadingSlash = /^\//;
const trailingSlash = /\/$/;
/**
* Find the file targeted by the request and get its stats. Redirect
* inexact paths in ?module mode so relative imports resolve correctly.
* Fetch and search the archive to try and find the requested file.
* Redirect to the "index" file if a directory was requested.
*/
function findFile(req, res, next) {
let filename = req.filename;
let useIndex = true;
fetchArchive(req.packageConfig).then(tarballStream => {
const entryName = req.filename
.replace(trailingSlash, "")
.replace(leadingSlash, "");
const wantsHTML = trailingSlash.test(req.filename);
if (req.query.module != null) {
// They want an ES module.
if (!filename) {
// See https://github.com/rollup/rollup/wiki/pkg.module
filename =
req.packageConfig.module || req.packageConfig["jsnext:main"] || "/";
}
} else if (filename) {
// They are requesting an explicit filename. Only try to find an
// index.js if they are NOT requesting an index page.
useIndex = filename.charAt(filename.length - 1) !== "/";
} else if (
req.query.main &&
typeof req.packageConfig[req.query.main] === "string"
) {
// They specified a custom ?main field.
// Deprecated, see https://github.com/unpkg/unpkg/issues/63
filename = req.packageConfig[req.query.main];
searchEntries(tarballStream, entryName, wantsHTML).then(
({ entries, foundEntry }) => {
if (!foundEntry) {
return res
.status(404)
.type("text")
.send(`Cannot find "${req.filename}" in ${req.packageSpec}`);
}
// Count which packages are using this so we can warn them when we
// remove this functionality.
incrementCounter(
"package-json-custom-main",
req.packageSpec + "?main=" + req.query.main,
1
// If the foundEntry is a directory and there is no trailing slash
// on the request path, we need to redirect to some "index" file
// inside that directory. This is so our URLs work in a similar way
// to require("lib") in node where it searches for `lib/index.js`
// and `lib/index.json` when `lib` is a directory.
if (foundEntry.type === "directory" && !wantsHTML) {
const indexEntry =
entries[path.join(entryName, "index.js")] ||
entries[path.join(entryName, "index.json")];
if (indexEntry && indexEntry.type === "file") {
return indexRedirect(req, res, indexEntry);
} else {
return res
.status(404)
.type("text")
.send(
`Cannot find an index in "${req.filename}" in ${
req.packageSpec
}`
);
}
}
req.entries = entries;
req.entry = foundEntry;
next();
}
);
} else if (typeof req.packageConfig.unpkg === "string") {
// The "unpkg" field allows packages to explicitly declare the
// file to serve at the bare URL.
filename = req.packageConfig.unpkg;
} else if (typeof req.packageConfig.browser === "string") {
// Fall back to the "browser" field if declared (only support strings).
// Deprecated, see https://github.com/unpkg/unpkg/issues/63
filename = req.packageConfig.browser;
// Count which packages + versions are actually using this fallback
// so we can warn them when we deprecate this functionality.
incrementCounter("package-json-browser-fallback", req.packageSpec, 1);
} else {
// Fall back to "main" or / (same as npm).
filename = req.packageConfig.main || "/";
}
resolveFile(
path.join(req.packageDir, filename),
useIndex,
(error, file, stats) => {
if (error) console.error(error);
if (file == null) {
return res
.status(404)
.type("text")
.send(
`Cannot find module "${filename}" in package ${req.packageSpec}`
);
}
filename = file.replace(req.packageDir, "");
if (req.query.main != null) {
// Permanently redirect ?main requests to their exact files.
// Deprecated, see https://github.com/unpkg/unpkg/issues/63
delete req.query.main;
return res.redirect(
301,
createPackageURL(
req.packageName,
req.packageVersion,
filename,
createSearch(req.query)
)
);
}
if (getBasename(req.filename) !== getBasename(filename)) {
// Redirect to the exact file so relative imports resolve correctly.
// Cache module redirects for 1 minute.
return res
.set({
"Cache-Control": "public, max-age=60",
"Cache-Tag": "redirect,module-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageVersion,
filename,
createSearch(req.query)
)
);
}
req.filename = filename;
req.stats = stats;
next();
}
);
});
}
module.exports = findFile;