Rename server => modules

modules/middleware/checkBlacklist.js

@@ -0,0 +1,26 @@
+const BlacklistAPI = require("../BlacklistAPI");
+
+function checkBlacklist(req, res, next) {
+  BlacklistAPI.includesPackage(req.packageName).then(
+    blacklisted => {
+      // Disallow packages that have been blacklisted.
+      if (blacklisted) {
+        res
+          .status(403)
+          .type("text")
+          .send(`Package "${req.packageName}" is blacklisted`);
+      } else {
+        next();
+      }
+    },
+    error => {
+      console.error(error);
+
+      res.status(500).send({
+        error: "Unable to fetch the blacklist"
+      });
+    }
+  );
+}
+
+module.exports = checkBlacklist;

modules/middleware/devAssets.js

@@ -0,0 +1,29 @@
+const invariant = require("invariant");
+
+const createAssets = require("./utils/createAssets");
+
+/**
+ * An express middleware that sets req.assets from the
+ * latest result from a running webpack compiler (i.e. using
+ * webpack-dev-middleware). Should only be used in dev.
+ */
+function devAssets(webpackCompiler) {
+  let assets;
+  webpackCompiler.plugin("done", stats => {
+    assets = createAssets(stats.toJson());
+  });
+
+  return (req, res, next) => {
+    invariant(
+      assets != null,
+      "devAssets middleware needs a running compiler; " +
+        "use webpack-dev-middleware in front of devAssets"
+    );
+
+    req.assets = assets;
+
+    next();
+  };
+}
+
+module.exports = devAssets;

modules/middleware/fetchPackage.js

@@ -0,0 +1,153 @@
+const semver = require("semver");
+
+const addLeadingSlash = require("../utils/addLeadingSlash");
+const createPackageURL = require("../utils/createPackageURL");
+const createSearch = require("../utils/createSearch");
+const getNpmPackageInfo = require("../utils/getNpmPackageInfo");
+const incrementCounter = require("../utils/incrementCounter");
+
+function tagRedirect(req, res) {
+  const version = req.packageInfo["dist-tags"][req.packageVersion];
+
+  // Cache tag redirects for 1 minute.
+  res
+    .set({
+      "Cache-Control": "public, max-age=60",
+      "Cache-Tag": "redirect,tag-redirect"
+    })
+    .redirect(
+      302,
+      createPackageURL(req.packageName, version, req.filename, req.search)
+    );
+}
+
+function semverRedirect(req, res) {
+  const maxVersion = semver.maxSatisfying(
+    Object.keys(req.packageInfo.versions),
+    req.packageVersion
+  );
+
+  if (maxVersion) {
+    // Cache semver redirects for 1 minute.
+    res
+      .set({
+        "Cache-Control": "public, max-age=60",
+        "Cache-Tag": "redirect,semver-redirect"
+      })
+      .redirect(
+        302,
+        createPackageURL(req.packageName, maxVersion, req.filename, req.search)
+      );
+  } else {
+    res
+      .status(404)
+      .type("text")
+      .send(`Cannot find package ${req.packageSpec}`);
+  }
+}
+
+function filenameRedirect(req, res) {
+  let filename;
+  if (req.query.module != null) {
+    // See https://github.com/rollup/rollup/wiki/pkg.module
+    filename =
+      req.packageConfig.module ||
+      req.packageConfig["jsnext:main"] ||
+      "/index.js";
+  } else if (
+    req.query.main &&
+    req.packageConfig[req.query.main] &&
+    typeof req.packageConfig[req.query.main] === "string"
+  ) {
+    // Deprecated, see #63
+    filename = req.packageConfig[req.query.main];
+
+    // Count which packages are using this so we can warn them when we
+    // remove this functionality.
+    incrementCounter(
+      "package-json-custom-main",
+      req.packageSpec + "?main=" + req.query.main,
+      1
+    );
+  } else if (
+    req.packageConfig.unpkg &&
+    typeof req.packageConfig.unpkg === "string"
+  ) {
+    filename = req.packageConfig.unpkg;
+  } else if (
+    req.packageConfig.browser &&
+    typeof req.packageConfig.browser === "string"
+  ) {
+    // Deprecated, see #63
+    filename = req.packageConfig.browser;
+
+    // Count which packages are using this so we can warn them when we
+    // remove this functionality.
+    incrementCounter("package-json-browser-fallback", req.packageSpec, 1);
+  } else {
+    filename = req.packageConfig.main || "/index.js";
+  }
+
+  // Redirect to the exact filename so relative imports
+  // and URLs resolve correctly.
+  // TODO: increase the max-age?
+  res
+    .set({
+      "Cache-Control": "public,max-age=60",
+      "Cache-Tag": "redirect,filename-redirect"
+    })
+    .redirect(
+      302,
+      createPackageURL(
+        req.packageName,
+        req.packageVersion,
+        addLeadingSlash(filename),
+        createSearch(req.query)
+      )
+    );
+}
+
+/**
+ * Fetch the package metadata and tarball from npm. Redirect to the exact
+ * version if the request targets a tag or uses a semver version, or to the
+ * exact filename if the request omits the filename.
+ */
+function fetchPackage(req, res, next) {
+  getNpmPackageInfo(req.packageName).then(
+    packageInfo => {
+      if (packageInfo == null || packageInfo.versions == null) {
+        return res
+          .status(404)
+          .type("text")
+          .send(`Cannot find package "${req.packageName}"`);
+      }
+
+      req.packageInfo = packageInfo;
+      req.packageConfig = req.packageInfo.versions[req.packageVersion];
+
+      if (!req.packageConfig) {
+        if (req.packageVersion in req.packageInfo["dist-tags"]) {
+          return tagRedirect(req, res);
+        } else {
+          return semverRedirect(req, res);
+        }
+      }
+
+      if (!req.filename) {
+        return filenameRedirect(req, res);
+      }
+
+      next();
+    },
+    error => {
+      console.error(error);
+
+      return res
+        .status(500)
+        .type("text")
+        .send(`Cannot get info for package "${req.packageName}"`);
+    }
+  );
+}
+
+module.exports = fetchPackage;

modules/middleware/findFile.js

@@ -0,0 +1,178 @@
+const path = require("path");
+
+const addLeadingSlash = require("../utils/addLeadingSlash");
+const createPackageURL = require("../utils/createPackageURL");
+const createSearch = require("../utils/createSearch");
+const fetchNpmPackage = require("../utils/fetchNpmPackage");
+const getIntegrity = require("../utils/getIntegrity");
+const getContentType = require("../utils/getContentType");
+
+function indexRedirect(req, res, entry) {
+  // Redirect to the index file so relative imports
+  // resolve correctly.
+  // TODO: increase the max-age?
+  res
+    .set({
+      "Cache-Control": "public,max-age=60",
+      "Cache-Tag": "redirect,index-redirect"
+    })
+    .redirect(
+      302,
+      createPackageURL(
+        req.packageName,
+        req.packageVersion,
+        addLeadingSlash(entry.name),
+        createSearch(req.query)
+      )
+    );
+}
+
+function stripLeadingSegment(name) {
+  return name.replace(/^[^\/]+\/?/, "");
+}
+
+function searchEntries(tarballStream, entryName, wantsHTML) {
+  return new Promise((resolve, reject) => {
+    const entries = {};
+    let foundEntry = null;
+
+    if (entryName === "") {
+      foundEntry = entries[""] = { name: "", type: "directory" };
+    }
+
+    tarballStream
+      .on("error", reject)
+      .on("finish", () => resolve({ entries, foundEntry }))
+      .on("entry", (header, stream, next) => {
+        const entry = {
+          // Most packages have header names that look like `package/index.js`
+          // so we shorten that to just `index.js` here. A few packages use a
+          // prefix other than `package/`. e.g. the firebase package uses the
+          // `firebase_npm/` prefix. So we just strip the first dir name.
+          name: stripLeadingSegment(header.name),
+          type: header.type
+        };
+
+        // We are only interested in files that match the entryName.
+        if (entry.type !== "file" || entry.name.indexOf(entryName) !== 0) {
+          stream.resume();
+          stream.on("end", next);
+          return;
+        }
+
+        entries[entry.name] = entry;
+
+        // Dynamically create "directory" entries for all directories
+        // that are in this file's path. Some tarballs omit these entries
+        // for some reason, so this is the brute force method.
+        let dirname = path.dirname(entry.name);
+        while (dirname !== ".") {
+          const directoryEntry = { name: dirname, type: "directory" };
+
+          if (!entries[dirname]) {
+            entries[dirname] = directoryEntry;
+
+            if (directoryEntry.name === entryName) {
+              foundEntry = directoryEntry;
+            }
+          }
+
+          dirname = path.dirname(dirname);
+        }
+
+        // Set the foundEntry variable if this entry name
+        // matches exactly or if it's an index.html file
+        // and the client wants HTML.
+        if (
+          entry.name === entryName ||
+          // Allow accessing e.g. `/lib/index.html` using `/lib/`
+          (wantsHTML && entry.name === `${entryName}/index.html`) ||
+          // Allow accessing e.g. `/index.js` or `/index.json` using
+          // `/index` for compatibility with CommonJS
+          (!wantsHTML && entry.name === `${entryName}.js`) ||
+          (!wantsHTML && entry.name === `${entryName}.json`)
+        ) {
+          foundEntry = entry;
+        }
+
+        const chunks = [];
+
+        stream.on("data", chunk => chunks.push(chunk)).on("end", () => {
+          const content = Buffer.concat(chunks);
+
+          // Set some extra properties for files that we will
+          // need to serve them and for ?meta listings.
+          entry.contentType = getContentType(entry.name);
+          entry.integrity = getIntegrity(content);
+          entry.lastModified = header.mtime.toUTCString();
+          entry.size = content.length;
+
+          // Set the content only for the foundEntry and
+          // discard the buffer for all others.
+          if (entry === foundEntry) {
+            entry.content = content;
+          }
+
+          next();
+        });
+      });
+  });
+}
+
+const leadingSlash = /^\//;
+const trailingSlash = /\/$/;
+
+/**
+ * Fetch and search the archive to try and find the requested file.
+ * Redirect to the "index" file if a directory was requested.
+ */
+function findFile(req, res, next) {
+  fetchNpmPackage(req.packageConfig).then(tarballStream => {
+    const entryName = req.filename
+      .replace(trailingSlash, "")
+      .replace(leadingSlash, "");
+    const wantsHTML = trailingSlash.test(req.filename);
+
+    searchEntries(tarballStream, entryName, wantsHTML).then(
+      ({ entries, foundEntry }) => {
+        if (!foundEntry) {
+          return res
+            .status(404)
+            .type("text")
+            .send(`Cannot find "${req.filename}" in ${req.packageSpec}`);
+        }
+
+        // If the foundEntry is a directory and there is no trailing slash
+        // on the request path, we need to redirect to some "index" file
+        // inside that directory. This is so our URLs work in a similar way
+        // to require("lib") in node where it searches for `lib/index.js`
+        // and `lib/index.json` when `lib` is a directory.
+        if (foundEntry.type === "directory" && !wantsHTML) {
+          const indexEntry =
+            entries[path.join(entryName, "index.js")] ||
+            entries[path.join(entryName, "index.json")];
+
+          if (indexEntry && indexEntry.type === "file") {
+            return indexRedirect(req, res, indexEntry);
+          } else {
+            return res
+              .status(404)
+              .type("text")
+              .send(
+                `Cannot find an index in "${req.filename}" in ${
+                  req.packageSpec
+                }`
+              );
+          }
+        }
+
+        req.entries = entries;
+        req.entry = foundEntry;
+
+        next();
+      }
+    );
+  });
+}
+
+module.exports = findFile;

modules/middleware/redirectLegacyURLs.js

@@ -0,0 +1,23 @@
+const createSearch = require("../utils/createSearch");
+
+/**
+ * Redirect old URLs that we no longer support.
+ */
+function redirectLegacyURLs(req, res, next) {
+  // Permanently redirect /_meta/path to /path?meta.
+  if (req.path.match(/^\/_meta\//)) {
+    req.query.meta = "";
+    return res.redirect(301, req.path.substr(6) + createSearch(req.query));
+  }
+
+  // Permanently redirect /path?json => /path?meta
+  if (req.query.json != null) {
+    delete req.query.json;
+    req.query.meta = "";
+    return res.redirect(301, req.path + createSearch(req.query));
+  }
+
+  next();
+}
+
+module.exports = redirectLegacyURLs;

modules/middleware/requireAuth.js

@@ -0,0 +1,40 @@
+/**
+ * Adds the given scope to the array in req.auth if the user has sufficient
+ * permissions. Otherwise rejects the request.
+ */
+function requireAuth(scope) {
+  let checkScopes;
+  if (scope.includes(".")) {
+    const parts = scope.split(".");
+    checkScopes = scopes =>
+      parts.reduce((memo, part) => memo && memo[part], scopes) != null;
+  } else {
+    checkScopes = scopes => scopes[scope] != null;
+  }
+
+  return function(req, res, next) {
+    if (req.auth && req.auth.includes(scope)) {
+      return next(); // Already auth'd
+    }
+
+    const user = req.user;
+
+    if (!user) {
+      return res.status(403).send({ error: "Missing auth token" });
+    }
+
+    if (!user.scopes || !checkScopes(user.scopes)) {
+      return res.status(403).send({ error: "Insufficient scopes" });
+    }
+
+    if (req.auth) {
+      req.auth.push(scope);
+    } else {
+      req.auth = [scope];
+    }
+
+    next();
+  };
+}
+
+module.exports = requireAuth;

modules/middleware/staticAssets.js

@@ -0,0 +1,31 @@
+const fs = require("fs");
+const invariant = require("invariant");
+
+const createAssets = require("./utils/createAssets");
+
+/**
+ * An express middleware that sets req.assets from the build
+ * info in the given stats file. Should be used in production.
+ */
+function staticAssets(webpackStatsFile) {
+  let stats;
+  try {
+    stats = JSON.parse(fs.readFileSync(webpackStatsFile, "utf8"));
+  } catch (error) {
+    invariant(
+      false,
+      "staticAssets middleware cannot read the build stats in %s; " +
+        "run the `build` script before starting the server",
+      webpackStatsFile
+    );
+  }
+
+  const assets = createAssets(stats);
+
+  return (req, res, next) => {
+    req.assets = assets;
+    next();
+  };
+}
+
+module.exports = staticAssets;

modules/middleware/userToken.js

@@ -0,0 +1,41 @@
+const AuthAPI = require("../AuthAPI");
+
+const ReadMethods = { GET: true, HEAD: true };
+
+/**
+ * Sets req.user from the payload in the auth token in the request.
+ */
+function userToken(req, res, next) {
+  if (req.user) {
+    return next();
+  }
+
+  const token = (ReadMethods[req.method] ? req.query : req.body).token;
+
+  if (!token) {
+    req.user = null;
+    return next();
+  }
+
+  AuthAPI.verifyToken(token).then(
+    payload => {
+      req.user = payload;
+      next();
+    },
+    error => {
+      if (error.name === "JsonWebTokenError") {
+        res.status(403).send({
+          error: `Bad auth token: ${error.message}`
+        });
+      } else {
+        console.error(error);
+
+        res.status(500).send({
+          error: "Unable to verify auth"
+        });
+      }
+    }
+  );
+}
+
+module.exports = userToken;

modules/middleware/utils/createAssets.js

@@ -0,0 +1,40 @@
+/**
+ * Creates an assets object that is stored on req.assets.
+ */
+function createAssets(webpackStats) {
+  const { publicPath, assetsByChunkName } = webpackStats;
+
+  /**
+   * Returns a public URL to the given asset.
+   */
+  const createURL = asset => publicPath + asset;
+
+  /**
+   * Returns an array of URLs to all assets in the given chunks.
+   */
+  const getAll = (chunks = ["main"]) =>
+    (Array.isArray(chunks) ? chunks : [chunks])
+      .reduce((memo, chunk) => memo.concat(assetsByChunkName[chunk] || []), [])
+      .map(createURL);
+
+  /**
+   * Returns an array of URLs to all JavaScript files in the given chunks.
+   */
+  const getScripts = (...chunks) =>
+    getAll(...chunks).filter(asset => /\.js$/.test(asset));
+
+  /**
+   * Returns an array of URLs to all CSS files in the given chunks.
+   */
+  const getStyles = (...chunks) =>
+    getAll(...chunks).filter(asset => /\.css$/.test(asset));
+
+  return {
+    createURL,
+    getAll,
+    getScripts,
+    getStyles
+  };
+}
+
+module.exports = createAssets;

modules/middleware/validatePackageName.js

@@ -0,0 +1,21 @@
+const validateNpmPackageName = require("validate-npm-package-name");
+
+/**
+ * Reject requests for invalid npm package names.
+ */
+function validatePackageName(req, res, next) {
+  const errors = validateNpmPackageName(req.packageName).errors;
+
+  if (errors) {
+    const reason = errors.join(", ");
+
+    return res
+      .status(403)
+      .type("text")
+      .send(`Invalid package name "${req.packageName}" (${reason})`);
+  }
+
+  next();
+}
+
+module.exports = validatePackageName;

modules/middleware/validatePackageURL.js

@@ -0,0 +1,25 @@
+const parsePackageURL = require("../utils/parsePackageURL");
+
+/**
+ * Parse the URL and add various properties to the request object to
+ * do with the package/file being requested. Reject invalid URLs.
+ */
+function validatePackageURL(req, res, next) {
+  const url = parsePackageURL(req.url);
+
+  if (url == null) {
+    return res.status(403).send({ error: `Invalid URL: ${req.url}` });
+  }
+
+  req.packageName = url.packageName;
+  req.packageVersion = url.packageVersion;
+  req.packageSpec = `${url.packageName}@${url.packageVersion}`;
+  req.pathname = url.pathname; // TODO: remove
+  req.filename = url.filename;
+  req.search = url.search;
+  req.query = url.query;
+
+  next();
+}
+
+module.exports = validatePackageURL;

modules/middleware/validateQuery.js

@@ -0,0 +1,34 @@
+const createSearch = require("../utils/createSearch");
+
+const knownQueryParams = {
+  main: true, // Deprecated, see #63
+  meta: true,
+  module: true
+};
+
+function isKnownQueryParam(param) {
+  return !!knownQueryParams[param];
+}
+
+function sanitizeQuery(originalQuery) {
+  const query = {};
+
+  Object.keys(originalQuery).forEach(param => {
+    if (isKnownQueryParam(param)) query[param] = originalQuery[param];
+  });
+
+  return query;
+}
+
+/**
+ * Reject URLs with invalid query parameters to increase cache hit rates.
+ */
+function validateQuery(req, res, next) {
+  if (!Object.keys(req.query).every(isKnownQueryParam)) {
+    return res.redirect(302, req.path + createSearch(sanitizeQuery(req.query)));
+  }
+
+  next();
+}
+
+module.exports = validateQuery;