Add fetch* utils

server/middleware/fetchPackage.js

@@ -52,16 +52,8 @@ function semverRedirect(req, res) {
  * version if the request targets a tag or uses a semver version.
  */
 function fetchPackage(req, res, next) {
-  getPackageInfo(req.packageName, (error, packageInfo) => {
+  getPackageInfo(req.packageName).then(
-    if (error) {
+    packageInfo => {
-      console.error(error);
-
-      return res
-        .status(500)
-        .type("text")
-        .send(`Cannot get info for package "${req.packageName}"`);
-    }
-
       if (packageInfo == null || packageInfo.versions == null) {
         return res
           .status(404)
@@ -75,25 +67,35 @@ function fetchPackage(req, res, next) {
         // A valid request for a package we haven't downloaded yet.
         req.packageConfig = req.packageInfo.versions[req.packageVersion];
 
-      getPackage(req.packageConfig, (error, outputDir) => {
+        getPackage(req.packageConfig).then(
-        if (error) {
+          outputDir => {
+            req.packageDir = outputDir;
+            next();
+          },
+          error => {
             console.error(error);
 
             res
               .status(500)
               .type("text")
               .send(`Cannot fetch package ${req.packageSpec}`);
-        } else {
-          req.packageDir = outputDir;
-          next();
           }
-      });
+        );
       } else if (req.packageVersion in req.packageInfo["dist-tags"]) {
         tagRedirect(req, res);
       } else {
         semverRedirect(req, res);
       }
-  });
+    },
+    error => {
+      console.error(error);
+
+      return res
+        .status(500)
+        .type("text")
+        .send(`Cannot get info for package "${req.packageName}"`);
+    }
+  );
 }
 
 module.exports = fetchPackage;

server/utils/createMutex.js

@@ -1,7 +1,20 @@
-function createMutex(doWork) {
+const invariant = require("invariant");
+
+function defaultCreateKey(payload) {
+  return payload;
+}
+
+function createMutex(doWork, createKey = defaultCreateKey) {
   const mutex = Object.create(null);
 
-  return (key, payload, callback) => {
+  return (payload, callback) => {
+    const key = createKey(payload);
+
+    invariant(
+      typeof key === "string",
+      "Mutex needs a string key; please provide a createKey function that returns a string"
+    );
+
     if (mutex[key]) {
       mutex[key].push(callback);
     } else {

server/utils/createTempPath.js

@@ -0,0 +1,9 @@
+const path = require("path");
+const tmpdir = require("os-tmpdir");
+
+function createTempPath(name, version) {
+  const hyphenName = name.replace(/\//g, "-");
+  return path.join(tmpdir(), `unpkg-${hyphenName}-${version}`);
+}
+
+module.exports = createTempPath;

server/utils/fetchPackage.js

@@ -0,0 +1,72 @@
+require("isomorphic-fetch");
+const fs = require("fs");
+const mkdirp = require("mkdirp");
+const gunzip = require("gunzip-maybe");
+const tar = require("tar-fs");
+
+const createTempPath = require("./createTempPath");
+
+function stripNamePrefix(headers) {
+  // Most packages have header names that look like "package/index.js"
+  // so we shorten that to just "index.js" here. A few packages use a
+  // prefix other than "package/". e.g. the firebase package uses the
+  // "firebase_npm/" prefix. So we just strip the first dir name.
+  headers.name = headers.name.replace(/^[^/]+\//, "");
+  return headers;
+}
+
+function ignoreLinks(file, headers) {
+  return headers.type === "link" || headers.type === "symlink";
+}
+
+function extractResponse(response, outputDir) {
+  return new Promise((resolve, reject) => {
+    const extract = tar.extract(outputDir, {
+      readable: true, // All dirs/files should be readable.
+      map: stripNamePrefix,
+      ignore: ignoreLinks
+    });
+
+    response.body
+      .pipe(gunzip())
+      .pipe(extract)
+      .on("finish", resolve)
+      .on("error", reject);
+  });
+}
+
+function fetchPackage(packageConfig) {
+  return new Promise((resolve, reject) => {
+    const tarballURL = packageConfig.dist.tarball;
+    const outputDir = createTempPath(packageConfig.name, packageConfig.version);
+
+    console.log(`info: Fetching ${tarballURL} and extracting to ${outputDir}`);
+
+    fs.access(outputDir, error => {
+      if (error) {
+        if (error.code === "ENOENT" || error.code === "ENOTDIR") {
+          // ENOENT or ENOTDIR are to be expected when we haven't yet
+          // fetched a package for the first time. Carry on!
+          mkdirp(outputDir, error => {
+            if (error) {
+              reject(error);
+            } else {
+              resolve(
+                fetch(tarballURL)
+                  .then(res => extractResponse(res, outputDir))
+                  .then(() => outputDir)
+              );
+            }
+          });
+        } else {
+          reject(error);
+        }
+      } else {
+        // Best case: we already have this package cached on disk!
+        resolve(outputDir);
+      }
+    });
+  });
+}
+
+module.exports = fetchPackage;

server/utils/fetchPackageInfo.js

@@ -0,0 +1,28 @@
+require("isomorphic-fetch");
+
+const config = require("../config");
+
+function fetchPackageInfo(packageName) {
+  return new Promise(resolve => {
+    let encodedPackageName;
+    if (packageName.charAt(0) === "@") {
+      encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`;
+    } else {
+      encodedPackageName = encodeURIComponent(packageName);
+    }
+
+    const url = `${config.registryURL}/${encodedPackageName}`;
+
+    console.log(`info: Fetching package info from ${url}`);
+
+    resolve(
+      fetch(url, {
+        headers: {
+          Accept: "application/json"
+        }
+      }).then(res => (res.status === 404 ? null : res.json()))
+    );
+  });
+}
+
+module.exports = fetchPackageInfo;

server/utils/getPackage.js

@@ -1,88 +1,26 @@
-require("isomorphic-fetch");
-const fs = require("fs");
-const path = require("path");
-const tmpdir = require("os-tmpdir");
-const gunzip = require("gunzip-maybe");
-const mkdirp = require("mkdirp");
-const tar = require("tar-fs");
-
 const createMutex = require("./createMutex");
+const fetchPackage = require("./fetchPackage");
 
-function createTempPath(name, version) {
+const fetchMutex = createMutex((packageConfig, callback) => {
-  const normalName = name.replace(/\//g, "-");
+  fetchPackage(packageConfig).then(
-  return path.join(tmpdir(), `unpkg-${normalName}-${version}`);
+    outputDir => {
+      callback(null, outputDir);
+    },
+    error => {
+      callback(error);
     }
+  );
+}, packageConfig => packageConfig.dist.tarball);
 
-function stripNamePrefix(headers) {
+function getPackage(packageConfig) {
-  // Most packages have header names that look like "package/index.js"
-  // so we shorten that to just "index.js" here. A few packages use a
-  // prefix other than "package/". e.g. the firebase package uses the
-  // "firebase_npm/" prefix. So we just strip the first dir name.
-  headers.name = headers.name.replace(/^[^/]+\//, "");
-  return headers;
-}
-
-function ignoreLinks(file, headers) {
-  return headers.type === "link" || headers.type === "symlink";
-}
-
-function extractResponse(response, outputDir) {
   return new Promise((resolve, reject) => {
-    const extract = tar.extract(outputDir, {
+    fetchMutex(packageConfig, (error, value) => {
-      readable: true, // All dirs/files should be readable.
-      map: stripNamePrefix,
-      ignore: ignoreLinks
-    });
-
-    response.body
-      .pipe(gunzip())
-      .pipe(extract)
-      .on("finish", resolve)
-      .on("error", reject);
-  });
-}
-
-function fetchAndExtract(tarballURL, outputDir) {
-  console.log(`info: Fetching ${tarballURL} and extracting to ${outputDir}`);
-
-  return fetch(tarballURL).then(response => {
-    return extractResponse(response, outputDir);
-  });
-}
-
-const fetchMutex = createMutex((payload, callback) => {
-  const { tarballURL, outputDir } = payload;
-
-  fs.access(outputDir, error => {
       if (error) {
-      if (error.code === "ENOENT" || error.code === "ENOTDIR") {
+        reject(error);
-        // ENOENT or ENOTDIR are to be expected when we haven't yet
-        // fetched a package for the first time. Carry on!
-        mkdirp(outputDir, error => {
-          if (error) {
-            callback(error);
       } else {
-            fetchAndExtract(tarballURL, outputDir).then(() => {
+        resolve(value);
-              callback();
-            }, callback);
       }
     });
-      } else {
-        callback(error);
-      }
-    } else {
-      // Best case: we already have this package cached on disk!
-      callback();
-    }
-  });
-});
-
-function getPackage(packageConfig, callback) {
-  const tarballURL = packageConfig.dist.tarball;
-  const outputDir = createTempPath(packageConfig.name, packageConfig.version);
-
-  fetchMutex(tarballURL, { tarballURL, outputDir }, error => {
-    callback(error, outputDir);
   });
 }
 

server/utils/getPackageInfo.js

@@ -1,34 +1,9 @@
-require("isomorphic-fetch");
-
-const config = require("../config");
-
 const createCache = require("./createCache");
 const createMutex = require("./createMutex");
+const fetchPackageInfo = require("./fetchPackageInfo");
 
 const packageInfoCache = createCache("packageInfo");
-
+const packageNotFound = "PackageNotFound";
-function fetchPackageInfo(packageName) {
-  console.log(`info: Fetching package info for ${packageName}`);
-
-  let encodedPackageName;
-  if (packageName.charAt(0) === "@") {
-    encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`;
-  } else {
-    encodedPackageName = encodeURIComponent(packageName);
-  }
-
-  const url = `${config.registryURL}/${encodedPackageName}`;
-
-  return fetch(url, {
-    headers: {
-      Accept: "application/json"
-    }
-  }).then(res => {
-    return res.status === 404 ? null : res.json();
-  });
-}
-
-const PackageNotFound = "PackageNotFound";
 
 // This mutex prevents multiple concurrent requests to
 // the registry for the same package info.
@@ -40,7 +15,7 @@ const fetchMutex = createMutex((packageName, callback) => {
         // unnecessary requests to the registry for bad package names.
         // In the worst case, a brand new package's info will be
         // available within 5 minutes.
-        packageInfoCache.set(packageName, PackageNotFound, 300, () => {
+        packageInfoCache.set(packageName, packageNotFound, 300, () => {
           callback(null, value);
         });
       } else {
@@ -59,14 +34,24 @@ const fetchMutex = createMutex((packageName, callback) => {
   );
 });
 
-function getPackageInfo(packageName, callback) {
+function getPackageInfo(packageName) {
+  return new Promise((resolve, reject) => {
     packageInfoCache.get(packageName, (error, value) => {
-    if (error || value != null) {
+      if (error) {
-      callback(error, value === PackageNotFound ? null : value);
+        reject(error);
+      } else if (value != null) {
+        resolve(value === packageNotFound ? null : value);
       } else {
-      fetchMutex(packageName, packageName, callback);
+        fetchMutex(packageName, (error, value) => {
+          if (error) {
+            reject(error);
+          } else {
+            resolve(value);
           }
         });
       }
+    });
+  });
+}
 
 module.exports = getPackageInfo;