Move some utils into middleware/utils

server/middleware/fetchFile.js

@@ -1,8 +1,8 @@
 const fs = require('fs')
 const path = require('path')
 const semver = require('semver')
-const PackageCache = require('../PackageCache')
-const PackageInfo = require('../PackageInfo')
+const getPackage = require('./utils/getPackage')
+const getPackageInfo = require('./utils/getPackageInfo')
 const PackageURL = require('../PackageURL')
 
 const FindExtensions = [ '', '.js', '.json' ]
@@ -47,7 +47,7 @@ function findFile(base, useIndex, callback) {
  * trailing slash.
  */
 function fetchFile(req, res, next) {
-  PackageInfo.get(req.packageName, function (error, packageInfo) {
+  getPackageInfo(req.packageName, function (error, packageInfo) {
     if (error) {
       console.error(error)
       return res.status(500).type('text').send(`Cannot get info for package "${req.packageName}"`)
@@ -64,7 +64,7 @@ function fetchFile(req, res, next) {
       // A valid request for a package we haven't downloaded yet.
       req.packageConfig = versions[req.packageVersion]
 
-      PackageCache.get(req.packageConfig, function (error, outputDir) {
+      getPackage(req.packageConfig, function (error, outputDir) {
         if (error) {
           console.error(error)
           res.status(500).type('text').send(`Cannot fetch package ${req.packageSpec}`)

server/middleware/utils/createCache.js

@@ -0,0 +1,29 @@
+const db = require('../../RedisClient')
+
+function createCache(keyPrefix) {
+  function createKey(key) {
+    return keyPrefix + '-' + key
+  }
+
+  function set(key, value, expiry, callback) {
+    db.setex(createKey(key), expiry, JSON.stringify(value), callback)
+  }
+
+  function get(key, callback) {
+    db.get(createKey(key), function (error, value) {
+      callback(error, value && JSON.parse(value))
+    })
+  }
+
+  function del(key, callback) {
+    db.del(createKey(key), callback)
+  }
+
+  return {
+    set,
+    get,
+    del
+  }
+}
+
+module.exports = createCache

server/middleware/utils/createMutex.js

@@ -0,0 +1,21 @@
+function createMutex(doWork) {
+  const mutex = {}
+
+  return function (key, payload, callback) {
+    if (mutex[key]) {
+      mutex[key].push(callback)
+    } else {
+      mutex[key] = [ function () {
+        delete mutex[key]
+      }, callback ]
+
+      doWork(payload, function (error, value) {
+        mutex[key].forEach(function (callback) {
+          callback(error, value)
+        })
+      })
+    }
+  }
+}
+
+module.exports = createMutex

server/middleware/utils/getPackage.js

@@ -0,0 +1,88 @@
+require('isomorphic-fetch')
+const fs = require('fs')
+const path = require('path')
+const tmpdir = require('os-tmpdir')
+const gunzip = require('gunzip-maybe')
+const mkdirp = require('mkdirp')
+const tar = require('tar-fs')
+const createMutex = require('./createMutex')
+
+function createTempPath(name, version) {
+  const normalName = name.replace(/\//g, '-')
+  return path.join(tmpdir(), `unpkg-${normalName}-${version}`)
+}
+
+function stripNamePrefix(headers) {
+  // Most packages have header names that look like "package/index.js"
+  // so we shorten that to just "index.js" here. A few packages use a
+  // prefix other than "package/". e.g. the firebase package uses the
+  // "firebase_npm/" prefix. So we just strip the first dir name.
+  headers.name = headers.name.replace(/^[^\/]+\//, '')
+  return headers
+}
+
+function ignoreSymlinks(file, headers) {
+  return headers.type === 'link'
+}
+
+function extractResponse(response, outputDir) {
+  return new Promise(function (resolve, reject) {
+    const extract = tar.extract(outputDir, {
+      readable: true, // All dirs/files should be readable.
+      map: stripNamePrefix,
+      ignore: ignoreSymlinks
+    })
+
+    response.body
+      .pipe(gunzip())
+      .pipe(extract)
+      .on('finish', resolve)
+      .on('error', reject)
+  })
+}
+
+function fetchAndExtract(tarballURL, outputDir) {
+  console.log(`info: Fetching ${tarballURL} and extracting to ${outputDir}`)
+
+  return fetch(tarballURL).then(function (response) {
+    return extractResponse(response, outputDir)
+  })
+}
+
+const fetchMutex = createMutex(function (payload, callback) {
+  const { tarballURL, outputDir } = payload
+
+  fs.access(outputDir, function (error) {
+    if (error) {
+      if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
+        // ENOENT or ENOTDIR are to be expected when we haven't yet
+        // fetched a package for the first time. Carry on!
+        mkdirp(outputDir, function (error) {
+          if (error) {
+            callback(error)
+          } else {
+            fetchAndExtract(tarballURL, outputDir).then(function () {
+              callback()
+            }, callback)
+          }
+        })
+      } else {
+        callback(error)
+      }
+    } else {
+      // Best case: we already have this package cached on disk!
+      callback()
+    }
+  })
+})
+
+function getPackage(packageConfig, callback) {
+  const tarballURL = packageConfig.dist.tarball
+  const outputDir = createTempPath(packageConfig.name, packageConfig.version)
+
+  fetchMutex(tarballURL, { tarballURL, outputDir }, function (error) {
+    callback(error, outputDir)
+  })
+}
+
+module.exports = getPackage

server/middleware/utils/getPackageInfo.js

@@ -0,0 +1,66 @@
+require('isomorphic-fetch')
+const createCache = require('./createCache')
+const createMutex = require('./createMutex')
+
+const RegistryURL = process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org'
+
+const PackageInfoCache = createCache('packageInfo')
+
+function fetchPackageInfo(packageName) {
+  console.log(`info: Fetching package info for ${packageName}`)
+
+  let encodedPackageName
+  if (packageName.charAt(0) === '@') {
+    encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`
+  } else {
+    encodedPackageName = encodeURIComponent(packageName)
+  }
+
+  const url = `${RegistryURL}/${encodedPackageName}`
+
+  return fetch(url, {
+    headers: { 'Accept': 'application/json' }
+  }).then(function (res) {
+    return res.status === 404 ? null : res.json()
+  })
+}
+
+const PackageNotFound = 'PackageNotFound'
+
+// This mutex prevents multiple concurrent requests to
+// the registry for the same package info.
+const fetchMutex = createMutex(function (packageName, callback) {
+  fetchPackageInfo(packageName).then(function (value) {
+    if (value == null) {
+      // Cache 404s for 5 minutes. This prevents us from making
+      // unnecessary requests to the registry for bad package names.
+      // In the worst case, a brand new package's info will be
+      // available within 5 minutes.
+      PackageInfoCache.set(packageName, PackageNotFound, 300, function () {
+        callback(null, value)
+      })
+    } else {
+      // Cache valid package info for 1 minute.
+      PackageInfoCache.set(packageName, value, 60, function () {
+        callback(null, value)
+      })
+    }
+  }, function (error) {
+    // Do not cache errors.
+    PackageInfoCache.del(packageName, function () {
+      callback(error)
+    })
+  })
+})
+
+function getPackageInfo(packageName, callback) {
+  PackageInfoCache.get(packageName, function (error, value) {
+    if (error || value != null) {
+      callback(error, value === PackageNotFound ? null : value)
+    } else {
+      fetchMutex(packageName, packageName, callback)
+    }
+  })
+}
+
+module.exports = getPackageInfo