Require packages to be downloaded >= 100x/day

This should make it more difficult for people who are publishing
malicious packages to npm to get them on the CDN.

client/About.md

@@ -43,7 +43,11 @@ unpkg is not affiliated with or supported by npm, Inc. in any way. Please do not
 
 ### Abuse
 
-unpkg blacklists some packages to prevent abuse. If you find a malicious package on npm, please take a moment to add it to [our blacklist](https://github.com/unpkg/unpkg.com/blob/master/server/PackageBlacklist.json).
+Currently, unpkg tries to prevent people from abusing the CDN in a few different ways.
+
+First, in order to be available on unpkg a package must have been downloaded from the npm registry an average of 100 times per day over the past week.
+
+Secondly, unpkg maintains a blacklist of packages that are known to be malicious. If you find such a package on npm, please take a moment to submit a PR that adds it to [our blacklist](https://github.com/unpkg/unpkg.com/blob/master/server/PackageBlacklist.json).
 
 ### Feedback
 

server/CloudflareAPI.js

@@ -1,6 +1,7 @@
 require('isomorphic-fetch')
 const invariant = require('invariant')
 
+const CloudflareAPIURL = 'https://api.cloudflare.com'
 const CloudflareEmail = process.env.CLOUDFLARE_EMAIL
 const CloudflareKey = process.env.CLOUDFLARE_KEY
 
@@ -15,8 +16,7 @@ invariant(
 )
 
 function get(path, headers) {
-  return fetch(`https://api.cloudflare.com/client/v4${path}`, {
-    method: 'GET',
+  return fetch(`${CloudflareAPIURL}/client/v4${path}`, {
     headers: Object.assign({}, headers, {
       'X-Auth-Email': CloudflareEmail,
       'X-Auth-Key': CloudflareKey

server/NPMAPI.js

@@ -0,0 +1,17 @@
+require('isomorphic-fetch')
+
+const NPMAPIURL = 'https://api.npmjs.org'
+
+function getJSON(path) {
+  return fetch(`${NPMAPIURL}${path}`, {
+    headers: {
+      Accept: 'application/json'
+    }
+  }).then(function (res) {
+    return res.status === 404 ? null : res.json()
+  })
+}
+
+module.exports = {
+  getJSON
+}

server/NPMDownloads.js

@@ -0,0 +1,53 @@
+const NPMAPI = require('./NPMAPI')
+const createCache = require('./createCache')
+const createMutex = require('./createMutex')
+
+const NPMDownloadsCache = createCache('npmDownloads')
+
+function fetchDailyDownloads(packageName) {
+  console.log(`info: Fetching downloads for ${packageName}`)
+
+  return NPMAPI.getJSON(`/downloads/point/last-week/${packageName}`).then(function (data) {
+    return data && Math.round(data.downloads / 7)
+  })
+}
+
+const PackageNotFound = 'PackageNotFound'
+
+const fetchMutex = createMutex(function (packageName, callback) {
+  fetchDailyDownloads(packageName).then(function (value) {
+    if (value == null) {
+      // Cache 404s for 5 minutes. This prevents us from making
+      // unnecessary requests to the NPM API for bad package names.
+      // In the worst case, a brand new package's downloads will be
+      // available within 5 minutes.
+      NPMDownloadsCache.set(packageName, PackageNotFound, 300, function () {
+        callback(null, value)
+      })
+    } else {
+      // Cache downloads for 1 minute.
+      NPMDownloadsCache.set(packageName, value, 60, function () {
+        callback(null, value)
+      })
+    }
+  }, function (error) {
+    // Do not cache errors.
+    NPMDownloadsCache.del(packageName, function () {
+      callback(error)
+    })
+  })
+})
+
+function getDailyDownloads(packageName, callback) {
+  NPMDownloadsCache.get(packageName, function (error, value) {
+    if (error || value != null) {
+      callback(error, value === PackageNotFound ? null : value)
+    } else {
+      fetchMutex(packageName, packageName, callback)
+    }
+  })
+}
+
+module.exports = {
+  getDaily: getDailyDownloads
+}

server/PackageInfo.js

@@ -1,9 +1,11 @@
 require('isomorphic-fetch')
-const PackageInfoCache = require('./PackageInfoCache')
+const createCache = require('./createCache')
 const createMutex = require('./createMutex')
 
 const RegistryURL = process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org'
 
+const PackageInfoCache = createCache('packageInfo')
+
 function fetchPackageInfo(packageName) {
   console.log(`info: Fetching package info for ${packageName}`)
 
@@ -18,8 +20,8 @@ function fetchPackageInfo(packageName) {
 
   return fetch(url, {
     headers: { 'Accept': 'application/json' }
-  }).then(function (response) {
-    return response.status === 404 ? null : response.json()
+  }).then(function (res) {
+    return res.status === 404 ? null : res.json()
   })
 }
 
@@ -53,10 +55,8 @@ const fetchMutex = createMutex(function (packageName, callback) {
 
 function getPackageInfo(packageName, callback) {
   PackageInfoCache.get(packageName, function (error, value) {
-    if (error) {
-      callback(error)
-    } else if (value) {
-      callback(null, value === PackageNotFound ? null : value)
+    if (error || value != null) {
+      callback(error, value === PackageNotFound ? null : value)
     } else {
       fetchMutex(packageName, packageName, callback)
     }

server/PackageInfoCache.js

@@ -1,25 +0,0 @@
-const db = require('./RedisClient')
-
-function createKey(packageName) {
-  return 'packageInfo-' + packageName
-}
-
-function set(packageName, value, expiry, callback) {
-  db.setex(createKey(packageName), expiry, JSON.stringify(value), callback)
-}
-
-function get(packageName, callback) {
-  db.get(createKey(packageName), function (error, value) {
-    callback(error, value && JSON.parse(value))
-  })
-}
-
-function del(packageName, callback) {
-  db.del(createKey(packageName), callback)
-}
-
-module.exports = {
-  set,
-  get,
-  del
-}

server/createCache.js

@@ -0,0 +1,29 @@
+const db = require('./RedisClient')
+
+function createCache(keyPrefix) {
+  function createKey(key) {
+    return keyPrefix + '-' + key
+  }
+
+  function set(key, value, expiry, callback) {
+    db.setex(createKey(key), expiry, JSON.stringify(value), callback)
+  }
+
+  function get(key, callback) {
+    db.get(createKey(key), function (error, value) {
+      callback(error, value && JSON.parse(value))
+    })
+  }
+
+  function del(key, callback) {
+    db.del(createKey(key), callback)
+  }
+
+  return {
+    set,
+    get,
+    del
+  }
+}
+
+module.exports = createCache

server/createServer.js

@@ -6,6 +6,8 @@ const cors = require('cors')
 const morgan = require('morgan')
 
 const { fetchStats } = require('./cloudflare')
+
+const checkMinDailyDownloads = require('./middleware/checkMinDailyDownloads')
 const parsePackageURL = require('./middleware/parsePackageURL')
 const fetchFile = require('./middleware/fetchFile')
 const serveFile = require('./middleware/serveFile')
@@ -67,8 +69,19 @@ function createServer() {
     maxAge: '365d'
   }))
 
-  app.use('/_meta', parsePackageURL, fetchFile, serveMetadata)
-  app.use('/', parsePackageURL, fetchFile, serveFile)
+  app.use('/_meta',
+    parsePackageURL,
+    checkMinDailyDownloads(100),
+    fetchFile,
+    serveMetadata
+  )
+
+  app.use('/',
+    parsePackageURL,
+    checkMinDailyDownloads(100),
+    fetchFile,
+    serveFile
+  )
 
   const server = http.createServer(app)
 

server/middleware/checkMinDailyDownloads.js

@@ -0,0 +1,20 @@
+const NPMDownloads = require('../NPMDownloads')
+
+function checkMinDailyDownloads(minDailyDownloads) {
+  return function (req, res, next) {
+    NPMDownloads.getDaily(req.packageName, function (error, downloads) {
+      if (error) {
+        console.error(error)
+        next() // Keep going; this error isn't critical.
+      } else if (downloads == null) {
+        res.status(404).type('text').send(`Cannot find package "${req.packageName}"`)
+      } else if (downloads >= minDailyDownloads) {
+        next()
+      } else {
+        res.status(404).type('text').send(`Cannot serve requests for package "${req.packageName}" because it has been downloaded on average only ${downloads} time${downloads > 1 ? 's' : ''} per day this week`)
+      }
+    })
+  }
+}
+
+module.exports = checkMinDailyDownloads