diff --git a/client/About.md b/client/About.md index ca36e06..63bd1fe 100644 --- a/client/About.md +++ b/client/About.md @@ -43,7 +43,11 @@ unpkg is not affiliated with or supported by npm, Inc. in any way. Please do not ### Abuse -unpkg blacklists some packages to prevent abuse. If you find a malicious package on npm, please take a moment to add it to [our blacklist](https://github.com/unpkg/unpkg.com/blob/master/server/PackageBlacklist.json). +Currently, unpkg tries to prevent people from abusing the CDN in a few different ways. + +First, in order to be available on unpkg a package must have been downloaded from the npm registry an average of 100 times per day over the past week. + +Secondly, unpkg maintains a blacklist of packages that are known to be malicious. If you find such a package on npm, please take a moment to submit a PR that adds it to [our blacklist](https://github.com/unpkg/unpkg.com/blob/master/server/PackageBlacklist.json). ### Feedback diff --git a/server/CloudflareAPI.js b/server/CloudflareAPI.js index 18de473..0b33d98 100644 --- a/server/CloudflareAPI.js +++ b/server/CloudflareAPI.js @@ -1,6 +1,7 @@ require('isomorphic-fetch') const invariant = require('invariant') +const CloudflareAPIURL = 'https://api.cloudflare.com' const CloudflareEmail = process.env.CLOUDFLARE_EMAIL const CloudflareKey = process.env.CLOUDFLARE_KEY @@ -15,8 +16,7 @@ invariant( ) function get(path, headers) { - return fetch(`https://api.cloudflare.com/client/v4${path}`, { - method: 'GET', + return fetch(`${CloudflareAPIURL}/client/v4${path}`, { headers: Object.assign({}, headers, { 'X-Auth-Email': CloudflareEmail, 'X-Auth-Key': CloudflareKey diff --git a/server/NPMAPI.js b/server/NPMAPI.js new file mode 100644 index 0000000..0fb4120 --- /dev/null +++ b/server/NPMAPI.js @@ -0,0 +1,17 @@ +require('isomorphic-fetch') + +const NPMAPIURL = 'https://api.npmjs.org' + +function getJSON(path) { + return fetch(`${NPMAPIURL}${path}`, { + headers: { + Accept: 'application/json' + } + }).then(function (res) { + return res.status === 404 ? null : res.json() + }) +} + +module.exports = { + getJSON +} diff --git a/server/NPMDownloads.js b/server/NPMDownloads.js new file mode 100644 index 0000000..0194b46 --- /dev/null +++ b/server/NPMDownloads.js @@ -0,0 +1,53 @@ +const NPMAPI = require('./NPMAPI') +const createCache = require('./createCache') +const createMutex = require('./createMutex') + +const NPMDownloadsCache = createCache('npmDownloads') + +function fetchDailyDownloads(packageName) { + console.log(`info: Fetching downloads for ${packageName}`) + + return NPMAPI.getJSON(`/downloads/point/last-week/${packageName}`).then(function (data) { + return data && Math.round(data.downloads / 7) + }) +} + +const PackageNotFound = 'PackageNotFound' + +const fetchMutex = createMutex(function (packageName, callback) { + fetchDailyDownloads(packageName).then(function (value) { + if (value == null) { + // Cache 404s for 5 minutes. This prevents us from making + // unnecessary requests to the NPM API for bad package names. + // In the worst case, a brand new package's downloads will be + // available within 5 minutes. + NPMDownloadsCache.set(packageName, PackageNotFound, 300, function () { + callback(null, value) + }) + } else { + // Cache downloads for 1 minute. + NPMDownloadsCache.set(packageName, value, 60, function () { + callback(null, value) + }) + } + }, function (error) { + // Do not cache errors. + NPMDownloadsCache.del(packageName, function () { + callback(error) + }) + }) +}) + +function getDailyDownloads(packageName, callback) { + NPMDownloadsCache.get(packageName, function (error, value) { + if (error || value != null) { + callback(error, value === PackageNotFound ? null : value) + } else { + fetchMutex(packageName, packageName, callback) + } + }) +} + +module.exports = { + getDaily: getDailyDownloads +} diff --git a/server/PackageInfo.js b/server/PackageInfo.js index 05b98b9..17419d6 100644 --- a/server/PackageInfo.js +++ b/server/PackageInfo.js @@ -1,9 +1,11 @@ require('isomorphic-fetch') -const PackageInfoCache = require('./PackageInfoCache') +const createCache = require('./createCache') const createMutex = require('./createMutex') const RegistryURL = process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org' +const PackageInfoCache = createCache('packageInfo') + function fetchPackageInfo(packageName) { console.log(`info: Fetching package info for ${packageName}`) @@ -18,8 +20,8 @@ function fetchPackageInfo(packageName) { return fetch(url, { headers: { 'Accept': 'application/json' } - }).then(function (response) { - return response.status === 404 ? null : response.json() + }).then(function (res) { + return res.status === 404 ? null : res.json() }) } @@ -53,10 +55,8 @@ const fetchMutex = createMutex(function (packageName, callback) { function getPackageInfo(packageName, callback) { PackageInfoCache.get(packageName, function (error, value) { - if (error) { - callback(error) - } else if (value) { - callback(null, value === PackageNotFound ? null : value) + if (error || value != null) { + callback(error, value === PackageNotFound ? null : value) } else { fetchMutex(packageName, packageName, callback) } diff --git a/server/PackageInfoCache.js b/server/PackageInfoCache.js deleted file mode 100644 index 3254e89..0000000 --- a/server/PackageInfoCache.js +++ /dev/null @@ -1,25 +0,0 @@ -const db = require('./RedisClient') - -function createKey(packageName) { - return 'packageInfo-' + packageName -} - -function set(packageName, value, expiry, callback) { - db.setex(createKey(packageName), expiry, JSON.stringify(value), callback) -} - -function get(packageName, callback) { - db.get(createKey(packageName), function (error, value) { - callback(error, value && JSON.parse(value)) - }) -} - -function del(packageName, callback) { - db.del(createKey(packageName), callback) -} - -module.exports = { - set, - get, - del -} diff --git a/server/createCache.js b/server/createCache.js new file mode 100644 index 0000000..805a8c2 --- /dev/null +++ b/server/createCache.js @@ -0,0 +1,29 @@ +const db = require('./RedisClient') + +function createCache(keyPrefix) { + function createKey(key) { + return keyPrefix + '-' + key + } + + function set(key, value, expiry, callback) { + db.setex(createKey(key), expiry, JSON.stringify(value), callback) + } + + function get(key, callback) { + db.get(createKey(key), function (error, value) { + callback(error, value && JSON.parse(value)) + }) + } + + function del(key, callback) { + db.del(createKey(key), callback) + } + + return { + set, + get, + del + } +} + +module.exports = createCache diff --git a/server/createServer.js b/server/createServer.js index 3c79dbe..8046ffb 100644 --- a/server/createServer.js +++ b/server/createServer.js @@ -6,6 +6,8 @@ const cors = require('cors') const morgan = require('morgan') const { fetchStats } = require('./cloudflare') + +const checkMinDailyDownloads = require('./middleware/checkMinDailyDownloads') const parsePackageURL = require('./middleware/parsePackageURL') const fetchFile = require('./middleware/fetchFile') const serveFile = require('./middleware/serveFile') @@ -67,8 +69,19 @@ function createServer() { maxAge: '365d' })) - app.use('/_meta', parsePackageURL, fetchFile, serveMetadata) - app.use('/', parsePackageURL, fetchFile, serveFile) + app.use('/_meta', + parsePackageURL, + checkMinDailyDownloads(100), + fetchFile, + serveMetadata + ) + + app.use('/', + parsePackageURL, + checkMinDailyDownloads(100), + fetchFile, + serveFile + ) const server = http.createServer(app) diff --git a/server/middleware/checkMinDailyDownloads.js b/server/middleware/checkMinDailyDownloads.js new file mode 100644 index 0000000..e30128d --- /dev/null +++ b/server/middleware/checkMinDailyDownloads.js @@ -0,0 +1,20 @@ +const NPMDownloads = require('../NPMDownloads') + +function checkMinDailyDownloads(minDailyDownloads) { + return function (req, res, next) { + NPMDownloads.getDaily(req.packageName, function (error, downloads) { + if (error) { + console.error(error) + next() // Keep going; this error isn't critical. + } else if (downloads == null) { + res.status(404).type('text').send(`Cannot find package "${req.packageName}"`) + } else if (downloads >= minDailyDownloads) { + next() + } else { + res.status(404).type('text').send(`Cannot serve requests for package "${req.packageName}" because it has been downloaded on average only ${downloads} time${downloads > 1 ? 's' : ''} per day this week`) + } + }) + } +} + +module.exports = checkMinDailyDownloads