Rename server => modules

This commit is contained in:
Michael Jackson
2018-07-31 10:13:26 -07:00
parent 135da0fdc5
commit bef8b2ebee
104 changed files with 13 additions and 13 deletions

17
modules/.eslintrc Normal file
View File

@ -0,0 +1,17 @@
{
"parser": "babel-eslint",
"env": {
"node": true
},
"extends": [
"eslint:recommended",
"plugin:import/errors"
],
"rules": {
"no-console": 0
},
"globals": {
"fetch": true,
"Promise": true
}
}

98
modules/AuthAPI.js Normal file
View File

@ -0,0 +1,98 @@
const crypto = require("crypto");
const jwt = require("jsonwebtoken");
const db = require("./utils/data");
const secretKey = require("./secretKey");
function getCurrentSeconds() {
return Math.floor(Date.now() / 1000);
}
function createTokenId() {
return crypto.randomBytes(16).toString("hex");
}
function createToken(scopes = {}) {
return new Promise((resolve, reject) => {
const payload = {
jti: createTokenId(),
iss: "https://unpkg.com",
iat: getCurrentSeconds(),
scopes
};
jwt.sign(
payload,
secretKey.private,
{ algorithm: "RS256" },
(error, token) => {
if (error) {
reject(error);
} else {
resolve(token);
}
}
);
});
}
const revokedTokensSet = "revoked-tokens";
function verifyToken(token) {
return new Promise((resolve, reject) => {
const options = { algorithms: ["RS256"] };
jwt.verify(token, secretKey.public, options, (error, payload) => {
if (error) {
reject(error);
} else {
if (payload.jti) {
db.sismember(revokedTokensSet, payload.jti, (error, value) => {
if (error) {
reject(error);
} else {
resolve(value === 0 ? payload : null);
}
});
} else {
resolve(null);
}
}
});
});
}
function revokeToken(token) {
return verifyToken(token).then(payload => {
if (payload) {
return new Promise((resolve, reject) => {
db.sadd(revokedTokensSet, payload.jti, error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
});
}
function removeAllRevokedTokens() {
return new Promise((resolve, reject) => {
db.del(revokedTokensSet, error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
module.exports = {
createToken,
verifyToken,
revokeToken,
removeAllRevokedTokens
};

71
modules/BlacklistAPI.js Normal file
View File

@ -0,0 +1,71 @@
const db = require("./utils/data");
const blacklistSet = "blacklisted-packages";
function addPackage(packageName) {
return new Promise((resolve, reject) => {
db.sadd(blacklistSet, packageName, (error, value) => {
if (error) {
reject(error);
} else {
resolve(value === 1);
}
});
});
}
function removePackage(packageName) {
return new Promise((resolve, reject) => {
db.srem(blacklistSet, packageName, (error, value) => {
if (error) {
reject(error);
} else {
resolve(value === 1);
}
});
});
}
function removeAllPackages() {
return new Promise((resolve, reject) => {
db.del(blacklistSet, error => {
if (error) {
reject(error);
} else {
resolve();
}
});
});
}
function getPackages() {
return new Promise((resolve, reject) => {
db.smembers(blacklistSet, (error, value) => {
if (error) {
reject(error);
} else {
resolve(value);
}
});
});
}
function includesPackage(packageName) {
return new Promise((resolve, reject) => {
db.sismember(blacklistSet, packageName, (error, value) => {
if (error) {
reject(error);
} else {
resolve(value === 1);
}
});
});
}
module.exports = {
addPackage,
removePackage,
removeAllPackages,
getPackages,
includesPackage
};

105
modules/CloudflareAPI.js Normal file
View File

@ -0,0 +1,105 @@
require("isomorphic-fetch");
const invariant = require("invariant");
const gunzip = require("gunzip-maybe");
const ndjson = require("ndjson");
const cloudflareURL = "https://api.cloudflare.com/client/v4";
const cloudflareEmail = process.env.CLOUDFLARE_EMAIL;
const cloudflareKey = process.env.CLOUDFLARE_KEY;
invariant(
cloudflareEmail,
"Missing the $CLOUDFLARE_EMAIL environment variable"
);
invariant(cloudflareKey, "Missing the $CLOUDFLARE_KEY environment variable");
function get(path, headers) {
return fetch(`${cloudflareURL}${path}`, {
headers: Object.assign({}, headers, {
"X-Auth-Email": cloudflareEmail,
"X-Auth-Key": cloudflareKey
})
});
}
function getJSON(path, headers) {
return get(path, headers)
.then(res => {
return res.json();
})
.then(data => {
if (!data.success) {
console.error(`CloudflareAPI.getJSON failed at ${path}`);
console.error(data);
throw new Error("Failed to getJSON from Cloudflare");
}
return data.result;
});
}
function getZones(domains) {
return Promise.all(
(Array.isArray(domains) ? domains : [domains]).map(domain =>
getJSON(`/zones?name=${domain}`)
)
).then(results => results.reduce((memo, zones) => memo.concat(zones)));
}
function reduceResults(target, values) {
Object.keys(values).forEach(key => {
const value = values[key];
if (typeof value === "object" && value) {
target[key] = reduceResults(target[key] || {}, value);
} else if (typeof value === "number") {
target[key] = (target[key] || 0) + values[key];
}
});
return target;
}
function getZoneAnalyticsDashboard(zones, since, until) {
return Promise.all(
(Array.isArray(zones) ? zones : [zones]).map(zone => {
return getJSON(
`/zones/${
zone.id
}/analytics/dashboard?since=${since.toISOString()}&until=${until.toISOString()}`
);
})
).then(results => results.reduce(reduceResults));
}
function getJSONStream(path, headers) {
const gzipHeaders = Object.assign({}, headers, {
"Accept-Encoding": "gzip"
});
return get(path, gzipHeaders)
.then(res => res.body.pipe(gunzip()))
.then(stream => stream.pipe(ndjson.parse()));
}
function getLogs(zoneId, startTime, endTime, fieldsArray) {
const fields = fieldsArray.join(",");
// console.log(
// `https://api.cloudflare.com/client/v4/zones/${zoneId}/logs/received?start=${startTime}&end=${endTime}&fields=${fields}`
// );
return getJSONStream(
`/zones/${zoneId}/logs/received?start=${startTime}&end=${endTime}&fields=${fields}`
);
}
module.exports = {
get,
getJSON,
getZones,
getZoneAnalyticsDashboard,
getJSONStream,
getLogs
};

173
modules/StatsAPI.js Normal file
View File

@ -0,0 +1,173 @@
const db = require("./utils/data");
const CloudflareAPI = require("./CloudflareAPI");
const BlacklistAPI = require("./BlacklistAPI");
function prunePackages(packagesMap) {
return Promise.all(
Object.keys(packagesMap).map(packageName =>
BlacklistAPI.includesPackage(packageName).then(blacklisted => {
if (blacklisted) {
delete packagesMap[packageName];
}
})
)
).then(() => packagesMap);
}
function createDayKey(date) {
return `${date.getUTCFullYear()}-${date.getUTCMonth()}-${date.getUTCDate()}`;
}
function createHourKey(date) {
return `${createDayKey(date)}-${date.getUTCHours()}`;
}
function createMinuteKey(date) {
return `${createHourKey(date)}-${date.getUTCMinutes()}`;
}
function createScoresMap(array) {
const map = {};
for (let i = 0; i < array.length; i += 2) {
map[array[i]] = parseInt(array[i + 1], 10);
}
return map;
}
function getScoresMap(key, n = 100) {
return new Promise((resolve, reject) => {
db.zrevrange(key, 0, n, "withscores", (error, value) => {
if (error) {
reject(error);
} else {
resolve(createScoresMap(value));
}
});
});
}
function getPackageRequests(date, n = 100) {
return getScoresMap(`stats-packageRequests-${createDayKey(date)}`, n).then(
prunePackages
);
}
function getPackageBandwidth(date, n = 100) {
return getScoresMap(`stats-packageBytes-${createDayKey(date)}`, n).then(
prunePackages
);
}
function getProtocolRequests(date) {
return getScoresMap(`stats-protocolRequests-${createDayKey(date)}`);
}
function addDailyMetricsToTimeseries(timeseries) {
const since = new Date(timeseries.since);
return Promise.all([
getPackageRequests(since),
getPackageBandwidth(since),
getProtocolRequests(since)
]).then(results => {
timeseries.requests.package = results[0];
timeseries.bandwidth.package = results[1];
timeseries.requests.protocol = results[2];
return timeseries;
});
}
function sumMaps(maps) {
return maps.reduce((memo, map) => {
Object.keys(map).forEach(key => {
memo[key] = (memo[key] || 0) + map[key];
});
return memo;
}, {});
}
function addDailyMetrics(result) {
return Promise.all(result.timeseries.map(addDailyMetricsToTimeseries)).then(
() => {
result.totals.requests.package = sumMaps(
result.timeseries.map(timeseries => {
return timeseries.requests.package;
})
);
result.totals.bandwidth.package = sumMaps(
result.timeseries.map(timeseries => timeseries.bandwidth.package)
);
result.totals.requests.protocol = sumMaps(
result.timeseries.map(timeseries => timeseries.requests.protocol)
);
return result;
}
);
}
function extractPublicInfo(data) {
return {
since: data.since,
until: data.until,
requests: {
all: data.requests.all,
cached: data.requests.cached,
country: data.requests.country,
status: data.requests.http_status
},
bandwidth: {
all: data.bandwidth.all,
cached: data.bandwidth.cached,
country: data.bandwidth.country
},
threats: {
all: data.threats.all,
country: data.threats.country
},
uniques: {
all: data.uniques.all
}
};
}
const DomainNames = ["unpkg.com", "npmcdn.com"];
function fetchStats(since, until) {
return CloudflareAPI.getZones(DomainNames).then(zones => {
return CloudflareAPI.getZoneAnalyticsDashboard(zones, since, until).then(
dashboard => {
return {
timeseries: dashboard.timeseries.map(extractPublicInfo),
totals: extractPublicInfo(dashboard.totals)
};
}
);
});
}
const oneMinute = 1000 * 60;
const oneHour = oneMinute * 60;
const oneDay = oneHour * 24;
function getStats(since, until) {
const promise = fetchStats(since, until);
return until - since > oneDay ? promise.then(addDailyMetrics) : promise;
}
module.exports = {
createDayKey,
createHourKey,
createMinuteKey,
getStats
};

View File

@ -0,0 +1,5 @@
{
"env": {
"jest": true
}
}

View File

@ -0,0 +1,39 @@
const AuthAPI = require("../AuthAPI");
describe("Auth API", () => {
beforeEach(done => {
AuthAPI.removeAllRevokedTokens().then(() => done(), done);
});
it("creates tokens with the right scopes", done => {
const scopes = {
blacklist: {
add: true,
remove: true
}
};
AuthAPI.createToken(scopes).then(token => {
AuthAPI.verifyToken(token).then(payload => {
expect(payload.jti).toEqual(expect.any(String));
expect(payload.iss).toEqual(expect.any(String));
expect(payload.iat).toEqual(expect.any(Number));
expect(payload.scopes).toMatchObject(scopes);
done();
});
});
});
it("refuses to verify revoked tokens", done => {
const scopes = {};
AuthAPI.createToken(scopes).then(token => {
AuthAPI.revokeToken(token).then(() => {
AuthAPI.verifyToken(token).then(payload => {
expect(payload).toBe(null);
done();
});
});
});
});
});

View File

@ -0,0 +1,24 @@
const BlacklistAPI = require("../BlacklistAPI");
describe("Blacklist API", () => {
beforeEach(done => {
BlacklistAPI.removeAllPackages().then(() => done(), done);
});
it("adds and removes packages to/from the blacklist", done => {
const packageName = "bad-package";
BlacklistAPI.addPackage(packageName).then(() => {
BlacklistAPI.getPackages().then(packageNames => {
expect(packageNames).toEqual([packageName]);
BlacklistAPI.removePackage(packageName).then(() => {
BlacklistAPI.getPackages().then(packageNames => {
expect(packageNames).toEqual([]);
done();
});
});
});
});
});
});

View File

@ -0,0 +1,229 @@
const request = require("supertest");
const createServer = require("../createServer");
const clearBlacklist = require("./utils/clearBlacklist");
const withBlacklist = require("./utils/withBlacklist");
const withRevokedToken = require("./utils/withRevokedToken");
const withToken = require("./utils/withToken");
describe("The production server", () => {
let server;
beforeEach(() => {
server = createServer();
});
it("redirects /_meta to ?meta", done => {
request(server)
.get("/_meta/react")
.end((err, res) => {
expect(res.statusCode).toBe(301);
expect(res.headers.location).toBe("/react?meta");
done();
});
});
it("redirects ?json to ?meta", done => {
request(server)
.get("/react?json")
.end((err, res) => {
expect(res.statusCode).toBe(301);
expect(res.headers.location).toBe("/react?meta");
done();
});
});
it("redirects invalid query params", done => {
request(server)
.get("/react?main=index&invalid")
.end((err, res) => {
expect(res.statusCode).toBe(302);
expect(res.headers.location).toBe("/react?main=index");
done();
});
});
it("rejects invalid package names", done => {
request(server)
.get("/_invalid/index.js")
.end((err, res) => {
expect(res.statusCode).toBe(403);
done();
});
});
it("does not serve blacklisted packages", done => {
withBlacklist(["bad-package"], () => {
request(server)
.get("/bad-package/index.js")
.end((err, res) => {
expect(res.statusCode).toBe(403);
done();
});
});
});
describe("GET /_publicKey", () => {
it("echoes the public key", done => {
request(server)
.get("/_publicKey")
.end((err, res) => {
expect(res.text).toMatch(/PUBLIC KEY/);
done();
});
});
});
describe("POST /_auth", () => {
it("creates a new auth token", done => {
request(server)
.post("/_auth")
.end((err, res) => {
expect(res.body).toHaveProperty("token");
done();
});
});
});
describe("GET /_auth", () => {
describe("with no auth", () => {
it("echoes back null", done => {
request(server)
.get("/_auth")
.end((err, res) => {
expect(res.body).toHaveProperty("auth");
expect(res.body.auth).toBe(null);
done();
});
});
});
describe("with a revoked auth token", () => {
it("echoes back null", done => {
withRevokedToken({ some: { scope: true } }, token => {
request(server)
.get("/_auth?token=" + token)
.end((err, res) => {
expect(res.body).toHaveProperty("auth");
expect(res.body.auth).toBe(null);
done();
});
});
});
});
describe("with a valid auth token", () => {
it("echoes back the auth payload", done => {
withToken({ some: { scope: true } }, token => {
request(server)
.get("/_auth?token=" + token)
.end((err, res) => {
expect(res.body).toHaveProperty("auth");
expect(typeof res.body.auth).toBe("object");
done();
});
});
});
});
});
describe("POST /_blacklist", () => {
afterEach(clearBlacklist);
describe("with no auth", () => {
it("is forbidden", done => {
request(server)
.post("/_blacklist")
.end((err, res) => {
expect(res.statusCode).toBe(403);
done();
});
});
});
describe('with the "blacklist.add" scope', () => {
it("can add to the blacklist", done => {
withToken({ blacklist: { add: true } }, token => {
request(server)
.post("/_blacklist")
.send({ token, packageName: "bad-package" })
.end((err, res) => {
expect(res.statusCode).toBe(200);
expect(res.headers["content-location"]).toEqual(
"/_blacklist/bad-package"
);
expect(res.body.ok).toBe(true);
done();
});
});
});
});
});
describe("GET /_blacklist", () => {
describe("with no auth", () => {
it("is forbidden", done => {
request(server)
.get("/_blacklist")
.end((err, res) => {
expect(res.statusCode).toBe(403);
done();
});
});
});
describe('with the "blacklist.read" scope', () => {
it("can read the blacklist", done => {
withToken({ blacklist: { read: true } }, token => {
request(server)
.get("/_blacklist?token=" + token)
.end((err, res) => {
expect(res.statusCode).toBe(200);
done();
});
});
});
});
});
describe("DELETE /_blacklist/:packageName", () => {
describe("with no auth", () => {
it("is forbidden", done => {
request(server)
.delete("/_blacklist/bad-package")
.end((err, res) => {
expect(res.statusCode).toBe(403);
done();
});
});
});
describe('with the "blacklist.remove" scope', () => {
it("can remove a package from the blacklist", done => {
withToken({ blacklist: { remove: true } }, token => {
request(server)
.delete("/_blacklist/bad-package")
.send({ token })
.end((err, res) => {
expect(res.statusCode).toBe(200);
expect(res.body.ok).toBe(true);
done();
});
});
});
it("can remove a scoped package from the blacklist", done => {
withToken({ blacklist: { remove: true } }, token => {
request(server)
.delete("/_blacklist/@scope/bad-package")
.send({ token })
.end((err, res) => {
expect(res.statusCode).toBe(200);
expect(res.body.ok).toBe(true);
done();
});
});
});
});
});
});

View File

@ -0,0 +1,7 @@
const BlacklistAPI = require("../../BlacklistAPI");
function clearBlacklist(done) {
BlacklistAPI.removeAllPackages().then(done, done);
}
module.exports = clearBlacklist;

View File

@ -0,0 +1,7 @@
const BlacklistAPI = require("../../BlacklistAPI");
function withBlacklist(blacklist, callback) {
return Promise.all(blacklist.map(BlacklistAPI.addPackage)).then(callback);
}
module.exports = withBlacklist;

View File

@ -0,0 +1,12 @@
const withToken = require("./withToken");
const AuthAPI = require("../../AuthAPI");
function withRevokedToken(scopes, callback) {
withToken(scopes, token => {
AuthAPI.revokeToken(token).then(() => {
callback(token);
});
});
}
module.exports = withRevokedToken;

View File

@ -0,0 +1,7 @@
const AuthAPI = require("../../AuthAPI");
function withToken(scopes, callback) {
AuthAPI.createToken(scopes).then(callback);
}
module.exports = withToken;

View File

@ -0,0 +1,48 @@
const validateNpmPackageName = require("validate-npm-package-name");
const BlacklistAPI = require("../BlacklistAPI");
function addToBlacklist(req, res) {
const packageName = req.body.packageName;
if (!packageName) {
return res
.status(403)
.send({ error: 'Missing "packageName" body parameter' });
}
const nameErrors = validateNpmPackageName(packageName).errors;
// Disallow invalid package names.
if (nameErrors) {
const reason = nameErrors.join(", ");
return res.status(403).send({
error: `Invalid package name "${packageName}" (${reason})`
});
}
BlacklistAPI.addPackage(packageName).then(
added => {
if (added) {
const userId = req.user.jti;
console.log(
`Package "${packageName}" was added to the blacklist by ${userId}`
);
}
res.set({ "Content-Location": `/_blacklist/${packageName}` }).send({
ok: true,
message: `Package "${packageName}" was ${
added ? "added to" : "already in"
} the blacklist`
});
},
error => {
console.error(error);
res.status(500).send({
error: `Unable to add "${packageName}" to the blacklist`
});
}
);
}
module.exports = addToBlacklist;

View File

@ -0,0 +1,24 @@
const AuthAPI = require("../AuthAPI");
const defaultScopes = {
blacklist: {
read: true
}
};
function createAuth(req, res) {
AuthAPI.createToken(defaultScopes).then(
token => {
res.send({ token });
},
error => {
console.error(error);
res.status(500).send({
error: "Unable to generate auth token"
});
}
);
}
module.exports = createAuth;

View File

@ -0,0 +1,32 @@
const BlacklistAPI = require("../BlacklistAPI");
function removeFromBlacklist(req, res) {
const packageName = req.packageName;
BlacklistAPI.removePackage(packageName).then(
removed => {
if (removed) {
const userId = req.user.jti;
console.log(
`Package "${packageName}" was removed from the blacklist by ${userId}`
);
}
res.send({
ok: true,
message: `Package "${packageName}" was ${
removed ? "removed from" : "not in"
} the blacklist`
});
},
error => {
console.error(error);
res.status(500).send({
error: `Unable to remove "${packageName}" from the blacklist`
});
}
);
}
module.exports = removeFromBlacklist;

View File

@ -0,0 +1,57 @@
const React = require("react");
const ReactDOMServer = require("react-dom/server");
const semver = require("semver");
const MainPage = require("../client/MainPage");
const AutoIndexApp = require("../client/autoIndex/App");
const renderPage = require("../utils/renderPage");
const globalScripts =
process.env.NODE_ENV === "production"
? [
"/react@16.4.1/umd/react.production.min.js",
"/react-dom@16.4.1/umd/react-dom.production.min.js",
"/react-router-dom@4.3.1/umd/react-router-dom.min.js"
]
: [
"/react@16.4.1/umd/react.development.js",
"/react-dom@16.4.1/umd/react-dom.development.js",
"/react-router-dom@4.3.1/umd/react-router-dom.js"
];
function byVersion(a, b) {
return semver.lt(a, b) ? -1 : semver.gt(a, b) ? 1 : 0;
}
function serveAutoIndexPage(req, res) {
const scripts = globalScripts.concat(req.assets.getScripts("autoIndex"));
const styles = req.assets.getStyles("autoIndex");
const props = {
packageName: req.packageName,
packageVersion: req.packageVersion,
availableVersions: Object.keys(req.packageInfo.versions).sort(byVersion),
filename: req.filename,
entry: req.entry,
entries: req.entries
};
const content = ReactDOMServer.renderToString(
React.createElement(AutoIndexApp, props)
);
const html = renderPage(MainPage, {
scripts: scripts,
styles: styles,
data: props,
content: content
});
res
.set({
"Cache-Control": "public,max-age=60", // 1 minute
"Cache-Tag": "auto-index"
})
.send(html);
}
module.exports = serveAutoIndexPage;

View File

@ -0,0 +1,25 @@
const serveAutoIndexPage = require("./serveAutoIndexPage");
const serveJavaScriptModule = require("./serveJavaScriptModule");
const serveStaticFile = require("./serveStaticFile");
const serveMetadata = require("./serveMetadata");
/**
* Send the file, JSON metadata, or HTML directory listing.
*/
function serveFile(req, res) {
if (req.query.meta != null) {
return serveMetadata(req, res);
}
if (req.entry.type === "directory") {
return serveAutoIndexPage(req, res);
}
if (req.query.module != null) {
return serveJavaScriptModule(req, res);
}
serveStaticFile(req, res);
}
module.exports = serveFile;

View File

@ -0,0 +1,70 @@
const etag = require("etag");
const babel = require("babel-core");
const getContentTypeHeader = require("../utils/getContentTypeHeader");
const unpkgRewrite = require("../plugins/unpkgRewrite");
function rewriteBareModuleIdentifiers(code, packageConfig) {
const dependencies = Object.assign(
{},
packageConfig.peerDependencies,
packageConfig.dependencies
);
const options = {
// Ignore .babelrc and package.json babel config
// because we haven't installed dependencies so
// we can't load plugins; see #84
babelrc: false,
plugins: [unpkgRewrite(dependencies)]
};
return babel.transform(code, options).code;
}
function serveJavaScriptModule(req, res) {
if (req.entry.contentType !== "application/javascript") {
return res
.status(403)
.type("text")
.send("?module mode is available only for JavaScript files");
}
try {
const code = rewriteBareModuleIdentifiers(
req.entry.content.toString("utf8"),
req.packageConfig
);
res
.set({
"Content-Length": Buffer.byteLength(code),
"Content-Type": getContentTypeHeader(req.entry.contentType),
"Cache-Control": "public,max-age=31536000", // 1 year
ETag: etag(code),
"Cache-Tag": "file,js-file,js-module"
})
.send(code);
} catch (error) {
console.error(error);
const errorName = error.constructor.name;
const errorMessage = error.message.replace(
/^.*?\/unpkg-.+?\//,
`/${req.packageSpec}/`
);
const codeFrame = error.codeFrame;
const debugInfo = `${errorName}: ${errorMessage}\n\n${codeFrame}`;
res
.status(500)
.type("text")
.send(
`Cannot generate module for ${req.packageSpec}${
req.filename
}\n\n${debugInfo}`
);
}
}
module.exports = serveJavaScriptModule;

View File

@ -0,0 +1,29 @@
const MainPage = require("../client/MainPage");
const renderPage = require("../utils/renderPage");
const globalScripts =
process.env.NODE_ENV === "production"
? [
"/react@16.4.1/umd/react.production.min.js",
"/react-dom@16.4.1/umd/react-dom.production.min.js",
"/react-router-dom@4.3.1/umd/react-router-dom.min.js"
]
: [
"/react@16.4.1/umd/react.development.js",
"/react-dom@16.4.1/umd/react-dom.development.js",
"/react-router-dom@4.3.1/umd/react-router-dom.js"
];
function serveMainPage(req, res) {
const scripts = globalScripts.concat(req.assets.getScripts("main"));
const styles = req.assets.getStyles("main");
const html = renderPage(MainPage, {
scripts: scripts,
styles: styles
});
res.send(html);
}
module.exports = serveMainPage;

View File

@ -0,0 +1,44 @@
const path = require("path");
const addLeadingSlash = require("../utils/addLeadingSlash");
function getMatchingEntries(entry, entries) {
const dirname = entry.name || ".";
return Object.keys(entries)
.filter(name => entry.name !== name && path.dirname(name) === dirname)
.map(name => entries[name]);
}
function getMetadata(entry, entries) {
const metadata = {
path: addLeadingSlash(entry.name),
type: entry.type
};
if (entry.type === "file") {
metadata.contentType = entry.contentType;
metadata.integrity = entry.integrity;
metadata.lastModified = entry.lastModified;
metadata.size = entry.size;
} else if (entry.type === "directory") {
metadata.files = getMatchingEntries(entry, entries).map(e =>
getMetadata(e, entries)
);
}
return metadata;
}
function serveMetadata(req, res) {
const metadata = getMetadata(req.entry, req.entries);
res
.set({
"Cache-Control": "public,max-age=31536000", // 1 year
"Cache-Tag": "meta"
})
.send(metadata);
}
module.exports = serveMetadata;

View File

@ -0,0 +1,26 @@
const path = require("path");
const etag = require("etag");
const getContentTypeHeader = require("../utils/getContentTypeHeader");
function serveStaticFile(req, res) {
const tags = ["file"];
const ext = path.extname(req.entry.name).substr(1);
if (ext) {
tags.push(`${ext}-file`);
}
res
.set({
"Content-Length": req.entry.size,
"Content-Type": getContentTypeHeader(req.entry.contentType),
"Cache-Control": "public,max-age=31536000", // 1 year
"Last-Modified": req.entry.lastModified,
ETag: etag(req.entry.content),
"Cache-Tag": tags.join(",")
})
.send(req.entry.content);
}
module.exports = serveStaticFile;

View File

@ -0,0 +1,5 @@
function showAuth(req, res) {
res.send({ auth: req.user });
}
module.exports = showAuth;

View File

@ -0,0 +1,17 @@
const BlacklistAPI = require("../BlacklistAPI");
function showBlacklist(req, res) {
BlacklistAPI.getPackages().then(
blacklist => {
res.send({ blacklist });
},
error => {
console.error(error);
res.status(500).send({
error: "Unable to fetch blacklist"
});
}
);
}
module.exports = showBlacklist;

View File

@ -0,0 +1,7 @@
const secretKey = require("../secretKey");
function showPublicKey(req, res) {
res.send({ publicKey: secretKey.public });
}
module.exports = showPublicKey;

View File

@ -0,0 +1,63 @@
const subDays = require("date-fns/sub_days");
const startOfDay = require("date-fns/start_of_day");
const startOfSecond = require("date-fns/start_of_second");
const StatsAPI = require("../StatsAPI");
function showStats(req, res) {
let since, until;
switch (req.query.period) {
case "last-day":
until = startOfDay(new Date());
since = subDays(until, 1);
break;
case "last-week":
until = startOfDay(new Date());
since = subDays(until, 7);
break;
case "last-month":
until = startOfDay(new Date());
since = subDays(until, 30);
break;
default:
until = req.query.until
? new Date(req.query.until)
: startOfSecond(new Date());
since = new Date(req.query.since);
}
if (isNaN(since.getTime())) {
return res.status(403).send({ error: "?since is not a valid date" });
}
if (isNaN(until.getTime())) {
return res.status(403).send({ error: "?until is not a valid date" });
}
if (until <= since) {
return res
.status(403)
.send({ error: "?until date must come after ?since date" });
}
if (until >= new Date()) {
return res.status(403).send({ error: "?until must be a date in the past" });
}
StatsAPI.getStats(since, until).then(
stats => {
res
.set({
"Cache-Control": "public, max-age=60",
"Cache-Tag": "stats"
})
.send(stats);
},
error => {
console.error(error);
res.status(500).send({ error: "Unable to fetch stats" });
}
);
}
module.exports = showStats;

8
modules/client/.babelrc Normal file
View File

@ -0,0 +1,8 @@
{
"presets": ["env", "stage-2", "react"],
"env": {
"production": {
"plugins": ["transform-react-remove-prop-types"]
}
}
}

14
modules/client/.eslintrc Normal file
View File

@ -0,0 +1,14 @@
{
"parser": "babel-eslint",
"env": {
"browser": true
},
"extends": [
"eslint:recommended",
"plugin:import/errors",
"plugin:react/recommended"
],
"rules": {
"react/no-children-prop": 0
}
}

View File

@ -0,0 +1,56 @@
const React = require("react");
const PropTypes = require("prop-types");
const h = require("./utils/createHTML");
const x = require("./utils/execScript");
function MainPage({ title, description, scripts, styles, data, content }) {
return (
<html lang="en">
<head>
<meta charSet="utf-8" />
<meta name="description" content={description} />
<meta httpEquiv="X-UA-Compatible" content="IE=edge,chrome=1" />
<meta
name="viewport"
content="width=device-width,initial-scale=1,maximum-scale=1"
/>
<meta name="timestamp" content={new Date().toISOString()} />
<link rel="shortcut icon" href="/favicon.ico" />
{styles.map(s => <link key={s} rel="stylesheet" href={s} />)}
{x(
"window.Promise || document.write('\\x3Cscript src=\"/_polyfills/es6-promise.min.js\">\\x3C/script>\\x3Cscript>ES6Promise.polyfill()\\x3C/script>')"
)}
{x(
"window.fetch || document.write('\\x3Cscript src=\"/_polyfills/fetch.min.js\">\\x3C/script>')"
)}
{x(`window.__DATA__ = ${JSON.stringify(data)}`)}
<title>{title}</title>
</head>
<body>
<div id="root" dangerouslySetInnerHTML={h(content)} />
{scripts.map(s => <script key={s} src={s} />)}
</body>
</html>
);
}
MainPage.propTypes = {
title: PropTypes.string,
description: PropTypes.string,
scripts: PropTypes.arrayOf(PropTypes.string),
styles: PropTypes.arrayOf(PropTypes.string),
data: PropTypes.any,
content: PropTypes.string
};
MainPage.defaultProps = {
title: "UNPKG",
description: "The CDN for everything on npm",
scripts: [],
styles: [],
data: {},
content: ""
};
module.exports = MainPage;

View File

@ -0,0 +1,8 @@
body {
font-size: 14px;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica,
Arial, sans-serif;
line-height: 1.7;
padding: 0px 10px 5px;
color: #000000;
}

View File

@ -0,0 +1,10 @@
require("./autoIndex.css");
const React = require("react");
const ReactDOM = require("react-dom");
const App = require("./autoIndex/App");
const props = window.__DATA__ || {};
ReactDOM.hydrate(<App {...props} />, document.getElementById("root"));

View File

@ -0,0 +1,23 @@
.app {
max-width: 900px;
margin: 0 auto;
}
.app-header {
display: flex;
flex-direction: row;
align-items: center;
justify-content: space-between;
}
.app-version-selector {
line-height: 2.25em;
float: right;
}
.app-version-selector select {
font-size: 1em;
}
.app-address {
text-align: right;
}

View File

@ -0,0 +1,77 @@
require("./App.css");
const React = require("react");
const DirectoryListing = require("./DirectoryListing");
class App extends React.Component {
static defaultProps = {
availableVersions: []
};
handleChange = event => {
window.location.href = window.location.href.replace(
"@" + this.props.packageVersion,
"@" + event.target.value
);
};
render() {
return (
<div className="app">
<header className="app-header">
<h1>
Index of /{this.props.packageName}@{this.props.packageVersion}
{this.props.filename}
</h1>
<div className="app-version-selector">
Version:{" "}
<select
id="version"
defaultValue={this.props.packageVersion}
onChange={this.handleChange}
>
{this.props.availableVersions.map(v => (
<option key={v} value={v}>
{v}
</option>
))}
</select>
</div>
</header>
<hr />
<DirectoryListing
filename={this.props.filename}
entry={this.props.entry}
entries={this.props.entries}
/>
<hr />
<address className="app-address">
{this.props.packageName}@{this.props.packageVersion}
</address>
</div>
);
}
}
if (process.env.NODE_ENV === "development") {
const PropTypes = require("prop-types");
const entryType = PropTypes.object;
App.propTypes = {
packageName: PropTypes.string.isRequired,
packageVersion: PropTypes.string.isRequired,
availableVersions: PropTypes.arrayOf(PropTypes.string),
filename: PropTypes.string.isRequired,
entry: entryType.isRequired,
entries: PropTypes.objectOf(entryType).isRequired
};
}
module.exports = App;

View File

@ -0,0 +1,17 @@
.directory-listing table {
width: 100%;
border-collapse: collapse;
font: 0.85em Monaco, monospace;
}
.directory-listing tr.even {
background-color: #eee;
}
.directory-listing th {
text-align: left;
}
.directory-listing th,
.directory-listing td {
padding: 0.5em 1em;
}

View File

@ -0,0 +1,126 @@
require("./DirectoryListing.css");
const React = require("react");
const formatBytes = require("pretty-bytes");
const sortBy = require("sort-by");
function getDirname(name) {
return (
name
.split("/")
.slice(0, -1)
.join("/") || "."
);
}
function getMatchingEntries(entry, entries) {
const dirname = entry.name || ".";
return Object.keys(entries)
.filter(name => entry.name !== name && getDirname(name) === dirname)
.map(name => entries[name]);
}
function getRelativeName(base, name) {
return base.length ? name.substr(base.length + 1) : name;
}
function DirectoryListing({ filename, entry, entries }) {
const rows = [];
if (filename !== "/") {
rows.push(
<tr key="..">
<td>
<a title="Parent directory" href="../">
..
</a>
</td>
<td>-</td>
<td>-</td>
<td>-</td>
</tr>
);
}
const matchingEntries = getMatchingEntries(entry, entries);
matchingEntries
.filter(({ type }) => type === "directory")
.sort(sortBy("name"))
.forEach(({ name }) => {
const relName = getRelativeName(entry.name, name);
const href = relName + "/";
rows.push(
<tr key={name}>
<td>
<a title={relName} href={href}>
{href}
</a>
</td>
<td>-</td>
<td>-</td>
<td>-</td>
</tr>
);
});
matchingEntries
.filter(({ type }) => type === "file")
.sort(sortBy("name"))
.forEach(({ name, size, contentType, lastModified }) => {
const relName = getRelativeName(entry.name, name);
rows.push(
<tr key={name}>
<td>
<a title={relName} href={relName}>
{relName}
</a>
</td>
<td>{contentType}</td>
<td>{formatBytes(size)}</td>
<td>{lastModified}</td>
</tr>
);
});
return (
<div className="directory-listing">
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Size</th>
<th>Last Modified</th>
</tr>
</thead>
<tbody>
{rows.map((row, index) =>
React.cloneElement(row, {
className: index % 2 ? "odd" : "even"
})
)}
</tbody>
</table>
</div>
);
}
if (process.env.NODE_ENV === "development") {
const PropTypes = require("prop-types");
const entryType = PropTypes.shape({
name: PropTypes.string.isRequired
});
DirectoryListing.propTypes = {
filename: PropTypes.string.isRequired,
entry: entryType.isRequired,
entries: PropTypes.objectOf(entryType).isRequired
};
}
module.exports = DirectoryListing;

63
modules/client/main.css Normal file
View File

@ -0,0 +1,63 @@
body {
font-size: 14px;
font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica,
Arial, sans-serif;
line-height: 1.7;
padding: 5px 20px;
color: #000000;
}
@media (min-width: 800px) {
body {
padding: 40px 20px 120px;
}
}
a:link {
color: blue;
}
a:visited {
color: rebeccapurple;
}
h1 {
font-size: 2em;
}
h2 {
font-size: 1.8em;
}
h3 {
font-size: 1.6em;
}
ul {
padding-left: 25px;
}
dd {
margin-left: 25px;
}
table {
border: 1px solid black;
border: 0;
}
th {
text-align: left;
background-color: #eee;
}
th,
td {
padding: 5px;
}
th {
vertical-align: bottom;
}
td {
vertical-align: top;
}
.wrapper {
max-width: 700px;
margin: 0 auto;
}

8
modules/client/main.js Normal file
View File

@ -0,0 +1,8 @@
require("./main.css");
const React = require("react");
const ReactDOM = require("react-dom");
const App = require("./main/App");
ReactDOM.render(<App />, document.getElementById("root"));

View File

@ -0,0 +1,13 @@
.about-logos {
margin: 2em 0;
display: flex;
justify-content: center;
}
.about-logo {
text-align: center;
flex: 1;
max-width: 80%;
}
.about-logo img {
max-width: 60%;
}

View File

@ -0,0 +1,12 @@
require("./About.css");
const React = require("react");
const h = require("../utils/createHTML");
const markup = require("./About.md");
function About() {
return <div className="wrapper" dangerouslySetInnerHTML={h(markup)} />;
}
module.exports = About;

View File

@ -0,0 +1,40 @@
unpkg is an [open source](https://github.com/unpkg) project built and maintained by [Michael Jackson](https://twitter.com/mjackson).
### Sponsors
The fast, global infrastructure that powers unpkg is generously donated by [Cloudflare](https://www.cloudflare.com) and [Heroku](https://www.heroku.com).
<div class="about-logos">
<div class="about-logo">
<a href="https://www.cloudflare.com"><img src="CloudflareLogo.png"></a>
</div>
<div class="about-logo">
<a href="https://www.heroku.com"><img src="HerokuLogo.png"></a>
</div>
</div>
### Cache Behavior
The CDN caches files based on their permanent URL, which includes the npm package version. This works because npm does not allow package authors to overwrite a package that has already been published with a different one at the same version number.
URLs that do not specify a package version number redirect to one that does. This is the `latest` version when no version is specified, or the `maxSatisfying` version when a [semver version](https://github.com/npm/node-semver) is given. Redirects are cached for 5 minutes.
Browsers are instructed (via the `Cache-Control` header) to cache assets for 4 hours.
### Support
unpkg is a free, best-effort service and cannot provide any uptime or support guarantees.
I do my best to keep it running, but sometimes things go wrong. Sometimes there are network or provider issues outside my control. Sometimes abusive traffic temporarily affects response times. Sometimes I break things by doing something dumb, but I try not to.
The goal of unpkg is to provide a hassle-free CDN for npm package authors. It's also a great resource for people creating demos and instructional material. However, if you rely on it to serve files that are crucial to your business, you should probably pay for a host with well-supported infrastructure and uptime guarantees.
unpkg is not affiliated with or supported by npm, Inc. in any way. Please do not contact npm for help with unpkg.
### Abuse
unpkg maintains a list of packages that are known to be malicious. If you find such a package on npm, please let us know!
### Feedback
If you think this is useful, we'd love to hear from you. Please reach out to [@unpkg](https://twitter.com/unpkg) with any questions or concerns.

View File

@ -0,0 +1,14 @@
const React = require("react");
const { HashRouter } = require("react-router-dom");
const Layout = require("./Layout");
function App() {
return (
<HashRouter>
<Layout />
</HashRouter>
);
}
module.exports = App;

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.1 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 13 KiB

View File

@ -0,0 +1,6 @@
.home-example {
text-align: center;
background-color: #eee;
margin: 2em 0;
padding: 5px 0;
}

View File

@ -0,0 +1,12 @@
require("./Home.css");
const React = require("react");
const h = require("../utils/createHTML");
const markup = require("./Home.md");
function Home() {
return <div className="wrapper" dangerouslySetInnerHTML={h(markup)} />;
}
module.exports = Home;

View File

@ -0,0 +1,48 @@
unpkg is a fast, global [content delivery network](https://en.wikipedia.org/wiki/Content_delivery_network) for everything on [npm](https://www.npmjs.com/). Use it to quickly and easily load any file from any package using a URL like:
<div class="home-example">unpkg.com/:package@:version/:file</div>
### Examples
Using a fixed version:
* [unpkg.com/react@16.0.0/umd/react.production.min.js](/react@16.0.0/umd/react.production.min.js)
* [unpkg.com/react-dom@16.0.0/umd/react-dom.production.min.js](/react-dom@16.0.0/umd/react-dom.production.min.js)
You may also use a [semver range](https://docs.npmjs.com/misc/semver) or a [tag](https://docs.npmjs.com/cli/dist-tag) instead of a fixed version number, or omit the version/tag entirely to use the `latest` tag.
* [unpkg.com/react@^16/umd/react.production.min.js](/react@^16/umd/react.production.min.js)
* [unpkg.com/react/umd/react.production.min.js](/react/umd/react.production.min.js)
If you omit the file path (i.e. use a "bare" URL), unpkg will serve the file specified by the `unpkg` field in `package.json`, or fall back to `main`.
* [unpkg.com/d3](/d3)
* [unpkg.com/jquery](/jquery)
* [unpkg.com/three](/three)
Append a `/` at the end of a URL to view a listing of all the files in a package.
* [unpkg.com/react/](/react/)
* [unpkg.com/lodash/](/lodash/)
### Query Parameters
<dl>
<dt>`?meta`</dt>
<dd>Return metadata about any file in a package as JSON (e.g. `/any/file?meta`)</dd>
<dt>`?module`</dt>
<dd>Expands all ["bare" `import` specifiers](https://html.spec.whatwg.org/multipage/webappapis.html#resolve-a-module-specifier) in JavaScript modules to unpkg URLs. This feature is *very experimental*</dd>
</dl>
### Workflow
For npm package authors, unpkg relieves the burden of publishing your code to a CDN in addition to the npm registry. All you need to do is include your [UMD](https://github.com/umdjs/umd) build in your npm package (not your repo, that's different!).
You can do this easily using the following setup:
* Add the `umd` (or `dist`) directory to your `.gitignore` file
* Add the `umd` directory to your [files array](https://docs.npmjs.com/files/package.json#files) in `package.json`
* Use a build script to generate your UMD build in the `umd` directory when you publish
That's it! Now when you `npm publish` you'll have a version available on unpkg as well.

View File

@ -0,0 +1,38 @@
.layout-title {
margin: 0;
text-transform: uppercase;
text-align: center;
font-size: 5em;
}
.layout-nav {
margin: 0 0 3em;
}
.layout-navList {
margin: 0;
padding: 0;
display: flex;
justify-content: center;
}
.layout-navList li {
flex-basis: auto;
list-style-type: none;
display: inline-block;
font-size: 1.1em;
margin: 0 10px;
}
.layout-navList li a:link {
text-decoration: none;
}
.layout-navList li a:link,
.layout-navList li a:visited {
color: black;
}
.layout-navUnderline {
height: 4px;
background-color: black;
position: absolute;
left: 0;
}

View File

@ -0,0 +1,135 @@
require("./Layout.css");
const React = require("react");
const PropTypes = require("prop-types");
const { Switch, Route, Link, withRouter } = require("react-router-dom");
const { Motion, spring } = require("react-motion");
const WindowSize = require("./WindowSize");
const About = require("./About");
const Stats = require("./Stats");
const Home = require("./Home");
class Layout extends React.Component {
static propTypes = {
location: PropTypes.object,
children: PropTypes.node
};
state = {
underlineLeft: 0,
underlineWidth: 0,
useSpring: false,
stats: null
};
adjustUnderline = (useSpring = false) => {
let itemIndex;
switch (this.props.location.pathname) {
case "/stats":
itemIndex = 1;
break;
case "/about":
itemIndex = 2;
break;
case "/":
default:
itemIndex = 0;
}
const itemNodes = this.listNode.querySelectorAll("li");
const currentNode = itemNodes[itemIndex];
this.setState({
underlineLeft: currentNode.offsetLeft,
underlineWidth: currentNode.offsetWidth,
useSpring
});
};
componentDidMount() {
this.adjustUnderline();
fetch("/_stats?period=last-month")
.then(res => res.json())
.then(stats => this.setState({ stats }));
if (window.localStorage) {
const savedStats = window.localStorage.savedStats;
if (savedStats) this.setState({ stats: JSON.parse(savedStats) });
window.onbeforeunload = () => {
localStorage.savedStats = JSON.stringify(this.state.stats);
};
}
}
componentDidUpdate(prevProps) {
if (prevProps.location.pathname !== this.props.location.pathname)
this.adjustUnderline(true);
}
render() {
const { underlineLeft, underlineWidth, useSpring } = this.state;
const style = {
left: useSpring
? spring(underlineLeft, { stiffness: 220 })
: underlineLeft,
width: useSpring ? spring(underlineWidth) : underlineWidth
};
return (
<div className="layout">
<WindowSize onChange={this.adjustUnderline} />
<div className="wrapper">
<header>
<h1 className="layout-title">unpkg</h1>
<nav className="layout-nav">
<ol
className="layout-navList"
ref={node => (this.listNode = node)}
>
<li>
<Link to="/">Home</Link>
</li>
<li>
<Link to="/stats">Stats</Link>
</li>
<li>
<Link to="/about">About</Link>
</li>
</ol>
<Motion
defaultStyle={{ left: underlineLeft, width: underlineWidth }}
style={style}
children={style => (
<div
className="layout-navUnderline"
style={{
WebkitTransform: `translate3d(${style.left}px,0,0)`,
transform: `translate3d(${style.left}px,0,0)`,
width: style.width
}}
/>
)}
/>
</nav>
</header>
</div>
<Switch>
<Route
path="/stats"
render={() => <Stats data={this.state.stats} />}
/>
<Route path="/about" component={About} />
<Route path="/" component={Home} />
</Switch>
</div>
);
}
}
module.exports = withRouter(Layout);

View File

@ -0,0 +1,8 @@
.table-filter {
font-size: 0.8em;
text-align: right;
}
.regions-table .country-row td.country-name {
padding-left: 20px;
}

View File

@ -0,0 +1,323 @@
require("./Stats.css");
const React = require("react");
const PropTypes = require("prop-types");
const formatBytes = require("pretty-bytes");
const formatDate = require("date-fns/format");
const parseDate = require("date-fns/parse");
const { continents, countries } = require("countries-list");
const formatNumber = require("../utils/formatNumber");
const formatPercent = require("../utils/formatPercent");
function getCountriesByContinent(continent) {
return Object.keys(countries).filter(
country => countries[country].continent === continent
);
}
function sumKeyValues(hash, keys) {
return keys.reduce((n, key) => n + (hash[key] || 0), 0);
}
function sumValues(hash) {
return Object.keys(hash).reduce((memo, key) => memo + hash[key], 0);
}
class Stats extends React.Component {
static propTypes = {
data: PropTypes.object
};
state = {
minPackageRequests: 1000000,
minCountryRequests: 1000000
};
render() {
const { data } = this.props;
if (data == null) return null;
const totals = data.totals;
// Summary data
const since = parseDate(totals.since);
const until = parseDate(totals.until);
// Packages
const packageRows = [];
Object.keys(totals.requests.package)
.sort((a, b) => {
return totals.requests.package[b] - totals.requests.package[a];
})
.forEach(packageName => {
const requests = totals.requests.package[packageName];
const bandwidth = totals.bandwidth.package[packageName];
if (requests >= this.state.minPackageRequests) {
packageRows.push(
<tr key={packageName}>
<td>
<a
href={`https://npmjs.org/package/${packageName}`}
title={`${packageName} on npm`}
>
{packageName}
</a>
</td>
<td>
{formatNumber(requests)} ({formatPercent(
requests / totals.requests.all
)}%)
</td>
{bandwidth ? (
<td>
{formatBytes(bandwidth)} ({formatPercent(
bandwidth / totals.bandwidth.all
)}%)
</td>
) : (
<td>-</td>
)}
</tr>
);
}
});
// Regions
const regionRows = [];
const continentsData = Object.keys(continents).reduce((memo, continent) => {
const localCountries = getCountriesByContinent(continent);
memo[continent] = {
countries: localCountries,
requests: sumKeyValues(totals.requests.country, localCountries),
bandwidth: sumKeyValues(totals.bandwidth.country, localCountries)
};
return memo;
}, {});
const topContinents = Object.keys(continentsData).sort((a, b) => {
return continentsData[b].requests - continentsData[a].requests;
});
topContinents.forEach(continent => {
const continentName = continents[continent];
const continentData = continentsData[continent];
if (
continentData.requests > this.state.minCountryRequests &&
continentData.bandwidth !== 0
) {
regionRows.push(
<tr key={continent} className="continent-row">
<td>
<strong>{continentName}</strong>
</td>
<td>
<strong>
{formatNumber(continentData.requests)} ({formatPercent(
continentData.requests / totals.requests.all
)}%)
</strong>
</td>
<td>
<strong>
{formatBytes(continentData.bandwidth)} ({formatPercent(
continentData.bandwidth / totals.bandwidth.all
)}%)
</strong>
</td>
</tr>
);
const topCountries = continentData.countries.sort((a, b) => {
return totals.requests.country[b] - totals.requests.country[a];
});
topCountries.forEach(country => {
const countryRequests = totals.requests.country[country];
const countryBandwidth = totals.bandwidth.country[country];
if (countryRequests > this.state.minCountryRequests) {
regionRows.push(
<tr key={continent + country} className="country-row">
<td className="country-name">{countries[country].name}</td>
<td>
{formatNumber(countryRequests)} ({formatPercent(
countryRequests / totals.requests.all
)}%)
</td>
<td>
{formatBytes(countryBandwidth)} ({formatPercent(
countryBandwidth / totals.bandwidth.all
)}%)
</td>
</tr>
);
}
});
}
});
// Protocols
const protocolRows = Object.keys(totals.requests.protocol)
.sort((a, b) => {
return totals.requests.protocol[b] - totals.requests.protocol[a];
})
.map(protocol => {
const requests = totals.requests.protocol[protocol];
return (
<tr key={protocol}>
<td>{protocol}</td>
<td>
{formatNumber(requests)} ({formatPercent(
requests / sumValues(totals.requests.protocol)
)}%)
</td>
</tr>
);
});
return (
<div className="wrapper">
<p>
From <strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong> unpkg served{" "}
<strong>{formatNumber(totals.requests.all)}</strong> requests and a
total of <strong>{formatBytes(totals.bandwidth.all)}</strong> of data
to <strong>{formatNumber(totals.uniques.all)}</strong> unique
visitors,{" "}
<strong>
{formatPercent(totals.requests.cached / totals.requests.all, 0)}%
</strong>{" "}
of which were served from the cache.
</p>
<h3>Packages</h3>
<p>
The table below shows the most popular packages served by unpkg from{" "}
<strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong>. Only the top{" "}
{Object.keys(totals.requests.package).length} packages are shown.
</p>
<p className="table-filter">
Include only packages that received at least{" "}
<select
value={this.state.minPackageRequests}
onChange={event =>
this.setState({
minPackageRequests: parseInt(event.target.value, 10)
})
}
>
<option value="0">0</option>
<option value="1000">1,000</option>
<option value="10000">10,000</option>
<option value="100000">100,000</option>
<option value="1000000">1,000,000</option>
<option value="10000000">10,000,000</option>
</select>{" "}
requests.
</p>
<table cellSpacing="0" cellPadding="0" style={{ width: "100%" }}>
<thead>
<tr>
<th>
<strong>Package</strong>
</th>
<th>
<strong>Requests (% of total)</strong>
</th>
<th>
<strong>Bandwidth (% of total)</strong>
</th>
</tr>
</thead>
<tbody>{packageRows}</tbody>
</table>
<h3>Regions</h3>
<p>
The table below breaks down requests to unpkg from{" "}
<strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong> by geographic region.
</p>
<p className="table-filter">
Include only countries that made at least{" "}
<select
value={this.state.minCountryRequests}
onChange={event =>
this.setState({
minCountryRequests: parseInt(event.target.value, 10)
})
}
>
<option value="0">0</option>
<option value="100000">100,000</option>
<option value="1000000">1,000,000</option>
<option value="10000000">10,000,000</option>
<option value="100000000">100,000,000</option>
</select>{" "}
requests.
</p>
<table
cellSpacing="0"
cellPadding="0"
style={{ width: "100%" }}
className="regions-table"
>
<thead>
<tr>
<th>
<strong>Region</strong>
</th>
<th>
<strong>Requests (% of total)</strong>
</th>
<th>
<strong>Bandwidth (% of total)</strong>
</th>
</tr>
</thead>
<tbody>{regionRows}</tbody>
</table>
<h3>Protocols</h3>
<p>
The table below breaks down requests to unpkg from{" "}
<strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong> by HTTP protocol.
</p>
<table cellSpacing="0" cellPadding="0" style={{ width: "100%" }}>
<thead>
<tr>
<th>
<strong>Protocol</strong>
</th>
<th>
<strong>Requests (% of total)</strong>
</th>
</tr>
</thead>
<tbody>{protocolRows}</tbody>
</table>
</div>
);
}
}
module.exports = Stats;

View File

@ -0,0 +1,35 @@
const React = require("react");
const PropTypes = require("prop-types");
const addEvent = require("../utils/addEvent");
const removeEvent = require("../utils/removeEvent");
const resizeEvent = "resize";
class WindowSize extends React.Component {
static propTypes = {
onChange: PropTypes.func
};
handleWindowResize = () => {
if (this.props.onChange)
this.props.onChange({
width: window.innerWidth,
height: window.innerHeight
});
};
componentDidMount() {
addEvent(window, resizeEvent, this.handleWindowResize);
}
componentWillUnmount() {
removeEvent(window, resizeEvent, this.handleWindowResize);
}
render() {
return null;
}
}
module.exports = WindowSize;

View File

@ -0,0 +1,9 @@
function addEvent(node, type, handler) {
if (node.addEventListener) {
node.addEventListener(type, handler, false);
} else if (node.attachEvent) {
node.attachEvent("on" + type, handler);
}
}
module.exports = addEvent;

View File

@ -0,0 +1,5 @@
function createHTML(code) {
return { __html: code };
}
module.exports = createHTML;

View File

@ -0,0 +1,9 @@
const React = require("react");
const h = require("./createHTML");
function execScript(code) {
return <script dangerouslySetInnerHTML={h(code)} />;
}
module.exports = execScript;

View File

@ -0,0 +1,12 @@
function formatNumber(n) {
const digits = String(n).split("");
const groups = [];
while (digits.length) {
groups.unshift(digits.splice(-3).join(""));
}
return groups.join(",");
}
module.exports = formatNumber;

View File

@ -0,0 +1,5 @@
function formatPercent(n, fixed = 1) {
return String((n.toPrecision(2) * 100).toFixed(fixed));
}
module.exports = formatPercent;

View File

@ -0,0 +1,5 @@
function parseNumber(s) {
return parseInt(s.replace(/,/g, ""), 10) || 0;
}
module.exports = parseNumber;

View File

@ -0,0 +1,9 @@
function removeEvent(node, type, handler) {
if (node.removeEventListener) {
node.removeEventListener(type, handler, false);
} else if (node.detachEvent) {
node.detachEvent("on" + type, handler);
}
}
module.exports = removeEvent;

9
modules/clientRuntime.js Normal file
View File

@ -0,0 +1,9 @@
// Use babel to compile JSX on the fly.
require("babel-register")({
only: /modules\/client/
});
// Ignore require("*.css") calls.
require.extensions[".css"] = function() {
return {};
};

View File

@ -0,0 +1,42 @@
const webpack = require("webpack");
/**
* Returns a modified copy of the given webpackEntry object with
* the moduleId in front of all other assets.
*/
function prependModuleId(webpackEntry, moduleId) {
if (typeof webpackEntry === "string") {
return [moduleId, webpackEntry];
}
if (Array.isArray(webpackEntry)) {
return [moduleId, ...webpackEntry];
}
if (webpackEntry && typeof webpackEntry === "object") {
const entry = { ...webpackEntry };
for (const chunkName in entry) {
if (entry.hasOwnProperty(chunkName)) {
entry[chunkName] = prependModuleId(entry[chunkName], moduleId);
}
}
return entry;
}
throw new Error("Invalid webpack entry object");
}
/**
* Creates a webpack compiler that automatically inlines the
* webpack dev runtime in all entry points.
*/
function createDevCompiler(webpackConfig, webpackRuntimeModuleId) {
return webpack({
...webpackConfig,
entry: prependModuleId(webpackConfig.entry, webpackRuntimeModuleId)
});
}
module.exports = createDevCompiler;

View File

@ -0,0 +1,54 @@
const express = require("express");
const morgan = require("morgan");
const WebpackDevServer = require("webpack-dev-server");
const devErrorHandler = require("errorhandler");
const devAssets = require("./middleware/devAssets");
const createDevCompiler = require("./createDevCompiler");
const createRouter = require("./createRouter");
function createDevServer(publicDir, webpackConfig, devOrigin) {
const compiler = createDevCompiler(
webpackConfig,
`webpack-dev-server/client?${devOrigin}`
);
const server = new WebpackDevServer(compiler, {
// webpack-dev-middleware options
publicPath: webpackConfig.output.publicPath,
quiet: false,
noInfo: false,
stats: {
// https://webpack.js.org/configuration/stats/
assets: true,
colors: true,
version: true,
hash: true,
timings: true,
chunks: false
},
// webpack-dev-server options
contentBase: false,
disableHostCheck: true,
before(app) {
// This runs before webpack-dev-middleware
app.disable("x-powered-by");
app.use(morgan("dev"));
}
});
// This runs after webpack-dev-middleware
server.use(devErrorHandler());
if (publicDir) {
server.use(express.static(publicDir));
}
server.use(devAssets(compiler));
server.use(createRouter());
return server;
}
module.exports = createDevServer;

71
modules/createRouter.js Normal file
View File

@ -0,0 +1,71 @@
const express = require("express");
const bodyParser = require("body-parser");
const cors = require("cors");
function route(setup) {
const app = express.Router();
setup(app);
return app;
}
function createRouter() {
const app = express.Router();
app.get("/", require("./actions/serveMainPage"));
app.use(cors());
app.use(bodyParser.json());
app.use(require("./middleware/userToken"));
app.get("/_publicKey", require("./actions/showPublicKey"));
app.use(
"/_auth",
route(app => {
app.post("/", require("./actions/createAuth"));
app.get("/", require("./actions/showAuth"));
})
);
app.use(
"/_blacklist",
route(app => {
app.post(
"/",
require("./middleware/requireAuth")("blacklist.add"),
require("./actions/addToBlacklist")
);
app.get(
"/",
require("./middleware/requireAuth")("blacklist.read"),
require("./actions/showBlacklist")
);
app.delete(
"*",
require("./middleware/requireAuth")("blacklist.remove"),
require("./middleware/validatePackageURL"),
require("./actions/removeFromBlacklist")
);
})
);
if (process.env.NODE_ENV !== "test") {
app.get("/_stats", require("./actions/showStats"));
}
app.get(
"*",
require("./middleware/redirectLegacyURLs"),
require("./middleware/validatePackageURL"),
require("./middleware/validatePackageName"),
require("./middleware/validateQuery"),
require("./middleware/checkBlacklist"),
require("./middleware/fetchPackage"),
require("./middleware/findFile"),
require("./actions/serveFile")
);
return app;
}
module.exports = createRouter;

74
modules/createServer.js Normal file
View File

@ -0,0 +1,74 @@
const http = require("http");
const express = require("express");
const morgan = require("morgan");
const staticAssets = require("./middleware/staticAssets");
const createRouter = require("./createRouter");
morgan.token("fwd", req => {
return req.get("x-forwarded-for").replace(/\s/g, "");
});
function errorHandler(err, req, res, next) {
console.error(err.stack);
res
.status(500)
.type("text")
.send("Internal Server Error");
next(err);
}
function createServer(publicDir, statsFile) {
const app = express();
app.disable("x-powered-by");
if (process.env.NODE_ENV !== "test") {
app.use(
morgan(
// Modified version of Heroku's log format
// https://devcenter.heroku.com/articles/http-routing#heroku-router-log-format
'method=:method path=":url" host=:req[host] request_id=:req[x-request-id] cf_ray=:req[cf-ray] fwd=:fwd status=:status bytes=:res[content-length]'
)
);
}
app.use(errorHandler);
if (publicDir) {
app.use(express.static(publicDir, { maxAge: "365d" }));
}
if (statsFile) {
app.use(staticAssets(statsFile));
}
app.use(createRouter());
const server = http.createServer(app);
// Heroku dynos automatically timeout after 30s. Set our
// own timeout here to force sockets to close before that.
// https://devcenter.heroku.com/articles/request-timeout
server.setTimeout(25000, socket => {
const message = `Timeout of 25 seconds exceeded`;
socket.end(
[
"HTTP/1.1 503 Service Unavailable",
"Date: " + new Date().toGMTString(),
"Content-Length: " + Buffer.byteLength(message),
"Content-Type: text/plain",
"Connection: close",
"",
message
].join("\r\n")
);
});
return server;
}
module.exports = createServer;

251
modules/ingestLogs.js Normal file
View File

@ -0,0 +1,251 @@
const parseURL = require("url").parse;
const startOfDay = require("date-fns/start_of_day");
const startOfMinute = require("date-fns/start_of_minute");
const addDays = require("date-fns/add_days");
const db = require("./utils/data");
const isValidPackageName = require("./utils/isValidPackageName");
const parsePackageURL = require("./utils/parsePackageURL");
const CloudflareAPI = require("./CloudflareAPI");
const StatsAPI = require("./StatsAPI");
/**
* Domains we want to analyze.
*/
const domainNames = [
"unpkg.com"
//"npmcdn.com" // We don't have log data on npmcdn.com yet :/
];
/**
* The window of time to download in a single fetch.
*/
const logWindowSeconds = 30;
/**
* The minimum time to wait between fetches.
*/
const minInterval = 15000;
function getSeconds(date) {
return Math.floor(date.getTime() / 1000);
}
function stringifySeconds(seconds) {
return new Date(seconds * 1000).toISOString().replace(/\.0+Z$/, "Z");
}
function toSeconds(ms) {
return Math.floor(ms / 1000);
}
const oneSecond = 1000;
const oneMinute = oneSecond * 60;
const oneHour = oneMinute * 60;
const oneDay = oneHour * 24;
function computeCounters(stream) {
return new Promise((resolve, reject) => {
const counters = {};
const expireat = {};
let totalEntries = 0;
function incr(key, member, by, expiry) {
counters[key] = counters[key] || {};
counters[key][member] = (counters[key][member] || 0) + by;
expireat[key] = expiry;
}
stream
.on("error", reject)
.on("data", entry => {
totalEntries += 1;
const date = new Date(Math.round(entry.EdgeStartTimestamp / 1000000));
const nextDay = startOfDay(addDays(date, 1));
const sevenDaysLater = getSeconds(addDays(nextDay, 7));
const thirtyDaysLater = getSeconds(addDays(nextDay, 30));
const dayKey = StatsAPI.createDayKey(date);
if (entry.EdgeResponseStatus === 200) {
// Q: How many requests do we serve for a package per day?
// Q: How many bytes do we serve for a package per day?
const url = parsePackageURL(entry.ClientRequestURI);
const packageName = url && url.packageName;
if (packageName && isValidPackageName(packageName)) {
incr(
`stats-packageRequests-${dayKey}`,
packageName,
1,
thirtyDaysLater
);
incr(
`stats-packageBytes-${dayKey}`,
packageName,
entry.EdgeResponseBytes,
thirtyDaysLater
);
}
}
// Q: How many requests per day do we receive via a protocol?
const protocol = entry.ClientRequestProtocol;
if (protocol) {
incr(
`stats-protocolRequests-${dayKey}`,
protocol,
1,
thirtyDaysLater
);
}
// Q: How many requests do we receive from a hostname per day?
// Q: How many bytes do we serve to a hostname per day?
const referer = entry.ClientRequestReferer;
const hostname = referer && parseURL(referer).hostname;
if (hostname) {
incr(`stats-hostnameRequests-${dayKey}`, hostname, 1, sevenDaysLater);
incr(
`stats-hostnameBytes-${dayKey}`,
hostname,
entry.EdgeResponseBytes,
sevenDaysLater
);
}
})
.on("end", () => {
resolve({ counters, expireat, totalEntries });
});
});
}
function processLogs(stream) {
return computeCounters(stream).then(
({ counters, expireat, totalEntries }) => {
Object.keys(counters).forEach(key => {
const values = counters[key];
Object.keys(values).forEach(member => {
db.zincrby(key, values[member], member);
});
if (expireat[key]) {
db.expireat(key, expireat[key]);
}
});
return totalEntries;
}
);
}
function ingestLogs(zone, startSeconds, endSeconds) {
const startFetchTime = Date.now();
const fields = [
"EdgeStartTimestamp",
"EdgeResponseStatus",
"EdgeResponseBytes",
"ClientRequestProtocol",
"ClientRequestURI",
"ClientRequestReferer"
];
return CloudflareAPI.getLogs(
zone.id,
stringifySeconds(startSeconds),
stringifySeconds(endSeconds),
fields
).then(stream => {
const endFetchTime = Date.now();
console.log(
"info: Fetched logs for %s from %s to %s (%dms)",
zone.name,
stringifySeconds(startSeconds),
stringifySeconds(endSeconds),
endFetchTime - startFetchTime
);
const startProcessTime = Date.now();
return processLogs(stream).then(totalEntries => {
const endProcessTime = Date.now();
console.log(
"info: Processed %d log entries for %s (%dms)",
totalEntries,
zone.name,
endProcessTime - startProcessTime
);
});
});
}
function startZone(zone) {
const suffix = zone.name.replace(".", "-");
const startSecondsKey = `ingestLogs-start-${suffix}`;
function takeATurn() {
const now = Date.now();
// Cloudflare keeps logs around for 7 days.
// https://support.cloudflare.com/hc/en-us/articles/216672448-Enterprise-Log-Share-Logpull-REST-API
const minSeconds = toSeconds(startOfMinute(now - oneDay * 5));
db.get(startSecondsKey, (error, value) => {
let startSeconds = value && parseInt(value, 10);
if (startSeconds == null) {
startSeconds = minSeconds;
} else if (startSeconds < minSeconds) {
console.warn(
"warning: Dropped logs for %s from %s to %s!",
zone.name,
stringifySeconds(startSeconds),
stringifySeconds(minSeconds)
);
startSeconds = minSeconds;
}
const endSeconds = startSeconds + logWindowSeconds;
// The log for a request is typically available within thirty (30) minutes
// of the request taking place under normal conditions. We deliver logs
// ordered by the time that the logs were created, i.e. the timestamp of
// the request when it was received by the edge. Given the order of
// delivery, we recommend waiting a full thirty minutes to ingest a full
// set of logs. This will help ensure that any congestion in the log
// pipeline has passed and a full set of logs can be ingested.
// https://support.cloudflare.com/hc/en-us/articles/216672448-Enterprise-Log-Share-REST-API
const maxSeconds = toSeconds(now - oneMinute * 30);
if (endSeconds < maxSeconds) {
ingestLogs(zone, startSeconds, endSeconds).then(
() => {
db.set(startSecondsKey, endSeconds);
setTimeout(takeATurn, minInterval);
},
error => {
console.error(error.stack);
process.exit(1);
}
);
} else {
setTimeout(takeATurn, (startSeconds - maxSeconds) * 1000);
}
});
}
takeATurn();
}
Promise.all(domainNames.map(CloudflareAPI.getZones)).then(results => {
const zones = results.reduce((memo, zones) => memo.concat(zones));
zones.forEach(startZone);
});

View File

@ -0,0 +1,26 @@
const BlacklistAPI = require("../BlacklistAPI");
function checkBlacklist(req, res, next) {
BlacklistAPI.includesPackage(req.packageName).then(
blacklisted => {
// Disallow packages that have been blacklisted.
if (blacklisted) {
res
.status(403)
.type("text")
.send(`Package "${req.packageName}" is blacklisted`);
} else {
next();
}
},
error => {
console.error(error);
res.status(500).send({
error: "Unable to fetch the blacklist"
});
}
);
}
module.exports = checkBlacklist;

View File

@ -0,0 +1,29 @@
const invariant = require("invariant");
const createAssets = require("./utils/createAssets");
/**
* An express middleware that sets req.assets from the
* latest result from a running webpack compiler (i.e. using
* webpack-dev-middleware). Should only be used in dev.
*/
function devAssets(webpackCompiler) {
let assets;
webpackCompiler.plugin("done", stats => {
assets = createAssets(stats.toJson());
});
return (req, res, next) => {
invariant(
assets != null,
"devAssets middleware needs a running compiler; " +
"use webpack-dev-middleware in front of devAssets"
);
req.assets = assets;
next();
};
}
module.exports = devAssets;

View File

@ -0,0 +1,153 @@
const semver = require("semver");
const addLeadingSlash = require("../utils/addLeadingSlash");
const createPackageURL = require("../utils/createPackageURL");
const createSearch = require("../utils/createSearch");
const getNpmPackageInfo = require("../utils/getNpmPackageInfo");
const incrementCounter = require("../utils/incrementCounter");
function tagRedirect(req, res) {
const version = req.packageInfo["dist-tags"][req.packageVersion];
// Cache tag redirects for 1 minute.
res
.set({
"Cache-Control": "public, max-age=60",
"Cache-Tag": "redirect,tag-redirect"
})
.redirect(
302,
createPackageURL(req.packageName, version, req.filename, req.search)
);
}
function semverRedirect(req, res) {
const maxVersion = semver.maxSatisfying(
Object.keys(req.packageInfo.versions),
req.packageVersion
);
if (maxVersion) {
// Cache semver redirects for 1 minute.
res
.set({
"Cache-Control": "public, max-age=60",
"Cache-Tag": "redirect,semver-redirect"
})
.redirect(
302,
createPackageURL(req.packageName, maxVersion, req.filename, req.search)
);
} else {
res
.status(404)
.type("text")
.send(`Cannot find package ${req.packageSpec}`);
}
}
function filenameRedirect(req, res) {
let filename;
if (req.query.module != null) {
// See https://github.com/rollup/rollup/wiki/pkg.module
filename =
req.packageConfig.module ||
req.packageConfig["jsnext:main"] ||
"/index.js";
} else if (
req.query.main &&
req.packageConfig[req.query.main] &&
typeof req.packageConfig[req.query.main] === "string"
) {
// Deprecated, see #63
filename = req.packageConfig[req.query.main];
// Count which packages are using this so we can warn them when we
// remove this functionality.
incrementCounter(
"package-json-custom-main",
req.packageSpec + "?main=" + req.query.main,
1
);
} else if (
req.packageConfig.unpkg &&
typeof req.packageConfig.unpkg === "string"
) {
filename = req.packageConfig.unpkg;
} else if (
req.packageConfig.browser &&
typeof req.packageConfig.browser === "string"
) {
// Deprecated, see #63
filename = req.packageConfig.browser;
// Count which packages are using this so we can warn them when we
// remove this functionality.
incrementCounter("package-json-browser-fallback", req.packageSpec, 1);
} else {
filename = req.packageConfig.main || "/index.js";
}
// Redirect to the exact filename so relative imports
// and URLs resolve correctly.
// TODO: increase the max-age?
res
.set({
"Cache-Control": "public,max-age=60",
"Cache-Tag": "redirect,filename-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageVersion,
addLeadingSlash(filename),
createSearch(req.query)
)
);
}
/**
* Fetch the package metadata and tarball from npm. Redirect to the exact
* version if the request targets a tag or uses a semver version, or to the
* exact filename if the request omits the filename.
*/
function fetchPackage(req, res, next) {
getNpmPackageInfo(req.packageName).then(
packageInfo => {
if (packageInfo == null || packageInfo.versions == null) {
return res
.status(404)
.type("text")
.send(`Cannot find package "${req.packageName}"`);
}
req.packageInfo = packageInfo;
req.packageConfig = req.packageInfo.versions[req.packageVersion];
if (!req.packageConfig) {
if (req.packageVersion in req.packageInfo["dist-tags"]) {
return tagRedirect(req, res);
} else {
return semverRedirect(req, res);
}
}
if (!req.filename) {
return filenameRedirect(req, res);
}
next();
},
error => {
console.error(error);
return res
.status(500)
.type("text")
.send(`Cannot get info for package "${req.packageName}"`);
}
);
}
module.exports = fetchPackage;

View File

@ -0,0 +1,178 @@
const path = require("path");
const addLeadingSlash = require("../utils/addLeadingSlash");
const createPackageURL = require("../utils/createPackageURL");
const createSearch = require("../utils/createSearch");
const fetchNpmPackage = require("../utils/fetchNpmPackage");
const getIntegrity = require("../utils/getIntegrity");
const getContentType = require("../utils/getContentType");
function indexRedirect(req, res, entry) {
// Redirect to the index file so relative imports
// resolve correctly.
// TODO: increase the max-age?
res
.set({
"Cache-Control": "public,max-age=60",
"Cache-Tag": "redirect,index-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageVersion,
addLeadingSlash(entry.name),
createSearch(req.query)
)
);
}
function stripLeadingSegment(name) {
return name.replace(/^[^\/]+\/?/, "");
}
function searchEntries(tarballStream, entryName, wantsHTML) {
return new Promise((resolve, reject) => {
const entries = {};
let foundEntry = null;
if (entryName === "") {
foundEntry = entries[""] = { name: "", type: "directory" };
}
tarballStream
.on("error", reject)
.on("finish", () => resolve({ entries, foundEntry }))
.on("entry", (header, stream, next) => {
const entry = {
// Most packages have header names that look like `package/index.js`
// so we shorten that to just `index.js` here. A few packages use a
// prefix other than `package/`. e.g. the firebase package uses the
// `firebase_npm/` prefix. So we just strip the first dir name.
name: stripLeadingSegment(header.name),
type: header.type
};
// We are only interested in files that match the entryName.
if (entry.type !== "file" || entry.name.indexOf(entryName) !== 0) {
stream.resume();
stream.on("end", next);
return;
}
entries[entry.name] = entry;
// Dynamically create "directory" entries for all directories
// that are in this file's path. Some tarballs omit these entries
// for some reason, so this is the brute force method.
let dirname = path.dirname(entry.name);
while (dirname !== ".") {
const directoryEntry = { name: dirname, type: "directory" };
if (!entries[dirname]) {
entries[dirname] = directoryEntry;
if (directoryEntry.name === entryName) {
foundEntry = directoryEntry;
}
}
dirname = path.dirname(dirname);
}
// Set the foundEntry variable if this entry name
// matches exactly or if it's an index.html file
// and the client wants HTML.
if (
entry.name === entryName ||
// Allow accessing e.g. `/lib/index.html` using `/lib/`
(wantsHTML && entry.name === `${entryName}/index.html`) ||
// Allow accessing e.g. `/index.js` or `/index.json` using
// `/index` for compatibility with CommonJS
(!wantsHTML && entry.name === `${entryName}.js`) ||
(!wantsHTML && entry.name === `${entryName}.json`)
) {
foundEntry = entry;
}
const chunks = [];
stream.on("data", chunk => chunks.push(chunk)).on("end", () => {
const content = Buffer.concat(chunks);
// Set some extra properties for files that we will
// need to serve them and for ?meta listings.
entry.contentType = getContentType(entry.name);
entry.integrity = getIntegrity(content);
entry.lastModified = header.mtime.toUTCString();
entry.size = content.length;
// Set the content only for the foundEntry and
// discard the buffer for all others.
if (entry === foundEntry) {
entry.content = content;
}
next();
});
});
});
}
const leadingSlash = /^\//;
const trailingSlash = /\/$/;
/**
* Fetch and search the archive to try and find the requested file.
* Redirect to the "index" file if a directory was requested.
*/
function findFile(req, res, next) {
fetchNpmPackage(req.packageConfig).then(tarballStream => {
const entryName = req.filename
.replace(trailingSlash, "")
.replace(leadingSlash, "");
const wantsHTML = trailingSlash.test(req.filename);
searchEntries(tarballStream, entryName, wantsHTML).then(
({ entries, foundEntry }) => {
if (!foundEntry) {
return res
.status(404)
.type("text")
.send(`Cannot find "${req.filename}" in ${req.packageSpec}`);
}
// If the foundEntry is a directory and there is no trailing slash
// on the request path, we need to redirect to some "index" file
// inside that directory. This is so our URLs work in a similar way
// to require("lib") in node where it searches for `lib/index.js`
// and `lib/index.json` when `lib` is a directory.
if (foundEntry.type === "directory" && !wantsHTML) {
const indexEntry =
entries[path.join(entryName, "index.js")] ||
entries[path.join(entryName, "index.json")];
if (indexEntry && indexEntry.type === "file") {
return indexRedirect(req, res, indexEntry);
} else {
return res
.status(404)
.type("text")
.send(
`Cannot find an index in "${req.filename}" in ${
req.packageSpec
}`
);
}
}
req.entries = entries;
req.entry = foundEntry;
next();
}
);
});
}
module.exports = findFile;

View File

@ -0,0 +1,23 @@
const createSearch = require("../utils/createSearch");
/**
* Redirect old URLs that we no longer support.
*/
function redirectLegacyURLs(req, res, next) {
// Permanently redirect /_meta/path to /path?meta.
if (req.path.match(/^\/_meta\//)) {
req.query.meta = "";
return res.redirect(301, req.path.substr(6) + createSearch(req.query));
}
// Permanently redirect /path?json => /path?meta
if (req.query.json != null) {
delete req.query.json;
req.query.meta = "";
return res.redirect(301, req.path + createSearch(req.query));
}
next();
}
module.exports = redirectLegacyURLs;

View File

@ -0,0 +1,40 @@
/**
* Adds the given scope to the array in req.auth if the user has sufficient
* permissions. Otherwise rejects the request.
*/
function requireAuth(scope) {
let checkScopes;
if (scope.includes(".")) {
const parts = scope.split(".");
checkScopes = scopes =>
parts.reduce((memo, part) => memo && memo[part], scopes) != null;
} else {
checkScopes = scopes => scopes[scope] != null;
}
return function(req, res, next) {
if (req.auth && req.auth.includes(scope)) {
return next(); // Already auth'd
}
const user = req.user;
if (!user) {
return res.status(403).send({ error: "Missing auth token" });
}
if (!user.scopes || !checkScopes(user.scopes)) {
return res.status(403).send({ error: "Insufficient scopes" });
}
if (req.auth) {
req.auth.push(scope);
} else {
req.auth = [scope];
}
next();
};
}
module.exports = requireAuth;

View File

@ -0,0 +1,31 @@
const fs = require("fs");
const invariant = require("invariant");
const createAssets = require("./utils/createAssets");
/**
* An express middleware that sets req.assets from the build
* info in the given stats file. Should be used in production.
*/
function staticAssets(webpackStatsFile) {
let stats;
try {
stats = JSON.parse(fs.readFileSync(webpackStatsFile, "utf8"));
} catch (error) {
invariant(
false,
"staticAssets middleware cannot read the build stats in %s; " +
"run the `build` script before starting the server",
webpackStatsFile
);
}
const assets = createAssets(stats);
return (req, res, next) => {
req.assets = assets;
next();
};
}
module.exports = staticAssets;

View File

@ -0,0 +1,41 @@
const AuthAPI = require("../AuthAPI");
const ReadMethods = { GET: true, HEAD: true };
/**
* Sets req.user from the payload in the auth token in the request.
*/
function userToken(req, res, next) {
if (req.user) {
return next();
}
const token = (ReadMethods[req.method] ? req.query : req.body).token;
if (!token) {
req.user = null;
return next();
}
AuthAPI.verifyToken(token).then(
payload => {
req.user = payload;
next();
},
error => {
if (error.name === "JsonWebTokenError") {
res.status(403).send({
error: `Bad auth token: ${error.message}`
});
} else {
console.error(error);
res.status(500).send({
error: "Unable to verify auth"
});
}
}
);
}
module.exports = userToken;

View File

@ -0,0 +1,40 @@
/**
* Creates an assets object that is stored on req.assets.
*/
function createAssets(webpackStats) {
const { publicPath, assetsByChunkName } = webpackStats;
/**
* Returns a public URL to the given asset.
*/
const createURL = asset => publicPath + asset;
/**
* Returns an array of URLs to all assets in the given chunks.
*/
const getAll = (chunks = ["main"]) =>
(Array.isArray(chunks) ? chunks : [chunks])
.reduce((memo, chunk) => memo.concat(assetsByChunkName[chunk] || []), [])
.map(createURL);
/**
* Returns an array of URLs to all JavaScript files in the given chunks.
*/
const getScripts = (...chunks) =>
getAll(...chunks).filter(asset => /\.js$/.test(asset));
/**
* Returns an array of URLs to all CSS files in the given chunks.
*/
const getStyles = (...chunks) =>
getAll(...chunks).filter(asset => /\.css$/.test(asset));
return {
createURL,
getAll,
getScripts,
getStyles
};
}
module.exports = createAssets;

View File

@ -0,0 +1,21 @@
const validateNpmPackageName = require("validate-npm-package-name");
/**
* Reject requests for invalid npm package names.
*/
function validatePackageName(req, res, next) {
const errors = validateNpmPackageName(req.packageName).errors;
if (errors) {
const reason = errors.join(", ");
return res
.status(403)
.type("text")
.send(`Invalid package name "${req.packageName}" (${reason})`);
}
next();
}
module.exports = validatePackageName;

View File

@ -0,0 +1,25 @@
const parsePackageURL = require("../utils/parsePackageURL");
/**
* Parse the URL and add various properties to the request object to
* do with the package/file being requested. Reject invalid URLs.
*/
function validatePackageURL(req, res, next) {
const url = parsePackageURL(req.url);
if (url == null) {
return res.status(403).send({ error: `Invalid URL: ${req.url}` });
}
req.packageName = url.packageName;
req.packageVersion = url.packageVersion;
req.packageSpec = `${url.packageName}@${url.packageVersion}`;
req.pathname = url.pathname; // TODO: remove
req.filename = url.filename;
req.search = url.search;
req.query = url.query;
next();
}
module.exports = validatePackageURL;

View File

@ -0,0 +1,34 @@
const createSearch = require("../utils/createSearch");
const knownQueryParams = {
main: true, // Deprecated, see #63
meta: true,
module: true
};
function isKnownQueryParam(param) {
return !!knownQueryParams[param];
}
function sanitizeQuery(originalQuery) {
const query = {};
Object.keys(originalQuery).forEach(param => {
if (isKnownQueryParam(param)) query[param] = originalQuery[param];
});
return query;
}
/**
* Reject URLs with invalid query parameters to increase cache hit rates.
*/
function validateQuery(req, res, next) {
if (!Object.keys(req.query).every(isKnownQueryParam)) {
return res.redirect(302, req.path + createSearch(sanitizeQuery(req.query)));
}
next();
}
module.exports = validateQuery;

View File

@ -0,0 +1,5 @@
{
"env": {
"jest": true
}
}

View File

@ -0,0 +1,77 @@
const babel = require("babel-core");
const unpkgRewrite = require("../unpkgRewrite");
const testCases = [
{
before: "import React from 'react';",
after: "import React from 'https://unpkg.com/react@15.6.1?module';"
},
{
before: "import router from '@angular/router';",
after:
"import router from 'https://unpkg.com/@angular/router@4.3.5?module';"
},
{
before: "import map from 'lodash.map';",
after: "import map from 'https://unpkg.com/lodash.map@4.6.0?module';"
},
{
before: "import fs from 'pn/fs';",
after: "import fs from 'https://unpkg.com/pn@1.0.0/fs?module';"
},
{
before: "import cupcakes from './cupcakes';",
after: "import cupcakes from './cupcakes?module';"
},
{
before: "import shoelaces from '/shoelaces';",
after: "import shoelaces from '/shoelaces?module';"
},
{
before: "import something from '//something.com/whatevs';",
after: "import something from '//something.com/whatevs';"
},
{
before: "import something from 'http://something.com/whatevs';",
after: "import something from 'http://something.com/whatevs';"
},
{
before: "let ReactDOM = require('react-dom');",
after: "let ReactDOM = require('react-dom');"
},
{
before: "export React from 'react';",
after: "export React from 'https://unpkg.com/react@15.6.1?module';"
},
{
before: "export { Component } from 'react';",
after: "export { Component } from 'https://unpkg.com/react@15.6.1?module';"
},
{
before: "export * from 'react';",
after: "export * from 'https://unpkg.com/react@15.6.1?module';"
},
{
before: "export var message = 'hello';",
after: "export var message = 'hello';"
}
];
const dependencies = {
react: "15.6.1",
"@angular/router": "4.3.5",
"lodash.map": "4.6.0",
pn: "1.0.0"
};
describe("Rewriting imports/exports", () => {
testCases.forEach(testCase => {
it(`successfully rewrites "${testCase.before}"`, () => {
const result = babel.transform(testCase.before, {
plugins: [unpkgRewrite(dependencies)]
});
expect(result.code).toEqual(testCase.after);
});
});
});

View File

@ -0,0 +1,47 @@
const URL = require("whatwg-url");
const warning = require("warning");
const origin = require("../serverConfig").origin;
const bareIdentifierFormat = /^((?:@[^/]+\/)?[^/]+)(\/.*)?$/;
function unpkgRewrite(dependencies = {}) {
return {
inherits: require("babel-plugin-syntax-export-extensions"),
visitor: {
"ImportDeclaration|ExportNamedDeclaration|ExportAllDeclaration"(path) {
if (!path.node.source) return; // probably a variable declaration
if (
URL.parseURL(path.node.source.value) != null ||
path.node.source.value.substr(0, 2) === "//"
)
return; // valid URL or URL w/o protocol, leave it alone
if ([".", "/"].indexOf(path.node.source.value.charAt(0)) >= 0) {
// local path
path.node.source.value = `${path.node.source.value}?module`;
} else {
// "bare" identifier
const match = bareIdentifierFormat.exec(path.node.source.value);
const packageName = match[1];
const file = match[2] || "";
warning(
dependencies[packageName],
'Missing version info for package "%s" in dependencies; falling back to "latest"',
packageName
);
const version = dependencies[packageName] || "latest";
const url = `${origin}/${packageName}@${version}${file}?module`;
path.node.source.value = url;
}
}
}
};
}
module.exports = unpkgRewrite;

31
modules/secretKey.js Normal file
View File

@ -0,0 +1,31 @@
const fs = require("fs");
const path = require("path");
const forge = require("node-forge");
const invariant = require("invariant");
let secretKey;
if (process.env.NODE_ENV === "production") {
invariant(
process.env.PRIVATE_KEY,
"Missing $PRIVATE_KEY environment variable"
);
secretKey = {
public: fs.readFileSync(
path.resolve(__dirname, "../secret_key.pub"),
"utf8"
),
private: process.env.PRIVATE_KEY
};
} else {
// Generate a random keypair for dev/testing.
// See https://gist.github.com/sebadoom/2b70969e70db5da9a203bebd9cff099f
const keypair = forge.rsa.generateKeyPair({ bits: 2048 });
secretKey = {
public: forge.pki.publicKeyToPem(keypair.publicKey, 72),
private: forge.pki.privateKeyToPem(keypair.privateKey, 72)
};
}
module.exports = secretKey;

9
modules/serverConfig.js Normal file
View File

@ -0,0 +1,9 @@
exports.port = parseInt(process.env.PORT, 10) || 5000;
exports.origin =
process.env.NODE_ENV === "production" || process.env.NODE_ENV === "test"
? "https://unpkg.com"
: `http://localhost:${exports.port}`;
exports.registryURL =
process.env.NPM_REGISTRY_URL || "https://registry.npmjs.org";

View File

@ -0,0 +1,5 @@
{
"env": {
"jest": true
}
}

View File

@ -0,0 +1,15 @@
const createSearch = require("../createSearch");
describe("createSearch", () => {
it("omits the trailing = for empty string values", () => {
expect(createSearch({ a: "a", b: "" })).toEqual("?a=a&b");
});
it("sorts keys", () => {
expect(createSearch({ b: "b", a: "a", c: "c" })).toEqual("?a=a&b=b&c=c");
});
it("returns an empty string when there are no params", () => {
expect(createSearch({})).toEqual("");
});
});

View File

@ -0,0 +1,39 @@
const getContentType = require("../getContentType");
it("gets a content type of text/plain for LICENSE|README|CHANGES|AUTHORS|Makefile", () => {
expect(getContentType("AUTHORS")).toBe("text/plain");
expect(getContentType("CHANGES")).toBe("text/plain");
expect(getContentType("LICENSE")).toBe("text/plain");
expect(getContentType("Makefile")).toBe("text/plain");
expect(getContentType("PATENTS")).toBe("text/plain");
expect(getContentType("README")).toBe("text/plain");
});
it("gets a content type of text/plain for .*rc files", () => {
expect(getContentType(".eslintrc")).toBe("text/plain");
expect(getContentType(".babelrc")).toBe("text/plain");
expect(getContentType(".anythingrc")).toBe("text/plain");
});
it("gets a content type of text/plain for .git* files", () => {
expect(getContentType(".gitignore")).toBe("text/plain");
expect(getContentType(".gitanything")).toBe("text/plain");
});
it("gets a content type of text/plain for .*ignore files", () => {
expect(getContentType(".eslintignore")).toBe("text/plain");
expect(getContentType(".anythingignore")).toBe("text/plain");
});
it("gets a content type of text/plain for .ts files", () => {
expect(getContentType("app.ts")).toBe("text/plain");
expect(getContentType("app.d.ts")).toBe("text/plain");
});
it("gets a content type of text/plain for .flow files", () => {
expect(getContentType("app.js.flow")).toBe("text/plain");
});
it("gets a content type of text/plain for .lock files", () => {
expect(getContentType("yarn.lock")).toBe("text/plain");
});

View File

@ -0,0 +1,80 @@
const parsePackageURL = require("../parsePackageURL");
describe("parsePackageURL", () => {
it("parses plain packages", () => {
expect(parsePackageURL("/history@1.0.0/umd/history.min.js")).toEqual({
pathname: "/history@1.0.0/umd/history.min.js",
search: "",
query: {},
packageName: "history",
packageVersion: "1.0.0",
filename: "/umd/history.min.js"
});
});
it("parses plain packages with a hyphen in the name", () => {
expect(parsePackageURL("/query-string@5.0.0/index.js")).toEqual({
pathname: "/query-string@5.0.0/index.js",
search: "",
query: {},
packageName: "query-string",
packageVersion: "5.0.0",
filename: "/index.js"
});
});
it("parses plain packages with no version specified", () => {
expect(parsePackageURL("/query-string/index.js")).toEqual({
pathname: "/query-string/index.js",
search: "",
query: {},
packageName: "query-string",
packageVersion: "latest",
filename: "/index.js"
});
});
it("parses plain packages with version spec", () => {
expect(parsePackageURL("/query-string@>=4.0.0/index.js")).toEqual({
pathname: "/query-string@>=4.0.0/index.js",
search: "",
query: {},
packageName: "query-string",
packageVersion: ">=4.0.0",
filename: "/index.js"
});
});
it("parses scoped packages", () => {
expect(parsePackageURL("/@angular/router@4.3.3/src/index.d.ts")).toEqual({
pathname: "/@angular/router@4.3.3/src/index.d.ts",
search: "",
query: {},
packageName: "@angular/router",
packageVersion: "4.3.3",
filename: "/src/index.d.ts"
});
});
it("parses package names with a period in them", () => {
expect(parsePackageURL("/index.js")).toEqual({
pathname: "/index.js",
search: "",
query: {},
packageName: "index.js",
packageVersion: "latest",
filename: ""
});
});
it("parses valid query parameters", () => {
expect(parsePackageURL("/history?main=browser")).toEqual({
pathname: "/history",
search: "?main=browser",
query: { main: "browser" },
packageName: "history",
packageVersion: "latest",
filename: ""
});
});
});

View File

@ -0,0 +1,5 @@
function addLeadingSlash(name) {
return name.charAt(0) === "/" ? name : "/" + name;
}
module.exports = addLeadingSlash;

View File

@ -0,0 +1,12 @@
function bufferStream(stream) {
return new Promise((resolve, reject) => {
const chunks = [];
stream
.on("error", reject)
.on("data", chunk => chunks.push(chunk))
.on("end", () => resolve(Buffer.concat(chunks)));
});
}
module.exports = bufferStream;

9
modules/utils/cache.js Normal file
View File

@ -0,0 +1,9 @@
const redis = require("redis");
redis.debug_mode = process.env.DEBUG_REDIS != null;
const client = redis.createClient(
process.env.CACHE_URL || process.env.OPENREDIS_URL || "redis://localhost:6379"
);
module.exports = client;

View File

@ -0,0 +1,11 @@
function createPackageURL(packageName, version, pathname, search) {
let url = `/${packageName}`;
if (version != null) url += `@${version}`;
if (pathname) url += pathname;
if (search) url += search;
return url;
}
module.exports = createPackageURL;

View File

@ -0,0 +1,16 @@
function createSearch(query) {
const keys = Object.keys(query).sort();
const params = keys.reduce(
(memo, key) =>
memo.concat(
query[key] === ""
? key // Omit the trailing "=" from key=
: `${key}=${encodeURIComponent(query[key])}`
),
[]
);
return params.length ? `?${params.join("&")}` : "";
}
module.exports = createSearch;

9
modules/utils/data.js Normal file
View File

@ -0,0 +1,9 @@
const redis = require("redis");
redis.debug_mode = process.env.DEBUG_REDIS != null;
const client = redis.createClient(
process.env.DATA_URL || process.env.OPENREDIS_URL || "redis://localhost:6379"
);
module.exports = client;

View File

@ -0,0 +1,46 @@
const url = require("url");
const https = require("https");
const gunzip = require("gunzip-maybe");
const tar = require("tar-stream");
const bufferStream = require("./bufferStream");
const agent = require("./registryAgent");
function fetchNpmPackage(packageConfig) {
return new Promise((resolve, reject) => {
const tarballURL = packageConfig.dist.tarball;
console.log(
`info: Fetching package for ${packageConfig.name} from ${tarballURL}`
);
const { hostname, pathname } = url.parse(tarballURL);
const options = {
agent: agent,
hostname: hostname,
path: pathname
};
https
.get(options, res => {
if (res.statusCode === 200) {
resolve(res.pipe(gunzip()).pipe(tar.extract()));
} else {
bufferStream(res).then(data => {
const spec = `${packageConfig.name}@${packageConfig.version}`;
const content = data.toString("utf-8");
const error = new Error(
`Failed to fetch tarball for ${spec}\nstatus: ${
res.statusCode
}\ndata: ${content}`
);
reject(error);
});
}
})
.on("error", reject);
});
}
module.exports = fetchNpmPackage;

View File

@ -0,0 +1,58 @@
const url = require("url");
const https = require("https");
const serverConfig = require("../serverConfig");
const bufferStream = require("./bufferStream");
const agent = require("./registryAgent");
function parseJSON(res) {
return bufferStream(res).then(JSON.parse);
}
function fetchNpmPackageInfo(packageName) {
return new Promise((resolve, reject) => {
const encodedPackageName =
packageName.charAt(0) === "@"
? `@${encodeURIComponent(packageName.substring(1))}`
: encodeURIComponent(packageName);
const infoURL = `${serverConfig.registryURL}/${encodedPackageName}`;
console.log(
`info: Fetching package info for ${packageName} from ${infoURL}`
);
const { hostname, pathname } = url.parse(infoURL);
const options = {
agent: agent,
hostname: hostname,
path: pathname,
headers: {
Accept: "application/json"
}
};
https
.get(options, res => {
if (res.statusCode === 200) {
resolve(parseJSON(res));
} else if (res.statusCode === 404) {
resolve(null);
} else {
bufferStream(res).then(data => {
const content = data.toString("utf-8");
const error = new Error(
`Failed to fetch info for ${packageName}\nstatus: ${
res.statusCode
}\ndata: ${content}`
);
reject(error);
});
}
})
.on("error", reject);
});
}
module.exports = fetchNpmPackageInfo;

View File

@ -0,0 +1,22 @@
const mime = require("mime");
mime.define({
"text/plain": [
"authors",
"changes",
"license",
"makefile",
"patents",
"readme",
"ts",
"flow"
]
});
const textFiles = /\/?(\.[a-z]*rc|\.git[a-z]*|\.[a-z]*ignore|\.lock)$/i;
function getContentType(file) {
return textFiles.test(file) ? "text/plain" : mime.lookup(file);
}
module.exports = getContentType;

View File

@ -0,0 +1,5 @@
function getContentTypeHeader(type) {
return type === "application/javascript" ? type + "; charset=utf-8" : type;
}
module.exports = getContentTypeHeader;

View File

@ -0,0 +1,7 @@
const SRIToolbox = require("sri-toolbox");
function getIntegrity(data) {
return SRIToolbox.generate({ algorithms: ["sha384"] }, data);
}
module.exports = getIntegrity;

View File

@ -0,0 +1,40 @@
const cache = require("./cache");
const fetchNpmPackageInfo = require("./fetchNpmPackageInfo");
const notFound = "PackageNotFound";
function getNpmPackageInfo(packageName) {
return new Promise((resolve, reject) => {
const key = `npmPackageInfo-${packageName}`;
cache.get(key, (error, value) => {
if (error) {
reject(error);
} else if (value != null) {
resolve(value === notFound ? null : JSON.parse(value));
} else {
fetchNpmPackageInfo(packageName).then(value => {
if (value == null) {
resolve(null);
// Cache 404s for 5 minutes. This prevents us from making
// unnecessary requests to the registry for bad package names.
// In the worst case, a brand new package's info will be
// available within 5 minutes.
cache.setex(key, 300, notFound);
} else {
resolve(value);
// Cache valid package info for 1 minute. In the worst case,
// new versions won't be available for 1 minute.
cache.setnx(key, JSON.stringify(value), (error, reply) => {
if (reply === 1) cache.expire(key, 60);
});
}
}, reject);
}
});
});
}
module.exports = getNpmPackageInfo;

View File

@ -0,0 +1,15 @@
const db = require("./data");
function incrementCounter(counter, key, by = 1) {
return new Promise((resolve, reject) => {
db.hincrby(counter, key, by, (error, value) => {
if (error) {
reject(error);
} else {
resolve(value);
}
});
});
}
module.exports = incrementCounter;

View File

@ -0,0 +1,7 @@
const validateNpmPackageName = require("validate-npm-package-name");
function isValidPackageName(packageName) {
return validateNpmPackageName(packageName).errors == null;
}
module.exports = isValidPackageName;

View File

@ -0,0 +1,41 @@
const url = require("url");
const packageURLFormat = /^\/((?:@[^/@]+\/)?[^/@]+)(?:@([^/]+))?(\/.*)?$/;
function decodeParam(param) {
if (param) {
try {
return decodeURIComponent(param);
} catch (error) {
// Ignore invalid params.
}
}
return "";
}
function parsePackageURL(originalURL) {
const { pathname, search, query } = url.parse(originalURL, true);
const match = packageURLFormat.exec(pathname);
// Disallow invalid URL formats.
if (match == null) {
return null;
}
const packageName = match[1];
const packageVersion = decodeParam(match[2]) || "latest";
const filename = decodeParam(match[3]);
return {
// If the URL is /@scope/name@version/file.js?main=browser:
pathname, // /@scope/name@version/path.js
search: search || "", // ?main=browser
query, // { main: 'browser' }
packageName, // @scope/name
packageVersion, // version
filename // /file.js
};
}
module.exports = parsePackageURL;

Some files were not shown because too many files have changed in this diff Show More