186526
/
unpkg
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Prettify

This commit is contained in:
MICHAEL JACKSON 2017-11-25 13:25:01 -08:00
parent f3974b5e2d
commit 3a309241da
64 changed files with 635 additions and 801 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
client/About.js
View File

@ -1,8 +1,6 @@
import React from 'react'
import contentHTML from './About.md'
import React from "react"
import contentHTML from "./About.md"
const About = () => (
<div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }} />
)
const About = () => <div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }} />
export default About

6
client/App.js
View File

@ -1,6 +1,6 @@
import React from 'react'
import { HashRouter } from 'react-router-dom'
import Layout from './Layout'
import React from "react"
import { HashRouter } from "react-router-dom"
import Layout from "./Layout"
const App = () => (
<HashRouter>

8
client/Home.js
View File

@ -1,8 +1,6 @@
import React from 'react'
import contentHTML from './Home.md'
import React from "react"
import contentHTML from "./Home.md"
const Home = () => (
<div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }} />
)
const Home = () => <div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }} />
export default Home

43
client/Layout.js
View File

@ -1,11 +1,11 @@
import React from 'react'
import PropTypes from 'prop-types'
import { Motion, spring } from 'react-motion'
import { Switch, Route, Link, withRouter } from 'react-router-dom'
import WindowSize from './WindowSize'
import About from './About'
import Stats from './Stats'
import Home from './Home'
import React from "react"
import PropTypes from "prop-types"
import { Motion, spring } from "react-motion"
import { Switch, Route, Link, withRouter } from "react-router-dom"
import WindowSize from "./WindowSize"
import About from "./About"
import Stats from "./Stats"
import Home from "./Home"
class Layout extends React.Component {
static propTypes = {
@ -23,18 +23,18 @@ class Layout extends React.Component {
adjustUnderline = (useSpring = false) => {
let itemIndex
switch (this.props.location.pathname) {
case '/stats':
case "/stats":
itemIndex = 1
break
case '/about':
case "/about":
itemIndex = 2
break
case '/':
case "/":
default:
itemIndex = 0
}
const itemNodes = this.listNode.querySelectorAll('li')
const itemNodes = this.listNode.querySelectorAll("li")
const currentNode = itemNodes[itemIndex]
this.setState({
@ -47,7 +47,7 @@ class Layout extends React.Component {
componentDidMount() {
this.adjustUnderline()
fetch('/_stats?period=last-month')
fetch("/_stats?period=last-month")
.then(res => res.json())
.then(stats => this.setState({ stats }))
@ -63,17 +63,14 @@ class Layout extends React.Component {
}
componentDidUpdate(prevProps) {
if (prevProps.location.pathname !== this.props.location.pathname)
this.adjustUnderline(true)
if (prevProps.location.pathname !== this.props.location.pathname) this.adjustUnderline(true)
}
render() {
const { underlineLeft, underlineWidth, useSpring } = this.state
const style = {
left: useSpring
? spring(underlineLeft, { stiffness: 220 })
: underlineLeft,
left: useSpring ? spring(underlineLeft, { stiffness: 220 }) : underlineLeft,
width: useSpring ? spring(underlineWidth) : underlineWidth
}
@ -84,10 +81,7 @@ class Layout extends React.Component {
<header>
<h1 className="layout-title">unpkg</h1>
<nav className="layout-nav">
<ol
className="layout-nav-list"
ref={node => (this.listNode = node)}
>
<ol className="layout-nav-list" ref={node => (this.listNode = node)}>
<li>
<Link to="/">Home</Link>
</li>
@ -117,10 +111,7 @@ class Layout extends React.Component {
</div>
<Switch>
<Route
path="/stats"
render={() => <Stats data={this.state.stats} />}
/>
<Route path="/stats" render={() => <Stats data={this.state.stats} />} />
<Route path="/about" component={About} />
<Route path="/" component={Home} />
</Switch>

15
client/NumberTextInput.js
View File

@ -1,6 +1,6 @@
import React from 'react'
import PropTypes from 'prop-types'
import { parseNumber, formatNumber } from './NumberUtils'
import React from "react"
import PropTypes from "prop-types"
import { parseNumber, formatNumber } from "./NumberUtils"
class NumberTextInput extends React.Component {
static propTypes = {
@ -37,14 +37,7 @@ class NumberTextInput extends React.Component {
const { parseNumber, formatNumber, ...props } = this.props // eslint-disable-line no-unused-vars
const displayValue = formatNumber(value)
return (
<input
{...props}
type="text"
value={displayValue}
onChange={this.handleChange}
/>
)
return <input {...props} type="text" value={displayValue} onChange={this.handleChange} />
}
}

93
client/Stats.js
View File

@ -1,23 +1,19 @@
import React from 'react'
import PropTypes from 'prop-types'
import formatBytes from 'pretty-bytes'
import formatDate from 'date-fns/format'
import parseDate from 'date-fns/parse'
import formatNumber from './utils/formatNumber'
import formatPercent from './utils/formatPercent'
import React from "react"
import PropTypes from "prop-types"
import formatBytes from "pretty-bytes"
import formatDate from "date-fns/format"
import parseDate from "date-fns/parse"
import formatNumber from "./utils/formatNumber"
import formatPercent from "./utils/formatPercent"
import { continents, countries } from 'countries-list'
import { continents, countries } from "countries-list"
const getCountriesByContinent = continent =>
Object.keys(countries).filter(
country => countries[country].continent === continent
)
Object.keys(countries).filter(country => countries[country].continent === continent)
const sumKeyValues = (hash, keys) =>
keys.reduce((n, key) => n + (hash[key] || 0), 0)
const sumKeyValues = (hash, keys) => keys.reduce((n, key) => n + (hash[key] || 0), 0)
const sumValues = hash =>
Object.keys(hash).reduce((memo, key) => memo + hash[key], 0)
const sumValues = hash => Object.keys(hash).reduce((memo, key) => memo + hash[key], 0)
class Stats extends React.Component {
static propTypes = {
@ -63,15 +59,11 @@ class Stats extends React.Component {
</a>
</td>
<td>
{formatNumber(requests)} ({formatPercent(
requests / totals.requests.all
)}%)
{formatNumber(requests)} ({formatPercent(requests / totals.requests.all)}%)
</td>
{bandwidth ? (
<td>
{formatBytes(bandwidth)} ({formatPercent(
bandwidth / totals.bandwidth.all
)}%)
{formatBytes(bandwidth)} ({formatPercent(bandwidth / totals.bandwidth.all)}%)
</td>
) : (
<td>-</td>
@ -104,10 +96,7 @@ class Stats extends React.Component {
const continentName = continents[continent]
const continentData = continentsData[continent]
if (
continentData.requests > this.state.minCountryRequests &&
continentData.bandwidth !== 0
) {
if (continentData.requests > this.state.minCountryRequests && continentData.bandwidth !== 0) {
regionRows.push(
<tr key={continent} className="continent-row">
<td>{continentName}</td>
@ -176,29 +165,26 @@ class Stats extends React.Component {
return (
<div className="wrapper">
<p>
From <strong>{formatDate(since, 'MMM D')}</strong> to{' '}
<strong>{formatDate(until, 'MMM D')}</strong> unpkg served{' '}
<strong>{formatNumber(totals.requests.all)}</strong> requests and a
total of <strong>{formatBytes(totals.bandwidth.all)}</strong> of data
to <strong>{formatNumber(totals.uniques.all)}</strong> unique
visitors,{' '}
<strong>
{formatPercent(totals.requests.cached / totals.requests.all, 0)}%
</strong>{' '}
of which were served from the cache.
From <strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong> unpkg served{" "}
<strong>{formatNumber(totals.requests.all)}</strong> requests and a total of{" "}
<strong>{formatBytes(totals.bandwidth.all)}</strong> of data to{" "}
<strong>{formatNumber(totals.uniques.all)}</strong> unique visitors,{" "}
<strong>{formatPercent(totals.requests.cached / totals.requests.all, 0)}%</strong> of
which were served from the cache.
</p>
<h3>Packages</h3>
<p>
The table below shows the most popular packages served by unpkg from{' '}
<strong>{formatDate(since, 'MMM D')}</strong> to{' '}
<strong>{formatDate(until, 'MMM D')}</strong>. Only the top{' '}
The table below shows the most popular packages served by unpkg from{" "}
<strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong>. Only the top{" "}
{Object.keys(totals.requests.package).length} packages are shown.
</p>
<p className="table-filter">
Include only packages that received at least{' '}
Include only packages that received at least{" "}
<select
value={this.state.minPackageRequests}
onChange={event =>
@ -213,11 +199,11 @@ class Stats extends React.Component {
<option value="100000">100,000</option>
<option value="1000000">1,000,000</option>
<option value="10000000">10,000,000</option>
</select>{' '}
</select>{" "}
requests.
</p>
<table cellSpacing="0" cellPadding="0" style={{ width: '100%' }}>
<table cellSpacing="0" cellPadding="0" style={{ width: "100%" }}>
<thead>
<tr>
<th>Package</th>
@ -231,13 +217,13 @@ class Stats extends React.Component {
<h3>Regions</h3>
<p>
The table below breaks down requests to unpkg from{' '}
<strong>{formatDate(since, 'MMM D')}</strong> to{' '}
<strong>{formatDate(until, 'MMM D')}</strong> by geographic region.
The table below breaks down requests to unpkg from{" "}
<strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong> by geographic region.
</p>
<p className="table-filter">
Include only countries that made at least{' '}
Include only countries that made at least{" "}
<select
value={this.state.minCountryRequests}
onChange={event =>
@ -251,16 +237,11 @@ class Stats extends React.Component {
<option value="1000000">1,000,000</option>
<option value="10000000">10,000,000</option>
<option value="100000000">100,000,000</option>
</select>{' '}
</select>{" "}
requests.
</p>
<table
cellSpacing="0"
cellPadding="0"
style={{ width: '100%' }}
className="regions-table"
>
<table cellSpacing="0" cellPadding="0" style={{ width: "100%" }} className="regions-table">
<thead>
<tr>
<th>Region</th>
@ -274,12 +255,12 @@ class Stats extends React.Component {
<h3>Protocols</h3>
<p>
The table below breaks down requests to unpkg from{' '}
<strong>{formatDate(since, 'MMM D')}</strong> to{' '}
<strong>{formatDate(until, 'MMM D')}</strong> by HTTP protocol.
The table below breaks down requests to unpkg from{" "}
<strong>{formatDate(since, "MMM D")}</strong> to{" "}
<strong>{formatDate(until, "MMM D")}</strong> by HTTP protocol.
</p>
<table cellSpacing="0" cellPadding="0" style={{ width: '100%' }}>
<table cellSpacing="0" cellPadding="0" style={{ width: "100%" }}>
<thead>
<tr>
<th>Protocol</th>

10
client/WindowSize.js
View File

@ -1,9 +1,9 @@
import React from 'react'
import PropTypes from 'prop-types'
import addEvent from './utils/addEvent'
import removeEvent from './utils/removeEvent'
import React from "react"
import PropTypes from "prop-types"
import addEvent from "./utils/addEvent"
import removeEvent from "./utils/removeEvent"
const ResizeEvent = 'resize'
const ResizeEvent = "resize"
class WindowSize extends React.Component {
static propTypes = {

10
client/index.js
View File

@ -1,6 +1,6 @@
import React from 'react'
import ReactDOM from 'react-dom'
import App from './App'
import './index.css'
import React from "react"
import ReactDOM from "react-dom"
import App from "./App"
import "./index.css"
ReactDOM.render(<App />, document.getElementById('root'))
ReactDOM.render(<App />, document.getElementById("root"))

2
client/utils/addEvent.js
View File

@ -2,7 +2,7 @@ const addEvent = (node, type, handler) => {
if (node.addEventListener) {
node.addEventListener(type, handler, false)
} else if (node.attachEvent) {
node.attachEvent('on' + type, handler)
node.attachEvent("on" + type, handler)
}
}

6
client/utils/formatNumber.js
View File

@ -1,10 +1,10 @@
const formatNumber = n => {
const digits = String(n).split('')
const digits = String(n).split("")
const groups = []
while (digits.length) groups.unshift(digits.splice(-3).join(''))
while (digits.length) groups.unshift(digits.splice(-3).join(""))
return groups.join(',')
return groups.join(",")
}
export default formatNumber

3
client/utils/formatPercent.js
View File

@ -1,4 +1,3 @@
const formatPercent = (n, fixed = 1) =>
String((n.toPrecision(2) * 100).toFixed(fixed))
const formatPercent = (n, fixed = 1) => String((n.toPrecision(2) * 100).toFixed(fixed))
export default formatPercent

2
client/utils/parseNumber.js
View File

@ -1,3 +1,3 @@
const parseNumber = s => parseInt(s.replace(/,/g, ''), 10) || 0
const parseNumber = s => parseInt(s.replace(/,/g, ""), 10) || 0
export default parseNumber

2
client/utils/removeEvent.js
View File

@ -2,7 +2,7 @@ const removeEvent = (node, type, handler) => {
if (node.removeEventListener) {
node.removeEventListener(type, handler, false)
} else if (node.detachEvent) {
node.detachEvent('on' + type, handler)
node.detachEvent("on" + type, handler)
}
}

4
package.json
View File

@ -122,7 +122,7 @@
"extends": "react-app"
},
"prettier": {
"semi": false,
"singleQuote": true
"printWidth": 100,
"semi": false
}
}

4
packages/unpkg/modules/index.js
View File

@ -1,2 +1,2 @@
exports.isBareModuleIdentifier = require('./utils/isBareModuleIdentifier')
exports.parseModuleIdentifier = require('./utils/parseModuleIdentifier')
exports.isBareModuleIdentifier = require("./utils/isBareModuleIdentifier")
exports.parseModuleIdentifier = require("./utils/parseModuleIdentifier")

24
packages/unpkg/modules/utils/__tests__/isBareModuleIdentifier-test.js
View File

@ -1,18 +1,16 @@
const isBareModuleIdentifier = require('../isBareModuleIdentifier')
const isBareModuleIdentifier = require("../isBareModuleIdentifier")
describe('isBareModuleIdentifier', () => {
it('returns true for bare module identifiers', () => {
expect(isBareModuleIdentifier('react')).toBe(true)
expect(isBareModuleIdentifier('react-dom')).toBe(true)
expect(isBareModuleIdentifier('react-dom/server')).toBe(true)
describe("isBareModuleIdentifier", () => {
it("returns true for bare module identifiers", () => {
expect(isBareModuleIdentifier("react")).toBe(true)
expect(isBareModuleIdentifier("react-dom")).toBe(true)
expect(isBareModuleIdentifier("react-dom/server")).toBe(true)
})
it('returns false for non-bare module identifiers', () => {
expect(isBareModuleIdentifier('/absolute-path')).toBe(false)
expect(isBareModuleIdentifier('./relative-path')).toBe(false)
expect(isBareModuleIdentifier('//www.example.com/script.js')).toBe(false)
expect(isBareModuleIdentifier('https://www.example.com/script.js')).toBe(
false
)
it("returns false for non-bare module identifiers", () => {
expect(isBareModuleIdentifier("/absolute-path")).toBe(false)
expect(isBareModuleIdentifier("./relative-path")).toBe(false)
expect(isBareModuleIdentifier("//www.example.com/script.js")).toBe(false)
expect(isBareModuleIdentifier("https://www.example.com/script.js")).toBe(false)
})
})

44
packages/unpkg/modules/utils/__tests__/parseBareModuleIdentifier-test.js
View File

@ -1,38 +1,38 @@
const parseBareModuleIdentifier = require('../parseBareModuleIdentifier')
const parseBareModuleIdentifier = require("../parseBareModuleIdentifier")
describe('parseBareModuleIdentifier', () => {
it('parses simple identifiers', () => {
expect(parseBareModuleIdentifier('react')).toEqual({
packageName: 'react',
file: ''
describe("parseBareModuleIdentifier", () => {
it("parses simple identifiers", () => {
expect(parseBareModuleIdentifier("react")).toEqual({
packageName: "react",
file: ""
})
})
it('parses hyphenated identifiers', () => {
expect(parseBareModuleIdentifier('react-dom')).toEqual({
packageName: 'react-dom',
file: ''
it("parses hyphenated identifiers", () => {
expect(parseBareModuleIdentifier("react-dom")).toEqual({
packageName: "react-dom",
file: ""
})
})
it('parses hyphenated identifiers with a filename', () => {
expect(parseBareModuleIdentifier('react-dom/server')).toEqual({
packageName: 'react-dom',
file: '/server'
it("parses hyphenated identifiers with a filename", () => {
expect(parseBareModuleIdentifier("react-dom/server")).toEqual({
packageName: "react-dom",
file: "/server"
})
})
it('parses scoped identifiers', () => {
expect(parseBareModuleIdentifier('@babel/core')).toEqual({
packageName: '@babel/core',
file: ''
it("parses scoped identifiers", () => {
expect(parseBareModuleIdentifier("@babel/core")).toEqual({
packageName: "@babel/core",
file: ""
})
})
it('parses scoped identifiers with a filename', () => {
expect(parseBareModuleIdentifier('@babel/core/package.json')).toEqual({
packageName: '@babel/core',
file: '/package.json'
it("parses scoped identifiers with a filename", () => {
expect(parseBareModuleIdentifier("@babel/core/package.json")).toEqual({
packageName: "@babel/core",
file: "/package.json"
})
})
})

8
packages/unpkg/modules/utils/isBareModuleIdentifier.js
View File

@ -1,11 +1,11 @@
const URL = require('whatwg-url')
const URL = require("whatwg-url")
function isBareModuleIdentifier(id) {
return !(
URL.parseURL(id) !== null || // fully qualified URL
id.substr(0, 2) === '//' || // URL w/out protocol
['.', '/'].includes(id.charAt(0)) // local path
)
id.substr(0, 2) === "//" || // URL w/out protocol
[".", "/"].includes(id.charAt(0))
) // local path
}
module.exports = isBareModuleIdentifier

2
packages/unpkg/modules/utils/parseBareModuleIdentifier.js
View File

@ -5,7 +5,7 @@ function parseBareModuleIdentifier(id) {
return {
packageName: match[1],
file: match[2] || ''
file: match[2] || ""
}
}

6
packages/unpkg/package.json
View File

@ -2,6 +2,7 @@
"name": "unpkg",
"version": "0.1.0",
"description": "The JavaScript API for unpkg",
"repository": "unpkg/unpkg",
"files": [
"modules/*.js",
"modules/utils/*.js"
@ -9,5 +10,8 @@
"main": "modules/index.js",
"dependencies": {
"whatwg-url": "^6.3.0"
}
},
"keywords": [
"unpkg"
]
}

30
server/AuthAPI.js
View File

@ -1,19 +1,19 @@
const fs = require('fs')
const path = require('path')
const crypto = require('crypto')
const jwt = require('jsonwebtoken')
const invariant = require('invariant')
const forge = require('node-forge')
const db = require('./RedisClient')
const fs = require("fs")
const path = require("path")
const crypto = require("crypto")
const jwt = require("jsonwebtoken")
const invariant = require("invariant")
const forge = require("node-forge")
const db = require("./RedisClient")
let keys
if (process.env.NODE_ENV === 'production') {
if (process.env.NODE_ENV === "production") {
keys = {
public: fs.readFileSync(path.resolve(__dirname, '../public.key'), 'utf8'),
public: fs.readFileSync(path.resolve(__dirname, "../public.key"), "utf8"),
private: process.env.PRIVATE_KEY
}
invariant(keys.private, 'Missing $PRIVATE_KEY environment variable')
invariant(keys.private, "Missing $PRIVATE_KEY environment variable")
} else {
// Generate a random keypair for dev/testing.
// See https://gist.github.com/sebadoom/2b70969e70db5da9a203bebd9cff099f
@ -29,19 +29,19 @@ function getCurrentSeconds() {
}
function createTokenId() {
return crypto.randomBytes(16).toString('hex')
return crypto.randomBytes(16).toString("hex")
}
function createToken(scopes = {}) {
return new Promise((resolve, reject) => {
const payload = {
jti: createTokenId(),
iss: 'https://unpkg.com',
iss: "https://unpkg.com",
iat: getCurrentSeconds(),
scopes
}
jwt.sign(payload, keys.private, { algorithm: 'RS256' }, (error, token) => {
jwt.sign(payload, keys.private, { algorithm: "RS256" }, (error, token) => {
if (error) {
reject(error)
} else {
@ -51,11 +51,11 @@ function createToken(scopes = {}) {
})
}
const RevokedTokensSet = 'revoked-tokens'
const RevokedTokensSet = "revoked-tokens"
function verifyToken(token) {
return new Promise((resolve, reject) => {
const options = { algorithms: ['RS256'] }
const options = { algorithms: ["RS256"] }
jwt.verify(token, keys.public, options, (error, payload) => {
if (error) {

4
server/BlacklistAPI.js
View File

@ -1,6 +1,6 @@
const db = require('./RedisClient')
const db = require("./RedisClient")
const BlacklistSet = 'blacklisted-packages'
const BlacklistSet = "blacklisted-packages"
function addPackage(packageName) {
return new Promise((resolve, reject) => {

30
server/CloudflareAPI.js
View File

@ -1,21 +1,21 @@
require('isomorphic-fetch')
const invariant = require('invariant')
const gunzip = require('gunzip-maybe')
const ndjson = require('ndjson')
require("isomorphic-fetch")
const invariant = require("invariant")
const gunzip = require("gunzip-maybe")
const ndjson = require("ndjson")
const CloudflareAPIURL = 'https://api.cloudflare.com'
const CloudflareAPIURL = "https://api.cloudflare.com"
const CloudflareEmail = process.env.CLOUDFLARE_EMAIL
const CloudflareKey = process.env.CLOUDFLARE_KEY
invariant(CloudflareEmail, 'Missing the $CLOUDFLARE_EMAIL environment variable')
invariant(CloudflareEmail, "Missing the $CLOUDFLARE_EMAIL environment variable")
invariant(CloudflareKey, 'Missing the $CLOUDFLARE_KEY environment variable')
invariant(CloudflareKey, "Missing the $CLOUDFLARE_KEY environment variable")
function get(path, headers) {
return fetch(`${CloudflareAPIURL}/client/v4${path}`, {
headers: Object.assign({}, headers, {
'X-Auth-Email': CloudflareEmail,
'X-Auth-Key': CloudflareKey
"X-Auth-Email": CloudflareEmail,
"X-Auth-Key": CloudflareKey
})
})
}
@ -29,7 +29,7 @@ function getJSON(path, headers) {
if (!data.success) {
console.error(`CloudflareAPI.getJSON failed at ${path}`)
console.error(data)
throw new Error('Failed to getJSON from Cloudflare')
throw new Error("Failed to getJSON from Cloudflare")
}
return data.result
@ -52,9 +52,9 @@ function reduceResults(target, values) {
Object.keys(values).forEach(key => {
const value = values[key]
if (typeof value === 'object' && value) {
if (typeof value === "object" && value) {
target[key] = reduceResults(target[key] || {}, value)
} else if (typeof value === 'number') {
} else if (typeof value === "number") {
target[key] = (target[key] || 0) + values[key]
}
})
@ -78,7 +78,7 @@ function getZoneAnalyticsDashboard(zones, since, until) {
function getJSONStream(path, headers) {
const acceptGzipHeaders = Object.assign({}, headers, {
'Accept-Encoding': 'gzip'
"Accept-Encoding": "gzip"
})
return get(path, acceptGzipHeaders)
@ -91,9 +91,7 @@ function getJSONStream(path, headers) {
}
function getLogs(zoneId, startTime, endTime) {
return getJSONStream(
`/zones/${zoneId}/logs/requests?start=${startTime}&end=${endTime}`
)
return getJSONStream(`/zones/${zoneId}/logs/requests?start=${startTime}&end=${endTime}`)
}
module.exports = {

5
server/RedisClient.js
View File

@ -1,9 +1,8 @@
const redis = require('redis')
const redis = require("redis")
redis.debug_mode = process.env.DEBUG_REDIS != null
const RedisURL =
process.env.OPENREDIS_URL || process.env.REDIS_URL || 'redis://localhost:6379'
const RedisURL = process.env.OPENREDIS_URL || process.env.REDIS_URL || "redis://localhost:6379"
const client = redis.createClient(RedisURL)

28
server/StatsAPI.js
View File

@ -1,6 +1,6 @@
const db = require('./RedisClient')
const CloudflareAPI = require('./CloudflareAPI')
const BlacklistAPI = require('./BlacklistAPI')
const db = require("./RedisClient")
const CloudflareAPI = require("./CloudflareAPI")
const BlacklistAPI = require("./BlacklistAPI")
function prunePackages(packagesMap) {
return Promise.all(
@ -38,7 +38,7 @@ function createScoresMap(array) {
function getScoresMap(key, n = 100) {
return new Promise((resolve, reject) => {
db.zrevrange(key, 0, n, 'withscores', (error, value) => {
db.zrevrange(key, 0, n, "withscores", (error, value) => {
if (error) {
reject(error)
} else {
@ -49,15 +49,11 @@ function getScoresMap(key, n = 100) {
}
function getPackageRequests(date, n = 100) {
return getScoresMap(`stats-packageRequests-${createDayKey(date)}`, n).then(
prunePackages
)
return getScoresMap(`stats-packageRequests-${createDayKey(date)}`, n).then(prunePackages)
}
function getPackageBandwidth(date, n = 100) {
return getScoresMap(`stats-packageBytes-${createDayKey(date)}`, n).then(
prunePackages
)
return getScoresMap(`stats-packageBytes-${createDayKey(date)}`, n).then(prunePackages)
}
function getProtocolRequests(date) {
@ -90,9 +86,7 @@ function sumMaps(maps) {
}
function addDailyMetrics(result) {
return Promise.all(
result.timeseries.map(addDailyMetricsToTimeseries)
).then(() => {
return Promise.all(result.timeseries.map(addDailyMetricsToTimeseries)).then(() => {
result.totals.requests.package = sumMaps(
result.timeseries.map(timeseries => {
return timeseries.requests.package
@ -140,15 +134,11 @@ function extractPublicInfo(data) {
}
}
const DomainNames = ['unpkg.com', 'npmcdn.com']
const DomainNames = ["unpkg.com", "npmcdn.com"]
function fetchStats(since, until) {
return CloudflareAPI.getZones(DomainNames).then(zones => {
return CloudflareAPI.getZoneAnalyticsDashboard(
zones,
since,
until
).then(dashboard => {
return CloudflareAPI.getZoneAnalyticsDashboard(zones, since, until).then(dashboard => {
return {
timeseries: dashboard.timeseries.map(extractPublicInfo),
totals: extractPublicInfo(dashboard.totals)

8
server/__tests__/AuthAPI-test.js
View File

@ -1,11 +1,11 @@
const AuthAPI = require('../AuthAPI')
const AuthAPI = require("../AuthAPI")
describe('Auth API', () => {
describe("Auth API", () => {
beforeEach(done => {
AuthAPI.removeAllRevokedTokens().then(() => done(), done)
})
it('creates tokens with the right scopes', done => {
it("creates tokens with the right scopes", done => {
const scopes = {
blacklist: {
add: true,
@ -24,7 +24,7 @@ describe('Auth API', () => {
})
})
it('refuses to verify revoked tokens', done => {
it("refuses to verify revoked tokens", done => {
const scopes = {}
AuthAPI.createToken(scopes).then(token => {

8
server/__tests__/BlacklistAPI-test.js
View File

@ -1,12 +1,12 @@
const BlacklistAPI = require('../BlacklistAPI')
const BlacklistAPI = require("../BlacklistAPI")
describe('Blacklist API', () => {
describe("Blacklist API", () => {
beforeEach(done => {
BlacklistAPI.removeAllPackages().then(() => done(), done)
})
it('adds and removes packages to/from the blacklist', done => {
const packageName = 'bad-package'
it("adds and removes packages to/from the blacklist", done => {
const packageName = "bad-package"
BlacklistAPI.addPackage(packageName).then(() => {
BlacklistAPI.getPackages().then(packageNames => {

124
server/__tests__/server-test.js
View File

@ -1,49 +1,49 @@
const request = require('supertest')
const createServer = require('../createServer')
const clearBlacklist = require('./utils/clearBlacklist')
const withBlacklist = require('./utils/withBlacklist')
const withRevokedToken = require('./utils/withRevokedToken')
const withToken = require('./utils/withToken')
const request = require("supertest")
const createServer = require("../createServer")
const clearBlacklist = require("./utils/clearBlacklist")
const withBlacklist = require("./utils/withBlacklist")
const withRevokedToken = require("./utils/withRevokedToken")
const withToken = require("./utils/withToken")
describe('The server', () => {
describe("The server", () => {
let server
beforeEach(() => {
server = createServer()
})
it('rejects invalid package names', done => {
it("rejects invalid package names", done => {
request(server)
.get('/_invalid/index.js')
.get("/_invalid/index.js")
.end((err, res) => {
expect(res.statusCode).toBe(403)
done()
})
})
it('redirects invalid query params', done => {
it("redirects invalid query params", done => {
request(server)
.get('/react?main=index&invalid')
.get("/react?main=index&invalid")
.end((err, res) => {
expect(res.statusCode).toBe(302)
expect(res.headers.location).toBe('/react?main=index')
expect(res.headers.location).toBe("/react?main=index")
done()
})
})
it('redirects /_meta to ?meta', done => {
it("redirects /_meta to ?meta", done => {
request(server)
.get('/_meta/react?main=index')
.get("/_meta/react?main=index")
.end((err, res) => {
expect(res.statusCode).toBe(302)
expect(res.headers.location).toBe('/react?main=index&meta')
expect(res.headers.location).toBe("/react?main=index&meta")
done()
})
})
it('does not serve blacklisted packages', done => {
withBlacklist(['bad-package'], () => {
it("does not serve blacklisted packages", done => {
withBlacklist(["bad-package"], () => {
request(server)
.get('/bad-package/index.js')
.get("/bad-package/index.js")
.end((err, res) => {
expect(res.statusCode).toBe(403)
done()
@ -51,37 +51,37 @@ describe('The server', () => {
})
})
describe('POST /_auth', () => {
it('creates a new auth token', done => {
describe("POST /_auth", () => {
it("creates a new auth token", done => {
request(server)
.post('/_auth')
.post("/_auth")
.end((err, res) => {
expect(res.body).toHaveProperty('token')
expect(res.body).toHaveProperty("token")
done()
})
})
})
describe('GET /_auth', () => {
describe('with no auth', () => {
it('echoes back null', done => {
describe("GET /_auth", () => {
describe("with no auth", () => {
it("echoes back null", done => {
request(server)
.get('/_auth')
.get("/_auth")
.end((err, res) => {
expect(res.body).toHaveProperty('auth')
expect(res.body).toHaveProperty("auth")
expect(res.body.auth).toBe(null)
done()
})
})
})
describe('with a revoked auth token', () => {
it('echoes back null', done => {
describe("with a revoked auth token", () => {
it("echoes back null", done => {
withRevokedToken({ some: { scope: true } }, token => {
request(server)
.get('/_auth?token=' + token)
.get("/_auth?token=" + token)
.end((err, res) => {
expect(res.body).toHaveProperty('auth')
expect(res.body).toHaveProperty("auth")
expect(res.body.auth).toBe(null)
done()
})
@ -89,14 +89,14 @@ describe('The server', () => {
})
})
describe('with a valid auth token', () => {
it('echoes back the auth payload', done => {
describe("with a valid auth token", () => {
it("echoes back the auth payload", done => {
withToken({ some: { scope: true } }, token => {
request(server)
.get('/_auth?token=' + token)
.get("/_auth?token=" + token)
.end((err, res) => {
expect(res.body).toHaveProperty('auth')
expect(typeof res.body.auth).toBe('object')
expect(res.body).toHaveProperty("auth")
expect(typeof res.body.auth).toBe("object")
done()
})
})
@ -104,10 +104,10 @@ describe('The server', () => {
})
})
describe('GET /_publicKey', () => {
it('echoes the public key', done => {
describe("GET /_publicKey", () => {
it("echoes the public key", done => {
request(server)
.get('/_publicKey')
.get("/_publicKey")
.end((err, res) => {
expect(res.text).toMatch(/PUBLIC KEY/)
done()
@ -115,13 +115,13 @@ describe('The server', () => {
})
})
describe('POST /_blacklist', () => {
describe("POST /_blacklist", () => {
afterEach(clearBlacklist)
describe('with no auth', () => {
it('is forbidden', done => {
describe("with no auth", () => {
it("is forbidden", done => {
request(server)
.post('/_blacklist')
.post("/_blacklist")
.end((err, res) => {
expect(res.statusCode).toBe(403)
done()
@ -130,16 +130,14 @@ describe('The server', () => {
})
describe('with the "blacklist.add" scope', () => {
it('can add to the blacklist', done => {
it("can add to the blacklist", done => {
withToken({ blacklist: { add: true } }, token => {
request(server)
.post('/_blacklist')
.send({ token, packageName: 'bad-package' })
.post("/_blacklist")
.send({ token, packageName: "bad-package" })
.end((err, res) => {
expect(res.statusCode).toBe(200)
expect(res.headers['content-location']).toEqual(
'/_blacklist/bad-package'
)
expect(res.headers["content-location"]).toEqual("/_blacklist/bad-package")
expect(res.body.ok).toBe(true)
done()
})
@ -148,11 +146,11 @@ describe('The server', () => {
})
})
describe('GET /_blacklist', () => {
describe('with no auth', () => {
it('is forbidden', done => {
describe("GET /_blacklist", () => {
describe("with no auth", () => {
it("is forbidden", done => {
request(server)
.get('/_blacklist')
.get("/_blacklist")
.end((err, res) => {
expect(res.statusCode).toBe(403)
done()
@ -161,10 +159,10 @@ describe('The server', () => {
})
describe('with the "blacklist.read" scope', () => {
it('can read the blacklist', done => {
it("can read the blacklist", done => {
withToken({ blacklist: { read: true } }, token => {
request(server)
.get('/_blacklist?token=' + token)
.get("/_blacklist?token=" + token)
.end((err, res) => {
expect(res.statusCode).toBe(200)
done()
@ -174,11 +172,11 @@ describe('The server', () => {
})
})
describe('DELETE /_blacklist/:packageName', () => {
describe('with no auth', () => {
it('is forbidden', done => {
describe("DELETE /_blacklist/:packageName", () => {
describe("with no auth", () => {
it("is forbidden", done => {
request(server)
.delete('/_blacklist/bad-package')
.delete("/_blacklist/bad-package")
.end((err, res) => {
expect(res.statusCode).toBe(403)
done()
@ -187,10 +185,10 @@ describe('The server', () => {
})
describe('with the "blacklist.remove" scope', () => {
it('can remove a package from the blacklist', done => {
it("can remove a package from the blacklist", done => {
withToken({ blacklist: { remove: true } }, token => {
request(server)
.delete('/_blacklist/bad-package')
.delete("/_blacklist/bad-package")
.send({ token })
.end((err, res) => {
expect(res.statusCode).toBe(200)
@ -200,10 +198,10 @@ describe('The server', () => {
})
})
it('can remove a scoped package from the blacklist', done => {
it("can remove a scoped package from the blacklist", done => {
withToken({ blacklist: { remove: true } }, token => {
request(server)
.delete('/_blacklist/@scope/bad-package')
.delete("/_blacklist/@scope/bad-package")
.send({ token })
.end((err, res) => {
expect(res.statusCode).toBe(200)

2
server/__tests__/utils/clearBlacklist.js
View File

@ -1,4 +1,4 @@
const BlacklistAPI = require('../../BlacklistAPI')
const BlacklistAPI = require("../../BlacklistAPI")
function clearBlacklist(done) {
BlacklistAPI.removeAllPackages().then(done, done)

2
server/__tests__/utils/withBlacklist.js
View File

@ -1,4 +1,4 @@
const BlacklistAPI = require('../../BlacklistAPI')
const BlacklistAPI = require("../../BlacklistAPI")
function withBlacklist(blacklist, callback) {
return Promise.all(blacklist.map(BlacklistAPI.addPackage)).then(callback)

4
server/__tests__/utils/withRevokedToken.js
View File

@ -1,5 +1,5 @@
const withToken = require('./withToken')
const AuthAPI = require('../../AuthAPI')
const withToken = require("./withToken")
const AuthAPI = require("../../AuthAPI")
function withRevokedToken(scopes, callback) {
withToken(scopes, token => {

2
server/__tests__/utils/withToken.js
View File

@ -1,4 +1,4 @@
const AuthAPI = require('../../AuthAPI')
const AuthAPI = require("../../AuthAPI")
function withToken(scopes, callback) {
AuthAPI.createToken(scopes).then(callback)

20
server/actions/addToBlacklist.js
View File

@ -1,20 +1,18 @@
const validateNpmPackageName = require('validate-npm-package-name')
const BlacklistAPI = require('../BlacklistAPI')
const validateNpmPackageName = require("validate-npm-package-name")
const BlacklistAPI = require("../BlacklistAPI")
function addToBlacklist(req, res) {
const packageName = req.body.packageName
if (!packageName) {
return res
.status(403)
.send({ error: 'Missing "packageName" body parameter' })
return res.status(403).send({ error: 'Missing "packageName" body parameter' })
}
const nameErrors = validateNpmPackageName(packageName).errors
// Disallow invalid package names.
if (nameErrors) {
const reason = nameErrors.join(', ')
const reason = nameErrors.join(", ")
return res.status(403).send({
error: `Invalid package name "${packageName}" (${reason})`
})
@ -24,16 +22,12 @@ function addToBlacklist(req, res) {
added => {
if (added) {
const userId = req.user.jti
console.log(
`Package "${packageName}" was added to the blacklist by ${userId}`
)
console.log(`Package "${packageName}" was added to the blacklist by ${userId}`)
}
res.set({ 'Content-Location': `/_blacklist/${packageName}` }).send({
res.set({ "Content-Location": `/_blacklist/${packageName}` }).send({
ok: true,
message: `Package "${packageName}" was ${added
? 'added to'
: 'already in'} the blacklist`
message: `Package "${packageName}" was ${added ? "added to" : "already in"} the blacklist`
})
},
error => {

4
server/actions/createAuth.js
View File

@ -1,4 +1,4 @@
const AuthAPI = require('../AuthAPI')
const AuthAPI = require("../AuthAPI")
const defaultScopes = {
blacklist: {
@ -15,7 +15,7 @@ function createAuth(req, res) {
console.error(error)
res.status(500).send({
error: 'Unable to generate auth token'
error: "Unable to generate auth token"
})
}
)

12
server/actions/removeFromBlacklist.js
View File

@ -1,5 +1,5 @@
const validateNpmPackageName = require('validate-npm-package-name')
const BlacklistAPI = require('../BlacklistAPI')
const validateNpmPackageName = require("validate-npm-package-name")
const BlacklistAPI = require("../BlacklistAPI")
function removeFromBlacklist(req, res) {
const packageName = req.packageName
@ -8,16 +8,12 @@ function removeFromBlacklist(req, res) {
removed => {
if (removed) {
const userId = req.user.jti
console.log(
`Package "${packageName}" was removed from the blacklist by ${userId}`
)
console.log(`Package "${packageName}" was removed from the blacklist by ${userId}`)
}
res.send({
ok: true,
message: `Package "${packageName}" was ${
removed ? 'removed from' : 'not in'
} the blacklist`
message: `Package "${packageName}" was ${removed ? "removed from" : "not in"} the blacklist`
})
},
error => {

4
server/actions/showBlacklist.js
View File

@ -1,4 +1,4 @@
const BlacklistAPI = require('../BlacklistAPI')
const BlacklistAPI = require("../BlacklistAPI")
function showBlacklist(req, res) {
BlacklistAPI.getPackages().then(
@ -8,7 +8,7 @@ function showBlacklist(req, res) {
error => {
console.error(error)
res.status(500).send({
error: 'Unable to fetch blacklist'
error: "Unable to fetch blacklist"
})
}
)

2
server/actions/showPublicKey.js
View File

@ -1,4 +1,4 @@
const AuthAPI = require('../AuthAPI')
const AuthAPI = require("../AuthAPI")
function showPublicKey(req, res) {
res.send({ publicKey: AuthAPI.getPublicKey() })

34
server/actions/showStats.js
View File

@ -1,60 +1,56 @@
const subDays = require('date-fns/sub_days')
const startOfDay = require('date-fns/start_of_day')
const startOfSecond = require('date-fns/start_of_second')
const StatsAPI = require('../StatsAPI')
const subDays = require("date-fns/sub_days")
const startOfDay = require("date-fns/start_of_day")
const startOfSecond = require("date-fns/start_of_second")
const StatsAPI = require("../StatsAPI")
function showStats(req, res) {
let since, until
switch (req.query.period) {
case 'last-day':
case "last-day":
until = startOfDay(new Date())
since = subDays(until, 1)
break
case 'last-week':
case "last-week":
until = startOfDay(new Date())
since = subDays(until, 7)
break
case 'last-month':
case "last-month":
until = startOfDay(new Date())
since = subDays(until, 30)
break
default:
until = req.query.until
? new Date(req.query.until)
: startOfSecond(new Date())
until = req.query.until ? new Date(req.query.until) : startOfSecond(new Date())
since = new Date(req.query.since)
}
if (isNaN(since.getTime())) {
return res.status(403).send({ error: '?since is not a valid date' })
return res.status(403).send({ error: "?since is not a valid date" })
}
if (isNaN(until.getTime())) {
return res.status(403).send({ error: '?until is not a valid date' })
return res.status(403).send({ error: "?until is not a valid date" })
}
if (until <= since) {
return res
.status(403)
.send({ error: '?until date must come after ?since date' })
return res.status(403).send({ error: "?until date must come after ?since date" })
}
if (until >= new Date()) {
return res.status(403).send({ error: '?until must be a date in the past' })
return res.status(403).send({ error: "?until must be a date in the past" })
}
StatsAPI.getStats(since, until).then(
stats => {
res
.set({
'Cache-Control': 'public, max-age=60',
'Cache-Tag': 'stats'
"Cache-Control": "public, max-age=60",
"Cache-Tag": "stats"
})
.send(stats)
},
error => {
console.error(error)
res.status(500).send({ error: 'Unable to fetch stats' })
res.status(500).send({ error: "Unable to fetch stats" })
}
)
}

78
server/createServer.js
View File

@ -1,20 +1,20 @@
const fs = require('fs')
const path = require('path')
const express = require('express')
const bodyParser = require('body-parser')
const cors = require('cors')
const morgan = require('morgan')
const fs = require("fs")
const path = require("path")
const express = require("express")
const bodyParser = require("body-parser")
const cors = require("cors")
const morgan = require("morgan")
const checkBlacklist = require('./middleware/checkBlacklist')
const fetchFile = require('./middleware/fetchFile')
const parseURL = require('./middleware/parseURL')
const requireAuth = require('./middleware/requireAuth')
const serveFile = require('./middleware/serveFile')
const userToken = require('./middleware/userToken')
const validatePackageURL = require('./middleware/validatePackageURL')
const checkBlacklist = require("./middleware/checkBlacklist")
const fetchFile = require("./middleware/fetchFile")
const parseURL = require("./middleware/parseURL")
const requireAuth = require("./middleware/requireAuth")
const serveFile = require("./middleware/serveFile")
const userToken = require("./middleware/userToken")
const validatePackageURL = require("./middleware/validatePackageURL")
morgan.token('fwd', function(req) {
return req.get('x-forwarded-for').replace(/\s/g, '')
morgan.token("fwd", function(req) {
return req.get("x-forwarded-for").replace(/\s/g, "")
})
function errorHandler(err, req, res, next) {
@ -22,8 +22,8 @@ function errorHandler(err, req, res, next) {
res
.status(500)
.type('text')
.send('Internal Server Error')
.type("text")
.send("Internal Server Error")
next(err)
}
@ -37,16 +37,16 @@ function createRouter(setup) {
function createServer() {
const app = express()
app.disable('x-powered-by')
app.disable("x-powered-by")
if (process.env.NODE_ENV !== 'test') {
if (process.env.NODE_ENV !== "test") {
app.use(
morgan(
process.env.NODE_ENV === 'production'
process.env.NODE_ENV === "production"
? // Modified version of the Heroku router's log format
// https://devcenter.heroku.com/articles/http-routing#heroku-router-log-format
'method=:method path=":url" host=:req[host] request_id=:req[x-request-id] cf_ray=:req[cf-ray] fwd=:fwd status=:status bytes=:res[content-length]'
: 'dev'
: "dev"
)
)
}
@ -54,8 +54,8 @@ function createServer() {
app.use(errorHandler)
app.use(
express.static('build', {
maxAge: '365d'
express.static("build", {
maxAge: "365d"
})
)
@ -63,43 +63,35 @@ function createServer() {
app.use(bodyParser.json())
app.use(userToken)
app.get('/_publicKey', require('./actions/showPublicKey'))
app.get("/_publicKey", require("./actions/showPublicKey"))
app.use(
'/_auth',
"/_auth",
createRouter(app => {
app.post('/', require('./actions/createAuth'))
app.get('/', require('./actions/showAuth'))
app.post("/", require("./actions/createAuth"))
app.get("/", require("./actions/showAuth"))
})
)
app.use(
'/_blacklist',
"/_blacklist",
createRouter(app => {
app.post(
'/',
requireAuth('blacklist.add'),
require('./actions/addToBlacklist')
)
app.get(
'/',
requireAuth('blacklist.read'),
require('./actions/showBlacklist')
)
app.post("/", requireAuth("blacklist.add"), require("./actions/addToBlacklist"))
app.get("/", requireAuth("blacklist.read"), require("./actions/showBlacklist"))
app.delete(
/.*/,
requireAuth('blacklist.remove'),
requireAuth("blacklist.remove"),
validatePackageURL,
require('./actions/removeFromBlacklist')
require("./actions/removeFromBlacklist")
)
})
)
if (process.env.NODE_ENV !== 'test') {
app.get('/_stats', require('./actions/showStats'))
if (process.env.NODE_ENV !== "test") {
app.get("/_stats", require("./actions/showStats"))
}
app.use('/', parseURL, checkBlacklist, fetchFile, serveFile)
app.use("/", parseURL, checkBlacklist, fetchFile, serveFile)
return app
}

59
server/ingestLogs.js
View File

@ -1,17 +1,17 @@
const parseURL = require('url').parse
const startOfDay = require('date-fns/start_of_day')
const addDays = require('date-fns/add_days')
const parsePackageURL = require('./utils/parsePackageURL')
const CloudflareAPI = require('./CloudflareAPI')
const StatsAPI = require('./StatsAPI')
const parseURL = require("url").parse
const startOfDay = require("date-fns/start_of_day")
const addDays = require("date-fns/add_days")
const parsePackageURL = require("./utils/parsePackageURL")
const CloudflareAPI = require("./CloudflareAPI")
const StatsAPI = require("./StatsAPI")
const db = require('./RedisClient')
const db = require("./RedisClient")
/**
* Domains we want to analyze.
*/
const DomainNames = [
'unpkg.com'
"unpkg.com"
//'npmcdn.com' // We don't have log data on npmcdn.com yet :/
]
@ -48,8 +48,8 @@ function computeCounters(stream) {
}
stream
.on('error', reject)
.on('data', function(entry) {
.on("error", reject)
.on("data", function(entry) {
const date = new Date(Math.round(entry.timestamp / 1000000))
const nextDay = startOfDay(addDays(date, 1))
@ -67,26 +67,15 @@ function computeCounters(stream) {
const packageName = url && url.packageName
if (packageName) {
incr(
`stats-packageRequests-${dayKey}`,
packageName,
1,
thirtyDaysLater
)
incr(
`stats-packageBytes-${dayKey}`,
packageName,
edgeResponse.bytes,
thirtyDaysLater
)
incr(`stats-packageRequests-${dayKey}`, packageName, 1, thirtyDaysLater)
incr(`stats-packageBytes-${dayKey}`, packageName, edgeResponse.bytes, thirtyDaysLater)
}
}
// Q: How many requests per day do we receive via a protocol?
const protocol = clientRequest.httpProtocol
if (protocol)
incr(`stats-protocolRequests-${dayKey}`, protocol, 1, thirtyDaysLater)
if (protocol) incr(`stats-protocolRequests-${dayKey}`, protocol, 1, thirtyDaysLater)
// Q: How many requests do we receive from a hostname per day?
// Q: How many bytes do we serve to a hostname per day?
@ -95,15 +84,10 @@ function computeCounters(stream) {
if (hostname) {
incr(`stats-hostnameRequests-${dayKey}`, hostname, 1, sevenDaysLater)
incr(
`stats-hostnameBytes-${dayKey}`,
hostname,
edgeResponse.bytes,
sevenDaysLater
)
incr(`stats-hostnameBytes-${dayKey}`, hostname, edgeResponse.bytes, sevenDaysLater)
}
})
.on('end', function() {
.on("end", function() {
resolve({ counters, expireat })
})
})
@ -126,7 +110,7 @@ function processLogs(stream) {
function ingestLogs(zone, startSeconds, endSeconds) {
return new Promise(resolve => {
console.log(
'info: Started ingesting logs for %s from %s to %s',
"info: Started ingesting logs for %s from %s to %s",
zone.name,
stringifySeconds(startSeconds),
stringifySeconds(endSeconds)
@ -139,7 +123,7 @@ function ingestLogs(zone, startSeconds, endSeconds) {
const endFetchTime = Date.now()
console.log(
'info: Fetched %ds worth of logs for %s in %dms',
"info: Fetched %ds worth of logs for %s in %dms",
endSeconds - startSeconds,
zone.name,
endFetchTime - startFetchTime
@ -151,7 +135,7 @@ function ingestLogs(zone, startSeconds, endSeconds) {
const endProcessTime = Date.now()
console.log(
'info: Processed %ds worth of logs for %s in %dms',
"info: Processed %ds worth of logs for %s in %dms",
endSeconds - startSeconds,
zone.name,
endProcessTime - startProcessTime
@ -163,10 +147,7 @@ function ingestLogs(zone, startSeconds, endSeconds) {
}
function startZone(zone) {
const startSecondsKey = `ingestLogsWorker-nextStartSeconds-${zone.name.replace(
'.',
'-'
)}`
const startSecondsKey = `ingestLogsWorker-nextStartSeconds-${zone.name.replace(".", "-")}`
function takeATurn() {
db.get(startSecondsKey, function(error, value) {
@ -182,7 +163,7 @@ function startZone(zone) {
startSeconds = minSeconds
} else if (startSeconds < minSeconds) {
console.warn(
'warning: Dropped logs for %s from %s to %s!',
"warning: Dropped logs for %s from %s to %s!",
zone.name,
stringifySeconds(startSeconds),
stringifySeconds(minSeconds)

6
server/middleware/checkBlacklist.js
View File

@ -1,4 +1,4 @@
const BlacklistAPI = require('../BlacklistAPI')
const BlacklistAPI = require("../BlacklistAPI")
function checkBlacklist(req, res, next) {
BlacklistAPI.includesPackage(req.packageName).then(
@ -7,7 +7,7 @@ function checkBlacklist(req, res, next) {
if (blacklisted) {
res
.status(403)
.type('text')
.type("text")
.send(`Package "${req.packageName}" is blacklisted`)
} else {
next()
@ -17,7 +17,7 @@ function checkBlacklist(req, res, next) {
console.error(error)
res.status(500).send({
error: 'Unable to fetch the blacklist'
error: "Unable to fetch the blacklist"
})
}
)

50
server/middleware/components/DirectoryListing.js
View File

@ -1,6 +1,6 @@
const React = require('react')
const prettyBytes = require('pretty-bytes')
const getFileContentType = require('../utils/getFileContentType')
const React = require("react")
const prettyBytes = require("pretty-bytes")
const getFileContentType = require("../utils/getFileContentType")
const e = React.createElement
@ -9,46 +9,46 @@ const formatTime = time => new Date(time).toISOString()
const DirectoryListing = ({ dir, entries }) => {
const rows = entries.map(({ file, stats }, index) => {
const isDir = stats.isDirectory()
const href = file + (isDir ? '/' : '')
const href = file + (isDir ? "/" : "")
return e(
'tr',
{ key: file, className: index % 2 ? 'odd' : 'even' },
e('td', null, e('a', { title: file, href }, file)),
e('td', null, isDir ? '-' : getFileContentType(file)),
e('td', null, isDir ? '-' : prettyBytes(stats.size)),
e('td', null, isDir ? '-' : formatTime(stats.mtime))
"tr",
{ key: file, className: index % 2 ? "odd" : "even" },
e("td", null, e("a", { title: file, href }, file)),
e("td", null, isDir ? "-" : getFileContentType(file)),
e("td", null, isDir ? "-" : prettyBytes(stats.size)),
e("td", null, isDir ? "-" : formatTime(stats.mtime))
)
})
if (dir !== '/')
if (dir !== "/")
rows.unshift(
e(
'tr',
{ key: '..', className: 'odd' },
e('td', null, e('a', { title: 'Parent directory', href: '../' }, '..')),
e('td', null, '-'),
e('td', null, '-'),
e('td', null, '-')
"tr",
{ key: "..", className: "odd" },
e("td", null, e("a", { title: "Parent directory", href: "../" }, "..")),
e("td", null, "-"),
e("td", null, "-"),
e("td", null, "-")
)
)
return e(
'table',
"table",
null,
e(
'thead',
"thead",
null,
e(
'tr',
"tr",
null,
e('th', null, 'Name'),
e('th', null, 'Type'),
e('th', null, 'Size'),
e('th', null, 'Last Modified')
e("th", null, "Name"),
e("th", null, "Type"),
e("th", null, "Size"),
e("th", null, "Last Modified")
)
),
e('tbody', null, rows)
e("tbody", null, rows)
)
}

46
server/middleware/components/IndexPage.js
View File

@ -1,11 +1,11 @@
const React = require('react')
const semver = require('semver')
const DirectoryListing = require('./DirectoryListing')
const readCSS = require('../utils/readCSS')
const React = require("react")
const semver = require("semver")
const DirectoryListing = require("./DirectoryListing")
const readCSS = require("../utils/readCSS")
const e = React.createElement
const IndexPageStyle = readCSS(__dirname, 'IndexPage.css')
const IndexPageStyle = readCSS(__dirname, "IndexPage.css")
const IndexPageScript = `
var s = document.getElementById('version'), v = s.value
s.onchange = function () {
@ -17,37 +17,35 @@ const byVersion = (a, b) => (semver.lt(a, b) ? -1 : semver.gt(a, b) ? 1 : 0)
const IndexPage = ({ packageInfo, version, dir, entries }) => {
const versions = Object.keys(packageInfo.versions).sort(byVersion)
const options = versions.map(v =>
e('option', { key: v, value: v }, `${packageInfo.name}@${v}`)
)
const options = versions.map(v => e("option", { key: v, value: v }, `${packageInfo.name}@${v}`))
return e(
'html',
"html",
null,
e(
'head',
"head",
null,
e('meta', { charSet: 'utf-8' }),
e('title', null, `Index of ${dir}`),
e('style', { dangerouslySetInnerHTML: { __html: IndexPageStyle } })
e("meta", { charSet: "utf-8" }),
e("title", null, `Index of ${dir}`),
e("style", { dangerouslySetInnerHTML: { __html: IndexPageStyle } })
),
e(
'body',
"body",
null,
e(
'div',
{ className: 'content-wrapper' },
"div",
{ className: "content-wrapper" },
e(
'div',
{ className: 'version-wrapper' },
e('select', { id: 'version', defaultValue: version }, options)
"div",
{ className: "version-wrapper" },
e("select", { id: "version", defaultValue: version }, options)
),
e('h1', null, `Index of ${dir}`),
e('script', { dangerouslySetInnerHTML: { __html: IndexPageScript } }),
e('hr'),
e("h1", null, `Index of ${dir}`),
e("script", { dangerouslySetInnerHTML: { __html: IndexPageScript } }),
e("hr"),
e(DirectoryListing, { dir, entries }),
e('hr'),
e('address', null, `${packageInfo.name}@${version}`)
e("hr"),
e("address", null, `${packageInfo.name}@${version}`)
)
)
)

109
server/middleware/fetchFile.js
View File

@ -1,11 +1,11 @@
const fs = require('fs')
const path = require('path')
const semver = require('semver')
const createPackageURL = require('../utils/createPackageURL')
const createSearch = require('./utils/createSearch')
const getPackageInfo = require('./utils/getPackageInfo')
const getPackage = require('./utils/getPackage')
const incrementCounter = require('./utils/incrementCounter')
const fs = require("fs")
const path = require("path")
const semver = require("semver")
const createPackageURL = require("../utils/createPackageURL")
const createSearch = require("./utils/createSearch")
const getPackageInfo = require("./utils/getPackageInfo")
const getPackage = require("./utils/getPackage")
const incrementCounter = require("./utils/incrementCounter")
function getBasename(file) {
return path.basename(file, path.extname(file))
@ -14,7 +14,7 @@ function getBasename(file) {
/**
* File extensions to look for when automatically resolving.
*/
const FindExtensions = ['', '.js', '.json']
const FindExtensions = ["", ".js", ".json"]
/**
* Resolves a path like "lib/file" into "lib/file.js" or "lib/file.json"
@ -27,17 +27,13 @@ function findFile(base, useIndex, callback) {
return function() {
fs.stat(file, function(error, stats) {
if (error) {
if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
if (error.code === "ENOENT" || error.code === "ENOTDIR") {
next()
} else {
callback(error)
}
} else if (useIndex && stats.isDirectory()) {
findFile(path.join(file, 'index'), false, function(
error,
indexFile,
indexStats
) {
findFile(path.join(file, "index"), false, function(error, indexFile, indexStats) {
if (error) {
callback(error)
} else if (indexFile) {
@ -65,14 +61,14 @@ function fetchFile(req, res, next) {
console.error(error)
return res
.status(500)
.type('text')
.type("text")
.send(`Cannot get info for package "${req.packageName}"`)
}
if (packageInfo == null || packageInfo.versions == null)
return res
.status(404)
.type('text')
.type("text")
.send(`Cannot find package "${req.packageName}"`)
req.packageInfo = packageInfo
@ -86,7 +82,7 @@ function fetchFile(req, res, next) {
console.error(error)
res
.status(500)
.type('text')
.type("text")
.send(`Cannot fetch package ${req.packageSpec}`)
} else {
req.packageDir = outputDir
@ -98,77 +94,56 @@ function fetchFile(req, res, next) {
// They want an ES module. Try "module", "jsnext:main", and "/"
// https://github.com/rollup/rollup/wiki/pkg.module
if (!filename)
filename =
req.packageConfig.module ||
req.packageConfig['jsnext:main'] ||
'/'
filename = req.packageConfig.module || req.packageConfig["jsnext:main"] || "/"
} else if (filename) {
// They are requesting an explicit filename. Only try to find an
// index file if they are NOT requesting an HTML directory listing.
useIndex = filename[filename.length - 1] !== '/'
} else if (
req.query.main &&
typeof req.packageConfig[req.query.main] === 'string'
) {
useIndex = filename[filename.length - 1] !== "/"
} else if (req.query.main && typeof req.packageConfig[req.query.main] === "string") {
// They specified a custom ?main field.
filename = req.packageConfig[req.query.main]
incrementCounter(
'package-json-custom-main',
req.packageSpec + '?main=' + req.query.main,
"package-json-custom-main",
req.packageSpec + "?main=" + req.query.main,
1
)
} else if (typeof req.packageConfig.unpkg === 'string') {
} else if (typeof req.packageConfig.unpkg === "string") {
// The "unpkg" field allows packages to explicitly declare the
// file to serve at the bare URL (see #59).
filename = req.packageConfig.unpkg
} else if (typeof req.packageConfig.browser === 'string') {
} else if (typeof req.packageConfig.browser === "string") {
// Fall back to the "browser" field if declared (only support strings).
filename = req.packageConfig.browser
// Count which packages + versions are actually using this fallback
// so we can warn them when we deprecate this functionality.
// See https://github.com/unpkg/unpkg/issues/63
incrementCounter(
'package-json-browser-fallback',
req.packageSpec,
1
)
incrementCounter("package-json-browser-fallback", req.packageSpec, 1)
} else {
// Fall back to "main" or / (same as npm).
filename = req.packageConfig.main || '/'
filename = req.packageConfig.main || "/"
}
findFile(path.join(req.packageDir, filename), useIndex, function(
error,
file,
stats
) {
findFile(path.join(req.packageDir, filename), useIndex, function(error, file, stats) {
if (error) console.error(error)
if (file == null)
return res
.status(404)
.type('text')
.send(
`Cannot find module "${filename}" in package ${
req.packageSpec
}`
)
.type("text")
.send(`Cannot find module "${filename}" in package ${req.packageSpec}`)
filename = file.replace(req.packageDir, '')
filename = file.replace(req.packageDir, "")
if (
req.query.main != null ||
getBasename(req.filename) !== getBasename(filename)
) {
if (req.query.main != null || getBasename(req.filename) !== getBasename(filename)) {
// Need to redirect to the module file so relative imports resolve
// correctly. Cache module redirects for 1 minute.
delete req.query.main
res
.set({
'Cache-Control': 'public, max-age=60',
'Cache-Tag': 'redirect,module-redirect'
"Cache-Control": "public, max-age=60",
"Cache-Tag": "redirect,module-redirect"
})
.redirect(
302,
@ -187,18 +162,18 @@ function fetchFile(req, res, next) {
})
}
})
} else if (req.packageVersion in req.packageInfo['dist-tags']) {
} else if (req.packageVersion in req.packageInfo["dist-tags"]) {
// Cache tag redirects for 1 minute.
res
.set({
'Cache-Control': 'public, max-age=60',
'Cache-Tag': 'redirect,tag-redirect'
"Cache-Control": "public, max-age=60",
"Cache-Tag": "redirect,tag-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
req.packageInfo['dist-tags'][req.packageVersion],
req.packageInfo["dist-tags"][req.packageVersion],
req.filename,
req.search
)
@ -213,22 +188,14 @@ function fetchFile(req, res, next) {
// Cache semver redirects for 1 minute.
res
.set({
'Cache-Control': 'public, max-age=60',
'Cache-Tag': 'redirect,semver-redirect'
"Cache-Control": "public, max-age=60",
"Cache-Tag": "redirect,semver-redirect"
})
.redirect(
302,
createPackageURL(
req.packageName,
maxVersion,
req.filename,
req.search
)
)
.redirect(302, createPackageURL(req.packageName, maxVersion, req.filename, req.search))
} else {
res
.status(404)
.type('text')
.type("text")
.send(`Cannot find package ${req.packageSpec}`)
}
}

16
server/middleware/parseURL.js
View File

@ -1,6 +1,6 @@
const validateNpmPackageName = require('validate-npm-package-name')
const parsePackageURL = require('../utils/parsePackageURL')
const createSearch = require('./utils/createSearch')
const validateNpmPackageName = require("validate-npm-package-name")
const parsePackageURL = require("../utils/parsePackageURL")
const createSearch = require("./utils/createSearch")
const KnownQueryParams = {
main: true,
@ -32,14 +32,14 @@ function sanitizeQuery(query) {
function parseURL(req, res, next) {
// Redirect /_meta/path to /path?meta.
if (req.path.match(/^\/_meta\//)) {
req.query.meta = ''
req.query.meta = ""
return res.redirect(302, req.path.substr(6) + createSearch(req.query))
}
// Redirect /path?json => /path?meta
if (req.query.json != null) {
delete req.query.json
req.query.meta = ''
req.query.meta = ""
return res.redirect(302, req.path + createSearch(req.query))
}
@ -56,7 +56,7 @@ function parseURL(req, res, next) {
if (url == null) {
return res
.status(403)
.type('text')
.type("text")
.send(`Invalid URL: ${req.url}`)
}
@ -64,10 +64,10 @@ function parseURL(req, res, next) {
// Disallow invalid package names.
if (nameErrors) {
const reason = nameErrors.join(', ')
const reason = nameErrors.join(", ")
return res
.status(403)
.type('text')
.type("text")
.send(`Invalid package name "${url.packageName}" (${reason})`)
}

11
server/middleware/requireAuth.js
View File

@ -4,10 +4,9 @@
*/
function requireAuth(scope) {
let checkScopes
if (scope.includes('.')) {
const parts = scope.split('.')
checkScopes = scopes =>
parts.reduce((memo, part) => memo && memo[part], scopes) != null
if (scope.includes(".")) {
const parts = scope.split(".")
checkScopes = scopes => parts.reduce((memo, part) => memo && memo[part], scopes) != null
} else {
checkScopes = scopes => scopes[scope] != null
}
@ -20,11 +19,11 @@ function requireAuth(scope) {
const user = req.user
if (!user) {
return res.status(403).send({ error: 'Missing auth token' })
return res.status(403).send({ error: "Missing auth token" })
}
if (!user.scopes || !checkScopes(user.scopes)) {
return res.status(403).send({ error: 'Insufficient scopes' })
return res.status(403).send({ error: "Insufficient scopes" })
}
if (req.auth) {

114
server/middleware/serveFile.js
View File

@ -1,11 +1,11 @@
const fs = require('fs')
const path = require('path')
const etag = require('etag')
const babel = require('babel-core')
const unpkgRewrite = require('babel-plugin-unpkg-rewrite')
const getMetadata = require('./utils/getMetadata')
const getFileContentType = require('./utils/getFileContentType')
const getIndexHTML = require('./utils/getIndexHTML')
const fs = require("fs")
const path = require("path")
const etag = require("etag")
const babel = require("babel-core")
const unpkgRewrite = require("babel-plugin-unpkg-rewrite")
const getMetadata = require("./utils/getMetadata")
const getFileContentType = require("./utils/getFileContentType")
const getIndexHTML = require("./utils/getIndexHTML")
/**
* Automatically generate HTML pages that show package contents.
@ -35,24 +35,19 @@ const FileTransforms = {
function serveFile(req, res, next) {
if (req.query.meta != null) {
// Serve JSON metadata.
getMetadata(req.packageDir, req.filename, req.stats, MaximumDepth, function(
error,
metadata
) {
getMetadata(req.packageDir, req.filename, req.stats, MaximumDepth, function(error, metadata) {
if (error) {
console.error(error)
res
.status(500)
.type('text')
.send(
`Cannot generate metadata for ${req.packageSpec}${req.filename}`
)
.type("text")
.send(`Cannot generate metadata for ${req.packageSpec}${req.filename}`)
} else {
// Cache metadata for 1 year.
res
.set({
'Cache-Control': 'public, max-age=31536000',
'Cache-Tag': 'meta'
"Cache-Control": "public, max-age=31536000",
"Cache-Tag": "meta"
})
.send(metadata)
}
@ -63,9 +58,9 @@ function serveFile(req, res, next) {
let contentType = getFileContentType(file)
if (contentType === 'text/html') contentType = 'text/plain' // We can't serve HTML because bad people :(
if (contentType === "text/html") contentType = "text/plain" // We can't serve HTML because bad people :(
if (contentType === 'application/javascript' && req.query.module != null) {
if (contentType === "application/javascript" && req.query.module != null) {
// Serve a JavaScript module.
const dependencies = Object.assign(
{},
@ -78,33 +73,29 @@ function serveFile(req, res, next) {
console.error(error)
const debugInfo =
error.constructor.name +
': ' +
": " +
error.message.replace(/^.*?\/unpkg-.+?\//, `/${req.packageSpec}/`) +
'\n\n' +
"\n\n" +
error.codeFrame
res
.status(500)
.type('text')
.send(
`Cannot generate module for ${req.packageSpec}${
req.filename
}\n\n${debugInfo}`
)
.type("text")
.send(`Cannot generate module for ${req.packageSpec}${req.filename}\n\n${debugInfo}`)
} else {
// Cache modules for 1 year.
res
.set({
'Content-Type': contentType,
'Content-Length': Buffer.byteLength(code),
'Cache-Control': 'public, max-age=31536000',
'Cache-Tag': 'file,js-file,js-module'
"Content-Type": contentType,
"Content-Length": Buffer.byteLength(code),
"Cache-Control": "public, max-age=31536000",
"Cache-Tag": "file,js-file,js-module"
})
.send(code)
}
})
} else {
// Serve some other static file.
const tags = ['file']
const tags = ["file"]
const ext = path.extname(req.filename).substr(1)
@ -112,17 +103,17 @@ function serveFile(req, res, next) {
// Cache files for 1 year.
res.set({
'Content-Type': contentType,
'Content-Length': req.stats.size,
'Cache-Control': 'public, max-age=31536000',
'Last-Modified': req.stats.mtime.toUTCString(),
"Content-Type": contentType,
"Content-Length": req.stats.size,
"Cache-Control": "public, max-age=31536000",
"Last-Modified": req.stats.mtime.toUTCString(),
ETag: etag(req.stats),
'Cache-Tag': tags.join(',')
"Cache-Tag": tags.join(",")
})
const stream = fs.createReadStream(file)
stream.on('error', function(error) {
stream.on("error", function(error) {
console.error(`Cannot send file ${req.packageSpec}${req.filename}`)
console.error(error)
res.sendStatus(500)
@ -132,35 +123,30 @@ function serveFile(req, res, next) {
}
} else if (AutoIndex && req.stats.isDirectory()) {
// Serve an HTML directory listing.
getIndexHTML(
req.packageInfo,
req.packageVersion,
req.packageDir,
req.filename,
function(error, html) {
if (error) {
console.error(error)
res
.status(500)
.type('text')
.send(
`Cannot generate index page for ${req.packageSpec}${req.filename}`
)
} else {
// Cache HTML directory listings for 1 minute.
res
.set({
'Cache-Control': 'public, max-age=60',
'Cache-Tag': 'index'
})
.send(html)
}
getIndexHTML(req.packageInfo, req.packageVersion, req.packageDir, req.filename, function(
error,
html
) {
if (error) {
console.error(error)
res
.status(500)
.type("text")
.send(`Cannot generate index page for ${req.packageSpec}${req.filename}`)
} else {
// Cache HTML directory listings for 1 minute.
res
.set({
"Cache-Control": "public, max-age=60",
"Cache-Tag": "index"
})
.send(html)
}
)
})
} else {
res
.status(403)
.type('text')
.type("text")
.send(`Cannot serve ${req.packageSpec}${req.filename}; it's not a file`)
}
}

6
server/middleware/userToken.js
View File

@ -1,4 +1,4 @@
const AuthAPI = require('../AuthAPI')
const AuthAPI = require("../AuthAPI")
const ReadMethods = { GET: true, HEAD: true }
@ -23,7 +23,7 @@ function userToken(req, res, next) {
next()
},
error => {
if (error.name === 'JsonWebTokenError') {
if (error.name === "JsonWebTokenError") {
res.status(403).send({
error: `Bad auth token: ${error.message}`
})
@ -31,7 +31,7 @@ function userToken(req, res, next) {
console.error(error)
res.status(500).send({
error: 'Unable to verify auth'
error: "Unable to verify auth"
})
}
}

46
server/middleware/utils/__tests__/getFileContentType-test.js
View File

@ -1,35 +1,35 @@
const getFileContentType = require('../getFileContentType')
const getFileContentType = require("../getFileContentType")
it('gets a content type of text/plain for LICENSE|README|CHANGES|AUTHORS|Makefile', () => {
expect(getFileContentType('AUTHORS')).toBe('text/plain')
expect(getFileContentType('CHANGES')).toBe('text/plain')
expect(getFileContentType('LICENSE')).toBe('text/plain')
expect(getFileContentType('Makefile')).toBe('text/plain')
expect(getFileContentType('PATENTS')).toBe('text/plain')
expect(getFileContentType('README')).toBe('text/plain')
it("gets a content type of text/plain for LICENSE|README|CHANGES|AUTHORS|Makefile", () => {
expect(getFileContentType("AUTHORS")).toBe("text/plain")
expect(getFileContentType("CHANGES")).toBe("text/plain")
expect(getFileContentType("LICENSE")).toBe("text/plain")
expect(getFileContentType("Makefile")).toBe("text/plain")
expect(getFileContentType("PATENTS")).toBe("text/plain")
expect(getFileContentType("README")).toBe("text/plain")
})
it('gets a content type of text/plain for .*rc files', () => {
expect(getFileContentType('.eslintrc')).toBe('text/plain')
expect(getFileContentType('.babelrc')).toBe('text/plain')
expect(getFileContentType('.anythingrc')).toBe('text/plain')
it("gets a content type of text/plain for .*rc files", () => {
expect(getFileContentType(".eslintrc")).toBe("text/plain")
expect(getFileContentType(".babelrc")).toBe("text/plain")
expect(getFileContentType(".anythingrc")).toBe("text/plain")
})
it('gets a content type of text/plain for .git* files', () => {
expect(getFileContentType('.gitignore')).toBe('text/plain')
expect(getFileContentType('.gitanything')).toBe('text/plain')
it("gets a content type of text/plain for .git* files", () => {
expect(getFileContentType(".gitignore")).toBe("text/plain")
expect(getFileContentType(".gitanything")).toBe("text/plain")
})
it('gets a content type of text/plain for .*ignore files', () => {
expect(getFileContentType('.eslintignore')).toBe('text/plain')
expect(getFileContentType('.anythingignore')).toBe('text/plain')
it("gets a content type of text/plain for .*ignore files", () => {
expect(getFileContentType(".eslintignore")).toBe("text/plain")
expect(getFileContentType(".anythingignore")).toBe("text/plain")
})
it('gets a content type of text/plain for .ts files', () => {
expect(getFileContentType('app.ts')).toBe('text/plain')
expect(getFileContentType('app.d.ts')).toBe('text/plain')
it("gets a content type of text/plain for .ts files", () => {
expect(getFileContentType("app.ts")).toBe("text/plain")
expect(getFileContentType("app.d.ts")).toBe("text/plain")
})
it('gets a content type of text/plain for .flow files', () => {
expect(getFileContentType('app.js.flow')).toBe('text/plain')
it("gets a content type of text/plain for .flow files", () => {
expect(getFileContentType("app.js.flow")).toBe("text/plain")
})

4
server/middleware/utils/createCache.js
View File

@ -1,8 +1,8 @@
const db = require('../../RedisClient')
const db = require("../../RedisClient")
function createCache(keyPrefix) {
function createKey(key) {
return keyPrefix + '-' + key
return keyPrefix + "-" + key
}
function set(key, value, expiry, callback) {

6
server/middleware/utils/createSearch.js
View File

@ -2,16 +2,16 @@ function createSearch(query) {
const params = []
Object.keys(query).forEach(param => {
if (query[param] === '') {
if (query[param] === "") {
params.push(param) // Omit the trailing "=" from param=
} else {
params.push(`${param}=${encodeURIComponent(query[param])}`)
}
})
const search = params.join('&')
const search = params.join("&")
return search ? `?${search}` : ''
return search ? `?${search}` : ""
}
module.exports = createSearch

15
server/middleware/utils/getFileContentType.js
View File

@ -1,22 +1,13 @@
const mime = require('mime')
const mime = require("mime")
mime.define({
'text/plain': [
'authors',
'changes',
'license',
'makefile',
'patents',
'readme',
'ts',
'flow'
]
"text/plain": ["authors", "changes", "license", "makefile", "patents", "readme", "ts", "flow"]
})
const TextFiles = /\/?(\.[a-z]*rc|\.git[a-z]*|\.[a-z]*ignore)$/i
function getFileContentType(file) {
return TextFiles.test(file) ? 'text/plain' : mime.lookup(file)
return TextFiles.test(file) ? "text/plain" : mime.lookup(file)
}
module.exports = getFileContentType

2
server/middleware/utils/getFileStats.js
View File

@ -1,4 +1,4 @@
const fs = require('fs')
const fs = require("fs")
function getFileStats(file) {
return new Promise((resolve, reject) => {

16
server/middleware/utils/getFileType.js
View File

@ -1,12 +1,12 @@
function getFileType(stats) {
if (stats.isFile()) return 'file'
if (stats.isDirectory()) return 'directory'
if (stats.isBlockDevice()) return 'blockDevice'
if (stats.isCharacterDevice()) return 'characterDevice'
if (stats.isSymbolicLink()) return 'symlink'
if (stats.isSocket()) return 'socket'
if (stats.isFIFO()) return 'fifo'
return 'unknown'
if (stats.isFile()) return "file"
if (stats.isDirectory()) return "directory"
if (stats.isBlockDevice()) return "blockDevice"
if (stats.isCharacterDevice()) return "characterDevice"
if (stats.isSymbolicLink()) return "symlink"
if (stats.isSocket()) return "socket"
if (stats.isFIFO()) return "fifo"
return "unknown"
}
module.exports = getFileType

18
server/middleware/utils/getIndexHTML.js
View File

@ -1,9 +1,9 @@
const fs = require('fs')
const path = require('path')
const React = require('react')
const ReactDOMServer = require('react-dom/server')
const getFileStats = require('./getFileStats')
const IndexPage = require('../components/IndexPage')
const fs = require("fs")
const path = require("path")
const React = require("react")
const ReactDOMServer = require("react-dom/server")
const getFileStats = require("./getFileStats")
const IndexPage = require("../components/IndexPage")
const e = React.createElement
@ -14,9 +14,7 @@ function getEntries(dir) {
reject(error)
} else {
resolve(
Promise.all(
files.map(file => getFileStats(path.join(dir, file)))
).then(statsArray => {
Promise.all(files.map(file => getFileStats(path.join(dir, file)))).then(statsArray => {
return statsArray.map((stats, index) => {
return { file: files[index], stats }
})
@ -27,7 +25,7 @@ function getEntries(dir) {
})
}
const DOCTYPE = '<!DOCTYPE html>'
const DOCTYPE = "<!DOCTYPE html>"
function createHTML(props) {
return DOCTYPE + ReactDOMServer.renderToStaticMarkup(e(IndexPage, props))

35
server/middleware/utils/getMetadata.js
View File

@ -1,9 +1,9 @@
const fs = require('fs')
const path = require('path')
const SRIToolbox = require('sri-toolbox')
const getFileContentType = require('./getFileContentType')
const getFileStats = require('./getFileStats')
const getFileType = require('./getFileType')
const fs = require("fs")
const path = require("path")
const SRIToolbox = require("sri-toolbox")
const getFileContentType = require("./getFileContentType")
const getFileStats = require("./getFileStats")
const getFileType = require("./getFileType")
function getEntries(dir, file, maximumDepth) {
return new Promise((resolve, reject) => {
@ -12,17 +12,10 @@ function getEntries(dir, file, maximumDepth) {
reject(error)
} else {
resolve(
Promise.all(
files.map(f => getFileStats(path.join(dir, file, f)))
).then(statsArray => {
Promise.all(files.map(f => getFileStats(path.join(dir, file, f)))).then(statsArray => {
return Promise.all(
statsArray.map((stats, index) =>
getMetadataRecursive(
dir,
path.join(file, files[index]),
stats,
maximumDepth - 1
)
getMetadataRecursive(dir, path.join(file, files[index]), stats, maximumDepth - 1)
)
)
})
@ -42,7 +35,7 @@ function getIntegrity(file) {
if (error) {
reject(error)
} else {
resolve(SRIToolbox.generate({ algorithms: ['sha384'] }, data))
resolve(SRIToolbox.generate({ algorithms: ["sha384"] }, data))
}
})
})
@ -64,8 +57,7 @@ function getMetadataRecursive(dir, file, stats, maximumDepth) {
})
}
if (!stats.isDirectory() || maximumDepth === 0)
return Promise.resolve(metadata)
if (!stats.isDirectory() || maximumDepth === 0) return Promise.resolve(metadata)
return getEntries(dir, file, maximumDepth).then(files => {
metadata.files = files
@ -74,12 +66,9 @@ function getMetadataRecursive(dir, file, stats, maximumDepth) {
}
function getMetadata(baseDir, path, stats, maximumDepth, callback) {
getMetadataRecursive(baseDir, path, stats, maximumDepth).then(function(
metadata
) {
getMetadataRecursive(baseDir, path, stats, maximumDepth).then(function(metadata) {
callback(null, metadata)
},
callback)
}, callback)
}
module.exports = getMetadata

28
server/middleware/utils/getPackage.js
View File

@ -1,14 +1,14 @@
require('isomorphic-fetch')
const fs = require('fs')
const path = require('path')
const tmpdir = require('os-tmpdir')
const gunzip = require('gunzip-maybe')
const mkdirp = require('mkdirp')
const tar = require('tar-fs')
const createMutex = require('./createMutex')
require("isomorphic-fetch")
const fs = require("fs")
const path = require("path")
const tmpdir = require("os-tmpdir")
const gunzip = require("gunzip-maybe")
const mkdirp = require("mkdirp")
const tar = require("tar-fs")
const createMutex = require("./createMutex")
function createTempPath(name, version) {
const normalName = name.replace(/\//g, '-')
const normalName = name.replace(/\//g, "-")
return path.join(tmpdir(), `unpkg-${normalName}-${version}`)
}
@ -17,12 +17,12 @@ function stripNamePrefix(headers) {
// so we shorten that to just "index.js" here. A few packages use a
// prefix other than "package/". e.g. the firebase package uses the
// "firebase_npm/" prefix. So we just strip the first dir name.
headers.name = headers.name.replace(/^[^\/]+\//, '')
headers.name = headers.name.replace(/^[^\/]+\//, "")
return headers
}
function ignoreSymlinks(file, headers) {
return headers.type === 'link'
return headers.type === "link"
}
function extractResponse(response, outputDir) {
@ -36,8 +36,8 @@ function extractResponse(response, outputDir) {
response.body
.pipe(gunzip())
.pipe(extract)
.on('finish', resolve)
.on('error', reject)
.on("finish", resolve)
.on("error", reject)
})
}
@ -54,7 +54,7 @@ const fetchMutex = createMutex((payload, callback) => {
fs.access(outputDir, function(error) {
if (error) {
if (error.code === 'ENOENT' || error.code === 'ENOTDIR') {
if (error.code === "ENOENT" || error.code === "ENOTDIR") {
// ENOENT or ENOTDIR are to be expected when we haven't yet
// fetched a package for the first time. Carry on!
mkdirp(outputDir, function(error) {

16
server/middleware/utils/getPackageInfo.js
View File

@ -1,16 +1,16 @@
require('isomorphic-fetch')
const createCache = require('./createCache')
const createMutex = require('./createMutex')
require("isomorphic-fetch")
const createCache = require("./createCache")
const createMutex = require("./createMutex")
const RegistryURL = process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org'
const RegistryURL = process.env.NPM_REGISTRY_URL || "https://registry.npmjs.org"
const PackageInfoCache = createCache('packageInfo')
const PackageInfoCache = createCache("packageInfo")
function fetchPackageInfo(packageName) {
console.log(`info: Fetching package info for ${packageName}`)
let encodedPackageName
if (packageName.charAt(0) === '@') {
if (packageName.charAt(0) === "@") {
encodedPackageName = `@${encodeURIComponent(packageName.substring(1))}`
} else {
encodedPackageName = encodeURIComponent(packageName)
@ -20,14 +20,14 @@ function fetchPackageInfo(packageName) {
return fetch(url, {
headers: {
Accept: 'application/json'
Accept: "application/json"
}
}).then(res => {
return res.status === 404 ? null : res.json()
})
}
const PackageNotFound = 'PackageNotFound'
const PackageNotFound = "PackageNotFound"
// This mutex prevents multiple concurrent requests to
// the registry for the same package info.

2
server/middleware/utils/incrementCounter.js
View File

@ -1,4 +1,4 @@
const db = require('../../RedisClient')
const db = require("../../RedisClient")
function incrementCounter(counter, key, by) {
return new Promise((resolve, reject) => {

8
server/middleware/utils/readCSS.js
View File

@ -1,9 +1,9 @@
const fs = require('fs')
const path = require('path')
const csso = require('csso')
const fs = require("fs")
const path = require("path")
const csso = require("csso")
function readCSS(...args) {
return csso.minify(fs.readFileSync(path.resolve(...args), 'utf8')).css
return csso.minify(fs.readFileSync(path.resolve(...args), "utf8")).css
}
module.exports = readCSS

2
server/middleware/validatePackageURL.js
View File

@ -1,4 +1,4 @@
const parsePackageURL = require('../utils/parsePackageURL')
const parsePackageURL = require("../utils/parsePackageURL")
/**
* Adds various properties to the request object to do with the

110
server/utils/__tests__/parsePackageURL-test.js
View File

@ -1,85 +1,85 @@
const parsePackageURL = require('../parsePackageURL')
const parsePackageURL = require("../parsePackageURL")
describe('parsePackageURL', () => {
it('parses plain packages', () => {
expect(parsePackageURL('/history@1.0.0/umd/history.min.js')).toEqual({
pathname: '/history@1.0.0/umd/history.min.js',
search: '',
describe("parsePackageURL", () => {
it("parses plain packages", () => {
expect(parsePackageURL("/history@1.0.0/umd/history.min.js")).toEqual({
pathname: "/history@1.0.0/umd/history.min.js",
search: "",
query: {},
packageName: 'history',
packageVersion: '1.0.0',
filename: '/umd/history.min.js'
packageName: "history",
packageVersion: "1.0.0",
filename: "/umd/history.min.js"
})
})
it('parses plain packages with a hyphen in the name', () => {
expect(parsePackageURL('/query-string@5.0.0/index.js')).toEqual({
pathname: '/query-string@5.0.0/index.js',
search: '',
it("parses plain packages with a hyphen in the name", () => {
expect(parsePackageURL("/query-string@5.0.0/index.js")).toEqual({
pathname: "/query-string@5.0.0/index.js",
search: "",
query: {},
packageName: 'query-string',
packageVersion: '5.0.0',
filename: '/index.js'
packageName: "query-string",
packageVersion: "5.0.0",
filename: "/index.js"
})
})
it('parses plain packages with no version specified', () => {
expect(parsePackageURL('/query-string/index.js')).toEqual({
pathname: '/query-string/index.js',
search: '',
it("parses plain packages with no version specified", () => {
expect(parsePackageURL("/query-string/index.js")).toEqual({
pathname: "/query-string/index.js",
search: "",
query: {},
packageName: 'query-string',
packageVersion: 'latest',
filename: '/index.js'
packageName: "query-string",
packageVersion: "latest",
filename: "/index.js"
})
})
it('parses plain packages with version spec', () => {
expect(parsePackageURL('/query-string@>=4.0.0/index.js')).toEqual({
pathname: '/query-string@>=4.0.0/index.js',
search: '',
it("parses plain packages with version spec", () => {
expect(parsePackageURL("/query-string@>=4.0.0/index.js")).toEqual({
pathname: "/query-string@>=4.0.0/index.js",
search: "",
query: {},
packageName: 'query-string',
packageVersion: '>=4.0.0',
filename: '/index.js'
packageName: "query-string",
packageVersion: ">=4.0.0",
filename: "/index.js"
})
})
it('parses scoped packages', () => {
expect(parsePackageURL('/@angular/router@4.3.3/src/index.d.ts')).toEqual({
pathname: '/@angular/router@4.3.3/src/index.d.ts',
search: '',
it("parses scoped packages", () => {
expect(parsePackageURL("/@angular/router@4.3.3/src/index.d.ts")).toEqual({
pathname: "/@angular/router@4.3.3/src/index.d.ts",
search: "",
query: {},
packageName: '@angular/router',
packageVersion: '4.3.3',
filename: '/src/index.d.ts'
packageName: "@angular/router",
packageVersion: "4.3.3",
filename: "/src/index.d.ts"
})
})
it('parses package names with a period in them', () => {
expect(parsePackageURL('/index.js')).toEqual({
pathname: '/index.js',
search: '',
it("parses package names with a period in them", () => {
expect(parsePackageURL("/index.js")).toEqual({
pathname: "/index.js",
search: "",
query: {},
packageName: 'index.js',
packageVersion: 'latest',
filename: ''
packageName: "index.js",
packageVersion: "latest",
filename: ""
})
})
it('parses valid query parameters', () => {
expect(parsePackageURL('/history?main=browser')).toEqual({
pathname: '/history',
search: '?main=browser',
query: { main: 'browser' },
packageName: 'history',
packageVersion: 'latest',
filename: ''
it("parses valid query parameters", () => {
expect(parsePackageURL("/history?main=browser")).toEqual({
pathname: "/history",
search: "?main=browser",
query: { main: "browser" },
packageName: "history",
packageVersion: "latest",
filename: ""
})
})
it('returns null for invalid pathnames', () => {
expect(parsePackageURL('history')).toBe(null)
expect(parsePackageURL('/.invalid')).toBe(null)
it("returns null for invalid pathnames", () => {
expect(parsePackageURL("history")).toBe(null)
expect(parsePackageURL("/.invalid")).toBe(null)
})
})

8
server/utils/parsePackageURL.js
View File

@ -1,5 +1,5 @@
const url = require('url')
const validatePackageName = require('./validatePackageName')
const url = require("url")
const validatePackageName = require("./validatePackageName")
const URLFormat = /^\/((?:@[^\/@]+\/)?[^\/@]+)(?:@([^\/]+))?(\/.*)?$/
@ -12,7 +12,7 @@ function decodeParam(param) {
}
}
return ''
return ""
}
function parsePackageURL(packageURL) {
@ -28,7 +28,7 @@ function parsePackageURL(packageURL) {
// Disallow invalid npm package names.
if (!validatePackageName(packageName)) return null
const packageVersion = decodeParam(match[2]) || 'latest'
const packageVersion = decodeParam(match[2]) || "latest"
const filename = decodeParam(match[3])
return {

2
server/utils/validatePackageName.js
View File

@ -1,4 +1,4 @@
const validateNpmPackageName = require('validate-npm-package-name')
const validateNpmPackageName = require("validate-npm-package-name")
function validatePackageName(packageName) {
return validateNpmPackageName(packageName).errors == null