Add /_stats endpoint

Also, remove ingest_stats worker and use the cache instead.

.travis.yml

@@ -1,10 +1,12 @@
 language: node_js
 node_js:
-- stable
+  - stable
 cache: yarn
 branches:
   only:
-  - master
+    - master
+services:
+  - redis-server
 before_deploy: yarn build
 deploy:
   provider: heroku

Procfile

@@ -1,3 +1,2 @@
 web: node server.js
 ingest_logs: node server/IngestLogsWorker.js
-ingest_stats: node server/IngestStatsWorker.js

client/About.js

@@ -1,8 +1,7 @@
 import React from 'react'
 import contentHTML from './About.md'
 
-function About() {
+const About = () =>
-  return <div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }}/>
+  <div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }}/>
-}
 
 export default About

client/About.md

@@ -1,12 +1,4 @@
-unpkg is an [open source](https://github.com/mjackson/unpkg) project built by me, [Michael Jackson](https://twitter.com/mjackson). I built it because, as an npm package author, it felt tedious for me to use existing, git-based CDNs to make my open source work available via CDN. Development was sponsored by my company, [React Training](https://reacttraining.com).
+unpkg is an [open source](https://github.com/unpkg) project built and maintained by [Michael Jackson](https://twitter.com/mjackson).
-
-<div class="about-logos">
-  <div class="about-logo">
-    <a href="https://reacttraining.com"><img src="./ReactTrainingLogo.png"></a>
-  </div>
-</div>
-
-We'd love to talk to you more about training your team on [React](https://facebook.github.io/react/). Please [get in touch](mailto:hello@reacttraining.com) if interested.
 
 ### Sponsors
 
@@ -21,8 +13,6 @@ The fast, global infrastructure that powers unpkg is generously donated by [Clou
   </div>
 </div>
 
-These sponsors provide some of the most robust, reliable infrastructure available today and I'm happy to be able to partner with them on unpkg.
-
 ### Cache Behavior
 
 The CDN caches all files based on their permanent URL, which includes the npm package version. This works because npm does not allow package authors to overwrite a package that has already been published with a different one at the same version number.

client/App.js

@@ -1,20 +1,10 @@
 import React from 'react'
-import { HashRouter as Router, Switch, Route } from 'react-router-dom'
+import { HashRouter } from 'react-router-dom'
 import Layout from './Layout'
-import About from './About'
-import Stats from './Stats'
-import Home from './Home'
 
-const App = () => (
+const App = () =>
-  <Router>
+  <HashRouter>
-    <Layout>
+    <Layout/>
-      <Switch>
+  </HashRouter>
-        <Route path="/stats" component={Stats}/>
-        <Route path="/about" component={About}/>
-        <Route path="/" component={Home}/>
-      </Switch>
-    </Layout>
-  </Router>
-)
 
 export default App

client/CountryUtils.js

@@ -1,13 +0,0 @@
-import {
-  continents as ContinentsIndex,
-  countries as CountriesIndex
-} from 'countries-list'
-
-const getCountriesByContinent = (continent) =>
-  Object.keys(CountriesIndex).filter(country => CountriesIndex[country].continent === continent)
-
-export {
-  ContinentsIndex,
-  CountriesIndex,
-  getCountriesByContinent
-}

client/DOMUtils.js

@@ -1,15 +0,0 @@
-export const addEvent = (node, type, handler) => {
-  if (node.addEventListener) {
-    node.addEventListener(type, handler, false)
-  } else if (node.attachEvent) {
-    node.attachEvent('on' + type, handler)
-  }
-}
-
-export const removeEvent = (node, type, handler) => {
-  if (node.removeEventListener) {
-    node.removeEventListener(type, handler, false)
-  } else if (node.detachEvent) {
-    node.detachEvent('on' + type, handler)
-  }
-}

client/Home.js

@@ -1,8 +1,7 @@
 import React from 'react'
 import contentHTML from './Home.md'
 
-function Home() {
+const Home = () =>
-  return <div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }}/>
+  <div className="wrapper" dangerouslySetInnerHTML={{ __html: contentHTML }}/>
-}
 
 export default Home

client/Home.md

@@ -1,60 +1,43 @@
-unpkg is a fast, global [content delivery network](https://en.wikipedia.org/wiki/Content_delivery_network) for everything on [npm](https://www.npmjs.com/). Use it to quickly and easily load files using a simple URL like:
+unpkg is a fast, global [content delivery network](https://en.wikipedia.org/wiki/Content_delivery_network) for everything on [npm](https://www.npmjs.com/). Use it to quickly and easily load any file from any package using a URL like:
 
-<div style="text-align:center">`https://unpkg.com/package@version/file`</div>
+<div class="home-example">unpkg.com/:package@:version/:file</div>
 
-A few examples:
+### Examples
 
-  * [https://unpkg.com/react@15.3.1/dist/react.min.js](/react@15.3.1/dist/react.min.js)
+Using a fixed version:
-  * [https://unpkg.com/react-dom@15.3.1/dist/react-dom.min.js](/react-dom@15.3.1/dist/react-dom.min.js)
-  * [https://unpkg.com/history@4.2.0/umd/history.min.js](/history@4.2.0/umd/history.min.js)
 
-You may also use a [tag](https://docs.npmjs.com/cli/dist-tag) or [version range](https://docs.npmjs.com/misc/semver) instead of a fixed version number, or omit the version/tag entirely to use the `latest` tag.
+  * [unpkg.com/react@15.3.1/dist/react.min.js](/react@15.3.1/dist/react.min.js)
+  * [unpkg.com/react-dom@15.3.1/dist/react-dom.min.js](/react-dom@15.3.1/dist/react-dom.min.js)
 
-  * [https://unpkg.com/react@^0.14/dist/react.min.js](/react@^0.14/dist/react.min.js)
+You may also use a [semver range](https://docs.npmjs.com/misc/semver) or a [tag](https://docs.npmjs.com/cli/dist-tag) instead of a fixed version number, or omit the version/tag entirely to use the `latest` tag.
-  * [https://unpkg.com/react/dist/react.min.js](/react/dist/react.min.js)
 
-If you omit the file path, unpkg will try to serve [the `browser` bundle](https://github.com/defunctzombie/package-browser-field-spec) if present, the [`main` module](https://docs.npmjs.com/files/package.json#main) otherwise.
+  * [unpkg.com/react@^15/dist/react.min.js](/react@^15/dist/react.min.js)
+  * [unpkg.com/react/dist/react.min.js](/react/dist/react.min.js)
 
-  * [https://unpkg.com/jquery](/jquery)
+If you omit the file path, unpkg will serve the package's "main" file.
-  * [https://unpkg.com/angular-formly](/angular-formly)
+
-  * [https://unpkg.com/three](/three)
+  * [unpkg.com/jquery](/jquery)
+  * [unpkg.com/three](/three)
 
 Append a `/` at the end of a URL to view a listing of all the files in a package.
 
-  * [https://unpkg.com/lodash/](/lodash/)
+  * [unpkg.com/react/](/react/)
-  * [https://unpkg.com/modernizr/](/modernizr/)
+  * [unpkg.com/lodash/](/lodash/)
-  * [https://unpkg.com/react/](/react/)
 
 ### Query Parameters
 
-<table cellpadding="0" cellspacing="0">
+<dl>
-  <thead>
+  <dt>`?main=:mainField`</dt>
-    <tr>
+  <dd>The name of the field in [package.json](https://docs.npmjs.com/files/package.json) to use as the main entry point when there is no file path in the URL. Defaults to using `unpkg`, [`browser`](https://github.com/defunctzombie/package-browser-field-spec), and then [`main`](https://docs.npmjs.com/files/package.json#main).</dd>
-      <th width="80px">Name</th>
-      <th width="120px">Default Value</th>
-      <th>Description</th>
-    </tr>
-  </thead>
-  <tbody>
-    <tr>
-      <td>`main`</td>
-      <td>`unpkg`, `browser`, `main`</td>
-      <td>The name of the field in [package.json](https://docs.npmjs.com/files/package.json) to use as the main entry point when there is no file path in the URL</td>
-    </tr>
-    <tr>
-      <td>`meta`</td>
-      <td></td>
-      <td>Return metadata about any file in a package as JSON (e.g. `/any/file?meta`)</td>
-    </tr>
-    <tr>
-      <td>`expand`</td>
-      <td></td>
-      <td>Expands all ["bare" `import` specifiers](https://html.spec.whatwg.org/multipage/webappapis.html#resolve-a-module-specifier) in JavaScript modules to unpkg URLs. This feature is *very experimental*</td>
-    </tr>
-  </tbody>
-</table>
 
-### Suggested Workflow
+  <dt>`?meta`</dt>
+  <dd>Return metadata about any file in a package as JSON (e.g. `/any/file?meta`)</dd>
+
+  <dt>`?module`</dt>
+  <dd>Expands all ["bare" `import` specifiers](https://html.spec.whatwg.org/multipage/webappapis.html#resolve-a-module-specifier) in JavaScript modules to unpkg URLs. This feature is *very experimental*</dd>
+</dl>
+
+### Workflow
 
 For npm package authors, unpkg relieves the burden of publishing your code to a CDN in addition to the npm registry. All you need to do is include your [UMD](https://github.com/umdjs/umd) build in your npm package (not your repo, that's different!).
 

client/Layout.js

@@ -1,9 +1,11 @@
 import React from 'react'
-import ReactDOM from 'react-dom'
 import PropTypes from 'prop-types'
 import { Motion, spring } from 'react-motion'
-import { withRouter, Link } from 'react-router-dom'
+import { Switch, Route, Link, withRouter } from 'react-router-dom'
 import WindowSize from './WindowSize'
+import About from './About'
+import Stats from './Stats'
+import Home from './Home'
 
 class Layout extends React.Component {
   static propTypes = {
@@ -14,24 +16,25 @@ class Layout extends React.Component {
   state = {
     underlineLeft: 0,
     underlineWidth: 0,
-    useSpring: false
+    useSpring: false,
+    stats: null
   }
 
   adjustUnderline = (useSpring = false) => {
     let itemIndex
     switch (this.props.location.pathname) {
-      case '/about':
-        itemIndex = 2
-        break
       case '/stats':
         itemIndex = 1
         break
+      case '/about':
+        itemIndex = 2
+        break
       case '/':
       default:
         itemIndex = 0
     }
 
-    const itemNodes = ReactDOM.findDOMNode(this).querySelectorAll('.underlist > li')
+    const itemNodes = this.listNode.querySelectorAll('li')
     const currentNode = itemNodes[itemIndex]
 
     this.setState({
@@ -43,6 +46,10 @@ class Layout extends React.Component {
 
   componentDidMount() {
     this.adjustUnderline()
+
+    fetch('/_stats/last-month')
+      .then(res => res.json())
+      .then(stats => this.setState({ stats }))
   }
 
   componentDidUpdate(prevProps) {
@@ -63,34 +70,36 @@ class Layout extends React.Component {
         <WindowSize onChange={this.adjustUnderline}/>
         <div className="wrapper">
           <header>
-            <h1>unpkg</h1>
+            <h1 className="layout-title">unpkg</h1>
-            <nav>
+            <nav className="layout-nav">
-              <ol className="underlist">
+              <ol className="layout-nav-list" ref={node => this.listNode = node}>
                 <li><Link to="/">Home</Link></li>
                 <li><Link to="/stats">Stats</Link></li>
                 <li><Link to="/about">About</Link></li>
               </ol>
-              <Motion defaultStyle={{ left: underlineLeft, width: underlineWidth }} style={style}>
+              <Motion
-                {s => (
+                defaultStyle={{ left: underlineLeft, width: underlineWidth }}
+                style={style}
+                children={style => (
                   <div
-                    className="underlist-underline"
+                    className="layout-nav-underline"
                     style={{
-                      WebkitTransform: `translate3d(${s.left}px,0,0)`,
+                      WebkitTransform: `translate3d(${style.left}px,0,0)`,
-                      transform: `translate3d(${s.left}px,0,0)`,
+                      transform: `translate3d(${style.left}px,0,0)`,
-                      width: s.width
+                      width: style.width
                     }}
                   />
                 )}
-              </Motion>
+              />
             </nav>
           </header>
         </div>
-        {this.props.children}
+
-        <div className="wrapper">
+        <Switch>
-          <footer>
+          <Route path="/stats" render={() => <Stats data={this.state.stats}/>}/>
-            <p>&copy; 2016-{(new Date()).getFullYear()} Michael Jackson</p>
+          <Route path="/about" component={About}/>
-          </footer>
+          <Route path="/" component={Home}/>
-        </div>
+        </Switch>
       </div>
     )
   }

client/NumberUtils.js

@@ -1,15 +0,0 @@
-export const formatNumber = (n) => {
-  const digits = String(n).split('')
-  const groups = []
-
-  while (digits.length)
-    groups.unshift(digits.splice(-3).join(''))
-
-  return groups.join(',')
-}
-
-export const parseNumber = (s) =>
-  parseInt(s.replace(/,/g, ''), 10) || 0
-
-export const formatPercent = (n, fixed = 1) =>
-  String((n.toPrecision(2) * 100).toFixed(fixed))

client/Stats.js

@@ -3,115 +3,125 @@ import PropTypes from 'prop-types'
 import formatBytes from 'pretty-bytes'
 import formatDate from 'date-fns/format'
 import parseDate from 'date-fns/parse'
-import { formatNumber, formatPercent } from './NumberUtils'
+import formatNumber from './utils/formatNumber'
-import { ContinentsIndex, CountriesIndex, getCountriesByContinent } from './CountryUtils'
+import formatPercent from './utils/formatPercent'
-import NumberTextInput from './NumberTextInput'
 
-const getSum = (data, countries) =>
+import { continents, countries } from 'countries-list'
-  countries.reduce((n, country) => n + (data[country] || 0), 0)
 
-const addValues = (a, b) => {
+const getCountriesByContinent = (continent) =>
-  for (const p in b) {
+  Object.keys(countries).filter(country => countries[country].continent === continent)
-    if (p in a) {
+
-      a[p] += b[p]
+const sumKeyValues = (hash, keys) =>
-    } else {
+  keys.reduce((n, key) => n + (hash[key] || 0), 0)
-      a[p] = b[p]
+
-    }
+const sumValues = (hash) =>
-  }
+  Object.keys(hash).reduce((memo, key) => memo + hash[key], 0)
-}
 
 class Stats extends React.Component {
   static propTypes = {
-    serverData: PropTypes.object
+    data: PropTypes.object
-  }
-
-  static defaultProps = {
-    serverData: window.serverData
   }
 
   state = {
-    minRequests: 5000000
+    minPackageRequests: 100000,
-  }
+    minCountryRequests: 1000000
-
-  updateMinRequests = (value) => {
-    this.setState({ minRequests: value })
   }
 
   render() {
-    const { minRequests } = this.state
+    const { data } = this.props
-    const stats = this.props.serverData.cloudflareStats
+
-    const { timeseries, totals } = stats
+    if (data == null)
+      return null
+
+    const totals = data.totals
 
     // Summary data
-    const sinceDate = parseDate(totals.since)
+    const since = parseDate(totals.since)
-    const untilDate = parseDate(totals.until)
+    const until = parseDate(totals.until)
-    const uniqueVisitors = totals.uniques.all
 
-    const totalRequests = totals.requests.all
+    // Packages
-    const cachedRequests = totals.requests.cached
+    const packageRows = []
-    const totalBandwidth = totals.bandwidth.all
-    const httpStatus = totals.requests.http_status
 
-    let errorRequests = 0
+    Object.keys(totals.requests.package).sort((a, b) => {
-    for (const status in httpStatus) {
+      return totals.requests.package[b] - totals.requests.package[a]
-      if (httpStatus.hasOwnProperty(status) && status >= 500)
+    }).forEach(packageName => {
-        errorRequests += httpStatus[status]
+      const requests = totals.requests.package[packageName]
-    }
+      const bandwidth = totals.bandwidth.package[packageName]
 
-    // By Region
+      if (requests >= this.state.minPackageRequests) {
-    const regionRows = []
+        packageRows.push(
-    const requestsByCountry = {}
+          <tr key={packageName}>
-    const bandwidthByCountry = {}
+            <td><a href={`https://npmjs.org/package/${packageName}`} title={`${packageName} on npm`}>{packageName}</a></td>
-
+            <td>{formatNumber(requests)} ({formatPercent(requests / totals.requests.all)}%)</td>
-    timeseries.forEach(ts => {
+            {bandwidth
-      addValues(requestsByCountry, ts.requests.country)
+              ? <td>{formatBytes(bandwidth)} ({formatPercent(bandwidth / totals.bandwidth.all)}%)</td>
-      addValues(bandwidthByCountry, ts.bandwidth.country)
+              : <td>-</td>
+            }
+          </tr>
+        )
+      }
     })
 
-    const byRequestsDescending = (a, b) =>
+    // Protocols
-      requestsByCountry[b] - requestsByCountry[a]
+    const protocolRows = Object.keys(totals.requests.protocol).sort((a, b) => {
+      return totals.requests.protocol[b] - totals.requests.protocol[a]
+    }).map(protocol => {
+      const requests = totals.requests.protocol[protocol]
 
-    const continentData = Object.keys(ContinentsIndex).reduce((memo, continent) => {
+      return (
-      const countries = getCountriesByContinent(continent)
+        <tr key={protocol}>
+          <td>{protocol}</td>
+          <td>{formatNumber(requests)} ({formatPercent(requests / sumValues(totals.requests.protocol))}%)</td>
+        </tr>
+      )
+    })
+
+    // Regions
+    const regionRows = []
+
+    const continentsData = Object.keys(continents).reduce((memo, continent) => {
+      const localCountries = getCountriesByContinent(continent)
 
       memo[continent] = {
-        countries,
+        countries: localCountries,
-        requests: getSum(requestsByCountry, countries),
+        requests: sumKeyValues(totals.requests.country, localCountries),
-        bandwidth: getSum(bandwidthByCountry, countries)
+        bandwidth: sumKeyValues(totals.bandwidth.country, localCountries)
       }
 
       return memo
     }, {})
 
-    const topContinents = Object.keys(continentData).sort((a, b) => {
+    const topContinents = Object.keys(continentsData).sort((a, b) => {
-      return continentData[b].requests - continentData[a].requests
+      return continentsData[b].requests - continentsData[a].requests
     })
 
     topContinents.forEach(continent => {
-      const continentName = ContinentsIndex[continent]
+      const continentName = continents[continent]
-      const { countries, requests, bandwidth } = continentData[continent]
+      const continentData = continentsData[continent]
 
-      if (bandwidth !== 0) {
+      if (continentData.requests > this.state.minCountryRequests && continentData.bandwidth !== 0) {
         regionRows.push(
           <tr key={continent} className="continent-row">
             <td>{continentName}</td>
-            <td>{formatNumber(requests)} ({formatPercent(requests / totalRequests)}%)</td>
+            <td>{formatNumber(continentData.requests)} ({formatPercent(continentData.requests / totals.requests.all)}%)</td>
-            <td>{formatBytes(bandwidth)} ({formatPercent(bandwidth / totalBandwidth)}%)</td>
+            <td>{formatBytes(continentData.bandwidth)} ({formatPercent(continentData.bandwidth / totals.bandwidth.all)}%)</td>
           </tr>
         )
 
-        const topCountries = countries.sort(byRequestsDescending)
+        const topCountries = continentData.countries.sort((a, b) => {
+          return totals.requests.country[b] - totals.requests.country[a]
+        })
 
         topCountries.forEach(country => {
-          const countryRequests = requestsByCountry[country]
+          const countryRequests = totals.requests.country[country]
-          const countryBandwidth = bandwidthByCountry[country]
+          const countryBandwidth = totals.bandwidth.country[country]
 
-          if (countryRequests > minRequests) {
+          if (countryRequests > this.state.minCountryRequests) {
             regionRows.push(
               <tr key={continent + country} className="country-row">
-                <td className="country-name">{CountriesIndex[country].name}</td>
+                <td className="country-name">{countries[country].name}</td>
-                <td>{formatNumber(countryRequests)} ({formatPercent(countryRequests / totalRequests)}%)</td>
+                <td>{formatNumber(countryRequests)} ({formatPercent(countryRequests / totals.requests.all)}%)</td>
-                <td>{formatBytes(countryBandwidth)} ({formatPercent(countryBandwidth / totalBandwidth)}%)</td>
+                <td>{formatBytes(countryBandwidth)} ({formatPercent(countryBandwidth / totals.bandwidth.all)}%)</td>
               </tr>
             )
           }
@@ -121,16 +131,67 @@ class Stats extends React.Component {
 
     return (
       <div className="wrapper">
-        <p>From <strong>{formatDate(sinceDate, 'MMM D')}</strong> to <strong>{formatDate(untilDate, 'MMM D')}</strong>, unpkg served <strong>{formatNumber(totalRequests)}</strong> requests to <strong>{formatNumber(uniqueVisitors)}</strong> unique visitors, <strong>{formatPercent(cachedRequests / totalRequests, 0)}%</strong> of which came from the cache (CDN). Over the same period, <strong>{formatPercent(errorRequests / totalRequests, 4)}%</strong> of requests resulted in server error (returned an HTTP status &ge; 500).</p>
+        <p>From <strong>{formatDate(since, 'MMM D')}</strong> to <strong>{formatDate(until, 'MMM D')}</strong> unpkg served <strong>{formatNumber(totals.requests.all)}</strong> requests and a total of <strong>{formatBytes(totals.bandwidth.all)}</strong> of data to <strong>{formatNumber(totals.uniques.all)}</strong> unique visitors, <strong>{formatPercent(totals.requests.cached / totals.requests.all, 0)}%</strong> of which were served from the cache.</p>
 
-        <h3>By Region</h3>
+        <h3>Packages</h3>
 
-        <label className="table-filter">Include countries that made at least <NumberTextInput value={minRequests} onChange={this.updateMinRequests}/> requests.</label>
+        <p className="table-filter">Include only packages that received at least <select
+            value={this.state.minPackageRequests}
+            onChange={event => this.setState({ minPackageRequests: parseInt(event.target.value, 10) })}
+          >
+            <option value="0">0</option>
+            <option value="1000">1,000</option>
+            <option value="10000">10,000</option>
+            <option value="100000">100,000</option>
+            <option value="1000000">1,000,000</option>
+          </select> requests.
+        </p>
 
         <table cellSpacing="0" cellPadding="0" style={{ width: '100%' }}>
           <thead>
             <tr>
-              <th>Name</th>
+              <th>Package</th>
+              <th>Requests (% of total)</th>
+              <th>Bandwidth (% of total)</th>
+            </tr>
+          </thead>
+          <tbody>
+            {packageRows}
+          </tbody>
+        </table>
+
+        <h3>Protocols</h3>
+
+        <table cellSpacing="0" cellPadding="0" style={{ width: '100%' }}>
+          <thead>
+            <tr>
+              <th>Protocol</th>
+              <th>Requests (% of total)</th>
+            </tr>
+          </thead>
+          <tbody>
+            {protocolRows}
+          </tbody>
+        </table>
+
+        <h3>Regions</h3>
+
+        <p className="table-filter">Include only countries that made at least <select
+            value={this.state.minCountryRequests}
+            onChange={event => this.setState({ minCountryRequests: parseInt(event.target.value, 10) })}
+          >
+            <option value="0">0</option>
+            <option value="100000">100,000</option>
+            <option value="1000000">1,000,000</option>
+            <option value="10000000">10,000,000</option>
+            <option value="100000000">100,000,000</option>
+          </select> requests.
+        </p>
+
+        <table cellSpacing="0" cellPadding="0" style={{ width: '100%' }} className="regions-table">
+          <thead>
+            <tr>
+              <th>Region</th>
               <th>Requests (% of total)</th>
               <th>Bandwidth (% of total)</th>
             </tr>

client/WindowSize.js

@@ -1,6 +1,7 @@
 import React from 'react'
 import PropTypes from 'prop-types'
-import { addEvent, removeEvent } from './DOMUtils'
+import addEvent from './utils/addEvent'
+import removeEvent from './utils/removeEvent'
 
 const ResizeEvent = 'resize'
 

client/index.css

@@ -11,9 +11,9 @@ body {
   padding: 5px 20px;
 }
 
-@media all and (min-width: 660px) {
+@media (min-width: 800px) {
   body {
-    padding: 50px 20px;
+    padding: 40px 20px 120px;
   }
 }
 
@@ -24,94 +24,103 @@ a:visited {
   color: rebeccapurple;
 }
 
-code {
+h1 {
-  background: #eee;
+  font-size: 2em;
+}
+h2 {
+  font-size: 1.8em;
+}
+h3 {
+  font-size: 1.6em;
+}
+
+ul {
+  padding-left: 25px;
+}
+
+dd {
+  margin-left: 25px;
 }
 
 table {
-  border-color: black;
+  border: 1px solid black;
-  border-style: solid;
+  border: 0;
-  border-width: 0 0 1px 1px;
 }
-table th, table td {
+th {
   text-align: left;
+  background-color: #eee;
+}
+th, td {
+  padding: 5px;
+}
+th {
+  vertical-align: bottom;
+}
+td {
   vertical-align: top;
-  padding: 5px 7px;
-  border-color: black;
-  border-style: solid;
-  border-width: 1px 1px 0 0;
-}
-
-.continent-row {
-  font-weight: bold;
-}
-.country-name {
-  padding-left: 2em;
-}
-
-.table-filter {
-  display: block;
-  text-align: right;
-  color: #666;
-  margin-bottom: 1em;
-}
-.table-filter input {
-  font-size: 1em;
-  text-align: right;
-  max-width: 100px;
-  color: #999;
 }
 
 .wrapper {
-  max-width: 600px;
+  max-width: 700px;
   margin: 0 auto;
 }
-h1, h2, h3, header nav {
-  font-family: Futura, Helvetica, sans-serif;
-  text-transform: uppercase;
-}
-h3 {
-  margin-top: 2em;
-}
 
-header h1 {
+.layout-title {
-  font-size: 4em;
-  line-height: 1;
-  text-align: center;
-  letter-spacing: 0.1em;
-}
-header nav {
-  margin-bottom: 4em;
-}
-header nav a:link,
-header nav a:visited {
-  color: black;
-}
-
-.underlist {
-  list-style-type: none;
-  padding: 0;
   margin: 0;
+  text-transform: uppercase;
+  text-align: center;
+  font-size: 5em;
+}
 
+.layout-nav {
+  margin: 0 0 3em;
+}
+.layout-nav-list {
+  margin: 0;
+  padding: 0;
   display: flex;
   justify-content: center;
 }
-.underlist li {
+.layout-nav-list li {
-  margin: 0 5px;
-  padding: 0 5px;
-
   flex-basis: auto;
+  list-style-type: none;
+  display: inline-block;
+  font-size: 1.1em;
+  margin: 0 10px;
 }
-.underlist a:link {
+.layout-nav-list li a:link {
   text-decoration: none;
 }
-.underlist-underline {
+.layout-nav-list li a:link,
+.layout-nav-list li a:visited {
+  color: black;
+}
+.layout-nav-underline {
   height: 4px;
   background-color: black;
   position: absolute;
   left: 0;
 }
 
+.home-example {
+  text-align: center;
+  background-color: #eee;
+  margin: 2em 0;
+  padding: 5px 0;
+}
+
+.table-filter {
+  font-size: 0.8em;
+  text-align: right;
+}
+
+.regions-table .continent-row {
+  font-weight: bold;
+}
+.regions-table .country-row td.country-name {
+  padding-left: 20px;
+}
+
 .about-logos {
   margin: 2em 0;
   display: flex;
@@ -125,10 +134,3 @@ header nav a:visited {
 .about-logo img {
   max-width: 60%;
 }
-
-footer {
-  margin-top: 40px;
-  border-top: 1px solid black;
-  font-size: 0.8em;
-  text-align: right;
-}

client/utils/addEvent.js

@@ -0,0 +1,9 @@
+const addEvent = (node, type, handler) => {
+  if (node.addEventListener) {
+    node.addEventListener(type, handler, false)
+  } else if (node.attachEvent) {
+    node.attachEvent('on' + type, handler)
+  }
+}
+
+export default addEvent

client/utils/formatNumber.js

@@ -0,0 +1,11 @@
+const formatNumber = (n) => {
+  const digits = String(n).split('')
+  const groups = []
+
+  while (digits.length)
+    groups.unshift(digits.splice(-3).join(''))
+
+  return groups.join(',')
+}
+
+export default formatNumber

client/utils/formatPercent.js

@@ -0,0 +1,4 @@
+const formatPercent = (n, fixed = 1) =>
+  String((n.toPrecision(2) * 100).toFixed(fixed))
+
+export default formatPercent

client/utils/parseNumber.js

@@ -0,0 +1,4 @@
+const parseNumber = (s) =>
+  parseInt(s.replace(/,/g, ''), 10) || 0
+
+export default parseNumber

client/utils/removeEvent.js

@@ -0,0 +1,9 @@
+const removeEvent = (node, type, handler) => {
+  if (node.removeEventListener) {
+    node.removeEventListener(type, handler, false)
+  } else if (node.detachEvent) {
+    node.detachEvent('on' + type, handler)
+  }
+}
+
+export default removeEvent

package.json

@@ -16,7 +16,6 @@
     "date-fns": "^1.28.1",
     "express": "^4.15.2",
     "gunzip-maybe": "^1.4.0",
-    "http-client": "^4.3.1",
     "invariant": "^2.2.2",
     "isomorphic-fetch": "^2.2.1",
     "mime": "^1.3.6",
@@ -35,7 +34,8 @@
     "sri-toolbox": "^0.2.0",
     "tar-fs": "^1.15.2",
     "throng": "^4.0.0",
-    "validate-npm-package-name": "^3.0.0"
+    "validate-npm-package-name": "^3.0.0",
+    "warning": "^3.0.0"
   },
   "devDependencies": {
     "autoprefixer": "6.7.2",

public/index.html

@@ -3,10 +3,9 @@
   <head>
     <meta charset="utf-8">
     <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
-    <meta name="description" content="A fast, global content delivery network for stuff that is published to npm">
+    <meta name="description" content="A fast, global content delivery network for everything on npm">
-    <meta name="viewport" content="width=700,maximum-scale=1">
+    <meta name="viewport" content="width=device-width,initial-scale=1,maximum-scale=1">
     <link rel="shortcut icon" href="%PUBLIC_URL%/favicon.ico">
-    <script>window.serverData = %REACT_APP_SERVER_DATA%</script>
     <!--
       Notice the use of %PUBLIC_URL% in the tag above.
       It will be replaced with the URL of the `public` folder during the build.

scripts/build.js

@@ -3,10 +3,6 @@
 // Do this as the first thing so that any code reading it knows the right env.
 process.env.NODE_ENV = 'production';
 
-// Use dynamic server data in production by inserting a token that will be
-// replaced when the HTML page is served.
-process.env.REACT_APP_SERVER_DATA = '__SERVER_DATA__';
-
 // Load environment variables from .env file. Suppress warnings using silent
 // if this file is missing. dotenv will never modify any environment variables
 // that have already been set.

scripts/start.js

@@ -2,9 +2,6 @@
 
 process.env.NODE_ENV = 'development';
 
-// Use static server data in development.
-process.env.REACT_APP_SERVER_DATA = JSON.stringify(require('../server/dev-data'))
-
 // Load environment variables from .env file. Suppress warnings using silent
 // if this file is missing. dotenv will never modify any environment variables
 // that have already been set.

server/CloudflareAPI.js

@@ -1,5 +1,5 @@
 require('isomorphic-fetch')
-const invariant = require('invariant')
+const warning = require('warning')
 const gunzip = require('gunzip-maybe')
 const ndjson = require('ndjson')
 
@@ -7,12 +7,12 @@ const CloudflareAPIURL = 'https://api.cloudflare.com'
 const CloudflareEmail = process.env.CLOUDFLARE_EMAIL
 const CloudflareKey = process.env.CLOUDFLARE_KEY
 
-invariant(
+warning(
   CloudflareEmail,
   'Missing the $CLOUDFLARE_EMAIL environment variable'
 )
 
-invariant(
+warning(
   CloudflareKey,
   'Missing the $CLOUDFLARE_KEY environment variable'
 )
@@ -30,16 +30,50 @@ function getJSON(path, headers) {
   return get(path, headers).then(function (res) {
     return res.json()
   }).then(function (data) {
+    if (!data.success) {
+      console.error(`CloudflareAPI.getJSON failed at ${path}`)
+      console.error(data)
+      throw new Error('Failed to getJSON from Cloudflare')
+    }
+
     return data.result
   })
 }
 
-function getZones(domain) {
+function getZones(domains) {
-  return getJSON(`/zones?name=${domain}`)
+  return Promise.all(
+    (Array.isArray(domains) ? domains : [ domains ]).map(function (domain) {
+      return getJSON(`/zones?name=${domain}`)
+    })
+  ).then(function (results) {
+    return results.reduce(function (memo, zones) {
+      return memo.concat(zones)
+    })
+  })
 }
 
-function getZoneAnalyticsDashboard(zoneId, since) {
+function reduceResults(target, values) {
-  return getJSON(`/zones/${zoneId}/analytics/dashboard?since=${since}&continuous=true`)
+  Object.keys(values).forEach(key => {
+    const value = values[key]
+
+    if (typeof value === 'object' && value) {
+      target[key] = reduceResults(target[key] || {}, value)
+    } else if (typeof value === 'number') {
+      target[key] = (target[key] || 0) + values[key]
+    }
+  })
+
+  return target
+}
+
+function getZoneAnalyticsDashboard(zones, since, until) {
+  return Promise.all(
+    (Array.isArray(zones) ? zones : [ zones ]).map(function (zone) {
+      return getJSON(`/zones/${zone.id}/analytics/dashboard?since=${since.toISOString()}&until=${until.toISOString()}`)
+    })
+  ).then(function (results) {
+    return results.reduce(reduceResults)
+  })
 }
 
 function getJSONStream(path, headers) {

server/IngestStatsWorker.js

@@ -1,345 +0,0 @@
-const invariant = require('invariant')
-const startOfDay = require('date-fns/start_of_day')
-const addDays = require('date-fns/add_days')
-const cf = require('./CloudflareAPI')
-const db = require('./RedisClient')
-const {
-  createDayKey,
-  createHourKey,
-  createMinuteKey
-} = require('./StatsServer')
-
-/**
- * Domains we want to analyze.
- */
-const DomainNames = [
-  'unpkg.com',
-  'npmcdn.com'
-]
-
-const oneSecond = 1000
-const oneMinute = oneSecond * 60
-const oneHour = oneMinute * 60
-const oneDay = oneHour * 24
-
-function getSeconds(date) {
-  return Math.floor(date.getTime() / 1000)
-}
-
-function reduceResults(memo, results) {
-  Object.keys(results).forEach(function (key) {
-    const value = results[key]
-
-    if (typeof value === 'object' && value) {
-      memo[key] = reduceResults(memo[key] || {}, value)
-    } else if (typeof value === 'number') {
-      memo[key] = (memo[key] || 0) + results[key]
-    }
-  })
-
-  return memo
-}
-
-function ingestStatsForZones(zones, since, processDashboard) {
-  return new Promise(function (resolve) {
-    const zoneNames = zones.map(function (zone) {
-      return zone.name
-    }).join(', ')
-
-    console.log(
-      'info: Started ingesting stats for zones %s since %d',
-      zoneNames,
-      since
-    )
-
-    const startFetchTime = Date.now()
-
-    resolve(
-      Promise.all(
-        zones.map(function (zone) {
-          return cf.getZoneAnalyticsDashboard(zone.id, since)
-        })
-      ).then(function (results) {
-          const endFetchTime = Date.now()
-
-          console.log(
-            'info: Fetched zone analytics dashboards for %s since %d in %dms',
-            zoneNames,
-            since,
-            endFetchTime - startFetchTime
-          )
-
-          // We don't have per-minute dashboards available for npmcdn.com yet,
-          // so the dashboard for that domain will be null when querying for
-          // per-minute data. Just filter it out here for now.
-          results = results.filter(Boolean)
-
-          return results.length ? results.reduce(reduceResults) : null
-      }).then(function (dashboard) {
-        if (dashboard == null) {
-          console.warn(
-            'warning: Missing dashboards for %s since %d',
-            zoneNames,
-            since
-          )
-
-          return
-        }
-
-        const startProcessTime = Date.now()
-
-        return processDashboard(dashboard).then(function () {
-          const endProcessTime = Date.now()
-
-          console.log(
-            'info: Processed zone analytics dashboards for %s since %d in %dms',
-            zoneNames,
-            since,
-            endProcessTime - startProcessTime
-          )
-        })
-      })
-    )
-  })
-}
-
-function ingestPerDayStats(zones) {
-  return ingestStatsForZones(zones, -10080, processPerDayDashboard)
-}
-
-function processPerDayDashboard(dashboard) {
-  return Promise.all(dashboard.timeseries.map(processPerDayTimeseries))
-}
-
-function errorCount(httpStatus) {
-  return Object.keys(httpStatus).reduce(function (memo, status) {
-    return parseInt(status, 10) >= 500 ? memo + httpStatus[status] : memo
-  }, 0)
-}
-
-function processPerDayTimeseries(ts) {
-  return new Promise(function (resolve) {
-    const since = new Date(ts.since)
-    const until = new Date(ts.until)
-
-    invariant(
-      since.getUTCHours() === 0 && since.getUTCMinutes() === 0 && since.getUTCSeconds() === 0,
-      'Per-day timeseries.since must begin exactly on the day'
-    )
-
-    invariant(
-      (until - since) === oneDay,
-      'Per-day timeseries must span exactly one day'
-    )
-
-    const nextDay = startOfDay(addDays(until, 1))
-    const oneYearLater = getSeconds(addDays(nextDay, 365))
-    const dayKey = createDayKey(since)
-
-    // Q: How many requests do we serve per day?
-    db.set(`stats-requests-${dayKey}`, ts.requests.all)
-    db.expireat(`stats-requests-${dayKey}`, oneYearLater)
-
-    // Q: How many requests do we serve per day from the cache?
-    db.set(`stats-cachedRequests-${dayKey}`, ts.requests.cached)
-    db.expireat(`stats-cachedRequests-${dayKey}`, oneYearLater)
-
-    // Q: How much bandwidth do we serve per day?
-    db.set(`stats-bandwidth-${dayKey}`, ts.bandwidth.all)
-    db.expireat(`stats-bandwidth-${dayKey}`, oneYearLater)
-
-    // Q: How much bandwidth do we serve per day from the cache?
-    db.set(`stats-cachedBandwidth-${dayKey}`, ts.bandwidth.cached)
-    db.expireat(`stats-cachedBandwidth-${dayKey}`, oneYearLater)
-
-    // Q: How many errors do we serve per day?
-    db.set(`stats-errors-${dayKey}`, errorCount(ts.requests.http_status))
-    db.expireat(`stats-errors-${dayKey}`, oneYearLater)
-
-    // Q: How many threats do we see each day?
-    db.set(`stats-threats-${dayKey}`, ts.threats.all)
-    db.expireat(`stats-threats-${dayKey}`, oneYearLater)
-
-    // Q: How many unique visitors do we see each day?
-    db.set(`stats-uniques-${dayKey}`, ts.uniques.all)
-    db.expireat(`stats-uniques-${dayKey}`, oneYearLater)
-
-    const requestsByCountry = []
-    const bandwidthByCountry = []
-
-    Object.keys(ts.requests.country).forEach(function (country) {
-      const requests = ts.requests.country[country]
-      const bandwidth = ts.bandwidth.country[country]
-
-      // Include only countries who made at least 100K requests.
-      if (requests > 100000) {
-        requestsByCountry.push(requests, country)
-        bandwidthByCountry.push(bandwidth, country)
-      }
-    })
-
-    const thirtyDaysLater = getSeconds(addDays(nextDay, 30))
-
-    // Q: How many requests do we serve to a country per day?
-    if (requestsByCountry.length) {
-      db.zadd([ `stats-countryRequests-${dayKey}`, ...requestsByCountry ])
-      db.expireat(`stats-countryRequests-${dayKey}`, thirtyDaysLater)
-    }
-
-    // Q: How much bandwidth do we serve to a country per day?
-    if (bandwidthByCountry.length) {
-      db.zadd([ `stats-countryBandwidth-${dayKey}`, ...bandwidthByCountry ])
-      db.expireat(`stats-countryBandwidth-${dayKey}`, thirtyDaysLater)
-    }
-
-    resolve()
-  })
-}
-
-function ingestPerHourStats(zones) {
-  return ingestStatsForZones(zones, -1440, processPerHourDashboard)
-}
-
-function processPerHourDashboard(dashboard) {
-  return Promise.all(dashboard.timeseries.map(processPerHourTimeseries))
-}
-
-function processPerHourTimeseries(ts) {
-  return new Promise(function (resolve) {
-    const since = new Date(ts.since)
-    const until = new Date(ts.until)
-
-    invariant(
-      since.getUTCMinutes() === 0 && since.getUTCSeconds() === 0,
-      'Per-hour timeseries.since must begin exactly on the hour'
-    )
-
-    invariant(
-      (until - since) === oneHour,
-      'Per-hour timeseries must span exactly one hour'
-    )
-
-    const nextDay = startOfDay(addDays(until, 1))
-    const sevenDaysLater = getSeconds(addDays(nextDay, 7))
-    const hourKey = createHourKey(since)
-
-    // Q: How many requests do we serve per hour?
-    db.set(`stats-requests-${hourKey}`, ts.requests.all)
-    db.expireat(`stats-requests-${hourKey}`, sevenDaysLater)
-
-    // Q: How many requests do we serve per hour from the cache?
-    db.set(`stats-cachedRequests-${hourKey}`, ts.requests.cached)
-    db.expireat(`stats-cachedRequests-${hourKey}`, sevenDaysLater)
-
-    // Q: How much bandwidth do we serve per hour?
-    db.set(`stats-bandwidth-${hourKey}`, ts.bandwidth.all)
-    db.expireat(`stats-bandwidth-${hourKey}`, sevenDaysLater)
-
-    // Q: How much bandwidth do we serve per hour from the cache?
-    db.set(`stats-cachedBandwidth-${hourKey}`, ts.bandwidth.cached)
-    db.expireat(`stats-cachedBandwidth-${hourKey}`, sevenDaysLater)
-
-    // Q: How many errors do we serve per hour?
-    db.set(`stats-errors-${hourKey}`, errorCount(ts.requests.http_status))
-    db.expireat(`stats-errors-${hourKey}`, sevenDaysLater)
-
-    // Q: How many threats do we see each hour?
-    db.set(`stats-threats-${hourKey}`, ts.threats.all)
-    db.expireat(`stats-threats-${hourKey}`, sevenDaysLater)
-
-    // Q: How many unique visitors do we see each hour?
-    db.set(`stats-uniques-${hourKey}`, ts.uniques.all)
-    db.expireat(`stats-uniques-${hourKey}`, sevenDaysLater)
-
-    resolve()
-  })
-}
-
-function ingestPerMinuteStats(zones) {
-  return ingestStatsForZones(zones, -30, processPerMinuteDashboard)
-}
-
-function processPerMinuteDashboard(dashboard) {
-  return Promise.all(dashboard.timeseries.map(processPerMinuteTimeseries))
-}
-
-function processPerMinuteTimeseries(ts) {
-  return new Promise(function (resolve) {
-    const since = new Date(ts.since)
-    const until = new Date(ts.until)
-
-    invariant(
-      since.getUTCSeconds() === 0,
-      'Per-minute timeseries.since must begin exactly on the minute'
-    )
-
-    invariant(
-      (until - since) === oneMinute,
-      'Per-minute timeseries must span exactly one minute'
-    )
-
-    const nextDay = startOfDay(addDays(until, 1))
-    const oneDayLater = getSeconds(addDays(nextDay, 1))
-    const minuteKey = createMinuteKey(since)
-
-    // Q: How many requests do we serve per minute?
-    db.set(`stats-requests-${minuteKey}`, ts.requests.all)
-    db.expireat(`stats-requests-${minuteKey}`, oneDayLater)
-
-    // Q: How many requests do we serve per minute from the cache?
-    db.set(`stats-cachedRequests-${minuteKey}`, ts.requests.cached)
-    db.expireat(`stats-cachedRequests-${minuteKey}`, oneDayLater)
-
-    // Q: How much bandwidth do we serve per minute?
-    db.set(`stats-bandwidth-${minuteKey}`, ts.bandwidth.all)
-    db.expireat(`stats-bandwidth-${minuteKey}`, oneDayLater)
-
-    // Q: How much bandwidth do we serve per minute from the cache?
-    db.set(`stats-cachedBandwidth-${minuteKey}`, ts.bandwidth.cached)
-    db.expireat(`stats-cachedBandwidth-${minuteKey}`, oneDayLater)
-
-    // Q: How many errors do we serve per hour?
-    db.set(`stats-errors-${minuteKey}`, errorCount(ts.requests.http_status))
-    db.expireat(`stats-errors-${minuteKey}`, oneDayLater)
-
-    // Q: How many threats do we see each minute?
-    db.set(`stats-threats-${minuteKey}`, ts.threats.all)
-    db.expireat(`stats-threats-${minuteKey}`, oneDayLater)
-
-    // Q: How many unique visitors do we see each minute?
-    db.set(`stats-uniques-${minuteKey}`, ts.uniques.all)
-    db.expireat(`stats-uniques-${minuteKey}`, oneDayLater)
-
-    resolve()
-  })
-}
-
-function startZones(zones) {
-  function takePerMinuteTurn() {
-    ingestPerMinuteStats(zones)
-  }
-
-  function takePerHourTurn() {
-    ingestPerHourStats(zones)
-  }
-
-  function takePerDayTurn() {
-    ingestPerDayStats(zones)
-  }
-
-  takePerMinuteTurn()
-  takePerHourTurn()
-  takePerDayTurn()
-
-  setInterval(takePerMinuteTurn, oneMinute)
-  setInterval(takePerHourTurn, oneHour / 2)
-  setInterval(takePerDayTurn, oneHour / 2)
-}
-
-Promise.all(DomainNames.map(cf.getZones)).then(function (results) {
-  const zones = results.reduce(function (memo, zones) {
-    return memo.concat(zones)
-  })
-
-  startZones(zones)
-})

server/StatsServer.js

@@ -1,25 +1,16 @@
+const cf = require('./CloudflareAPI')
 const db = require('./RedisClient')
 
-function sumValues(array) {
+function createDayKey(date) {
-  return array.reduce(function (memo, n) {
+  return `${date.getUTCFullYear()}-${date.getUTCMonth()}-${date.getUTCDate()}`
-    return memo + (parseInt(n, 10) || 0)
-  }, 0)
 }
 
-function getKeyValues(keys) {
+function createHourKey(date) {
-  return new Promise(function (resolve, reject) {
+  return `${createDayKey(date)}-${date.getUTCHours()}`
-    db.mget(keys, function (error, values) {
-      if (error) {
-        reject(error)
-      } else {
-        resolve(values)
-      }
-    })
-  })
 }
 
-function sumKeys(keys) {
+function createMinuteKey(date) {
-  return getKeyValues(keys).then(sumValues)
+  return `${createHourKey(date)}-${date.getUTCMinutes()}`
 }
 
 function createScoresMap(array) {
@@ -31,7 +22,7 @@ function createScoresMap(array) {
   return map
 }
 
-function getScoresMap(key, n = 10) {
+function getScoresMap(key, n = 100) {
   return new Promise(function (resolve, reject) {
     db.zrevrange(key, 0, n, 'withscores', function (error, value) {
       if (error) {
@@ -43,16 +34,31 @@ function getScoresMap(key, n = 10) {
   })
 }
 
-function createTopScores(map) {
+function getPackageRequests(date, n = 100) {
-  return Object.keys(map).reduce(function (memo, key) {
+  return getScoresMap(`stats-packageRequests-${createDayKey(date)}`, n)
-    return memo.concat([ [ key, map[key] ] ])
-  }, []).sort(function (a, b) {
-    return b[1] - a[1]
-  })
 }
 
-function getTopScores(key, n) {
+function getPackageBandwidth(date, n = 100) {
-  return getScoresMap(key, n).then(createTopScores)
+  return getScoresMap(`stats-packageBytes-${createDayKey(date)}`, n)
+}
+
+function getProtocolRequests(date) {
+  return getScoresMap(`stats-protocolRequests-${createDayKey(date)}`)
+}
+
+function addDailyMetricsToTimeseries(timeseries) {
+  const since = new Date(timeseries.since)
+
+  return Promise.all([
+    getPackageRequests(since),
+    getPackageBandwidth(since),
+    getProtocolRequests(since)
+  ]).then(function (results) {
+    timeseries.requests.package = results[0]
+    timeseries.bandwidth.package = results[1]
+    timeseries.requests.protocol = results[2]
+    return timeseries
+  })
 }
 
 function sumMaps(maps) {
@@ -65,36 +71,95 @@ function sumMaps(maps) {
   }, {})
 }
 
-function sumTopScores(keys, n) {
+function addDailyMetrics(result) {
   return Promise.all(
-    keys.map(function (key) {
+    result.timeseries.map(addDailyMetricsToTimeseries)
-      return getScoresMap(key, n)
+  ).then(function () {
+    result.totals.requests.package = sumMaps(
+      result.timeseries.map(function (timeseries) {
+        return timeseries.requests.package
+      })
+    )
+
+    result.totals.bandwidth.package = sumMaps(
+      result.timeseries.map(function (timeseries) {
+        return timeseries.bandwidth.package
+      })
+    )
+
+    result.totals.requests.protocol = sumMaps(
+      result.timeseries.map(function (timeseries) {
+        return timeseries.requests.protocol
+      })
+    )
+
+    return result
+  })
+}
+
+function extractPublicInfo(data) {
+  return {
+    since: data.since,
+    until: data.until,
+
+    requests: {
+      all: data.requests.all,
+      cached: data.requests.cached,
+      country: data.requests.country,
+      status: data.requests.http_status
+    },
+
+    bandwidth: {
+      all: data.bandwidth.all,
+      cached: data.bandwidth.cached,
+      country: data.bandwidth.country
+    },
+
+    threats: {
+      all: data.threats.all,
+      country: data.threats.country
+    },
+
+    uniques: {
+      all: data.uniques.all
+    }
+  }
+}
+
+const DomainNames = [
+  'unpkg.com',
+  'npmcdn.com'
+]
+
+function fetchStats(since, until) {
+  return cf.getZones(DomainNames).then(function (zones) {
+    return cf.getZoneAnalyticsDashboard(zones, since, until).then(function (dashboard) {
+      return {
+        timeseries: dashboard.timeseries.map(extractPublicInfo),
+        totals: extractPublicInfo(dashboard.totals)
+      }
     })
-  ).then(sumMaps).then(createTopScores)
+  })
 }
 
-function createKey(...args) {
+const oneMinute = 1000 * 60
-  return args.join('-')
+const oneHour = oneMinute * 60
-}
+const oneDay = oneHour * 24
 
-function createDayKey(date) {
+function getStats(since, until, callback) {
-  return createKey(date.getUTCFullYear(), date.getUTCMonth(), date.getUTCDate())
+  let promise = fetchStats(since, until)
-}
 
-function createHourKey(date) {
+  if ((until - since) > oneDay)
-  return createKey(createDayKey(date), date.getUTCHours())
+    promise = promise.then(addDailyMetrics)
-}
 
-function createMinuteKey(date) {
+  promise.then(function (value) {
-  return createKey(createHourKey(date), date.getUTCMinutes())
+    callback(null, value)
+  }, callback)
 }
 
 module.exports = {
-  getKeyValues,
-  sumKeys,
-  getTopScores,
-  sumTopScores,
   createDayKey,
   createHourKey,
-  createMinuteKey
+  createMinuteKey,
+  getStats
 }

server/cloudflare.js

@@ -1,93 +0,0 @@
-require('isomorphic-fetch')
-const { createStack, createFetch, header, base, query, parseJSON, onResponse } = require('http-client')
-const invariant = require('invariant')
-
-const CloudflareKey = process.env.CLOUDFLARE_KEY
-const CloudflareEmail = process.env.CLOUDFLARE_EMAIL
-
-invariant(
-  CloudflareKey,
-  'Missing $CLOUDFLARE_KEY environment variable'
-)
-
-invariant(
-  CloudflareEmail,
-  'Missing $CLOUDFLARE_EMAIL environment variable'
-)
-
-const createRangeQuery = (since, until) =>
-  query({
-    since: since.toISOString(),
-    until: until.toISOString()
-  })
-
-const createNameQuery = (name) =>
-  query({ name })
-
-const getResult = () =>
-  createStack(
-    parseJSON(),
-    onResponse(response => response.jsonData.result)
-  )
-
-const commonStack = createStack(
-  header('X-Auth-Key', CloudflareKey),
-  header('X-Auth-Email', CloudflareEmail),
-  base('https://api.cloudflare.com/client/v4'),
-  getResult()
-)
-
-const getZones = (domainName) =>
-  createFetch(
-    commonStack,
-    createNameQuery(domainName)
-  )('/zones')
-
-const getZoneAnalyticsDashboard = (zone, since, until) =>
-  createFetch(
-    commonStack,
-    createRangeQuery(since, until)
-  )(`/zones/${zone.id}/analytics/dashboard`)
-
-const getAnalyticsDashboards = (domainNames, since, until) =>
-  Promise.all(
-    domainNames.map(domainName => getZones(domainName))
-  ).then(
-    domainZones => domainZones.reduce((memo, zones) => memo.concat(zones))
-  ).then(
-    zones => Promise.all(zones.map(zone => getZoneAnalyticsDashboard(zone, since, until)))
-  ).then(
-    results => results.reduce(reduceResults)
-  )
-
-const reduceResults = (target, results) => {
-  Object.keys(results).forEach(key => {
-    const value = results[key]
-
-    if (typeof value === 'object' && value) {
-      target[key] = reduceResults(target[key] || {}, value)
-    } else if (typeof value === 'number') {
-      target[key] = (target[key] || 0) + results[key]
-    }
-  })
-
-  return target
-}
-
-const OneMinute = 1000 * 60
-const ThirtyDays = OneMinute * 60 * 24 * 30
-
-const fetchStats = (callback) => {
-  const since = new Date(Date.now() - ThirtyDays)
-  const until = new Date(Date.now() - OneMinute)
-
-  getAnalyticsDashboards([ 'npmcdn.com', 'unpkg.com' ], since, until)
-    .then(result => callback(null, result), callback)
-}
-
-module.exports = {
-  getZones,
-  getZoneAnalyticsDashboard,
-  getAnalyticsDashboards,
-  fetchStats
-}

server/createApp.js

@@ -4,46 +4,20 @@ const express = require('express')
 const cors = require('cors')
 const morgan = require('morgan')
 
-const { fetchStats } = require('./cloudflare')
-
 const checkBlacklist = require('./middleware/checkBlacklist')
 const packageURL = require('./middleware/packageURL')
 const fetchFile = require('./middleware/fetchFile')
 const serveFile = require('./middleware/serveFile')
-
+const serveStats = require('./middleware/serveStats')
-/**
- * A list of packages we refuse to serve.
- */
-const PackageBlacklist = require('./PackageBlacklist').blacklist
 
 morgan.token('fwd', function (req) {
   return req.get('x-forwarded-for').replace(/\s/g, '')
 })
 
-function sendHomePage(publicDir) {
+/**
-  const html = fs.readFileSync(path.join(publicDir, 'index.html'), 'utf8')
+ * A list of packages we refuse to serve.
-
+ */
-  return function (req, res, next) {
+const PackageBlacklist = require('./PackageBlacklist').blacklist
-    fetchStats(function (error, stats) {
-      if (error) {
-        next(error)
-      } else {
-        res.set({
-          'Cache-Control': 'public, max-age=60',
-          'Cache-Tag': 'home'
-        })
-
-        res.send(
-          // Replace the __SERVER_DATA__ token that was added to the
-          // HTML file in the build process (see scripts/build.js).
-          html.replace('__SERVER_DATA__', JSON.stringify({
-            cloudflareStats: stats
-          }))
-        )
-      }
-    })
-  }
-}
 
 function errorHandler(err, req, res, next) {
   console.error(err.stack)
@@ -68,12 +42,12 @@ function createApp() {
   app.use(errorHandler)
   app.use(cors())
 
-  app.get('/', sendHomePage('build'))
-
   app.use(express.static('build', {
     maxAge: '365d'
   }))
 
+  app.use('/_stats', serveStats())
+
   app.use('/',
     packageURL,
     checkBlacklist(PackageBlacklist),

server/middleware/serveStats.js

@@ -0,0 +1,77 @@
+const express = require('express')
+const subDays = require('date-fns/sub_days')
+const startOfDay = require('date-fns/start_of_day')
+const startOfSecond = require('date-fns/start_of_second')
+const StatsServer = require('../StatsServer')
+
+function serveArbitraryStats(req, res) {
+  const now = startOfSecond(new Date)
+  const since = req.query.since ? new Date(req.query.since) : subDays(now, 30)
+  const until = req.query.until ? new Date(req.query.until) : now
+
+  if (isNaN(since.getTime()))
+    return res.status(403).send({ error: '?since is not a valid date' })
+
+  if (isNaN(until.getTime()))
+    return res.status(403).send({ error: '?until is not a valid date' })
+
+  if (until <= since)
+    return res.status(403).send({ error: '?until date must come after ?since date' })
+
+  if (until > now)
+    return res.status(403).send({ error: '?until must be a date in the past' })
+
+  StatsServer.getStats(since, until, function (error, stats) {
+    if (error) {
+      console.error(error)
+      res.status(500).send({ error: 'Unable to fetch stats' })
+    } else {
+      res.set({
+        'Cache-Control': 'public, max-age=60',
+        'Cache-Tag': 'stats'
+      }).send(stats)
+    }
+  })
+}
+
+function servePastDaysStats(days, req, res) {
+  const until = startOfDay(new Date)
+  const since = subDays(until, days)
+
+  StatsServer.getStats(since, until, function (error, stats) {
+    if (error) {
+      console.error(error)
+      res.status(500).send({ error: 'Unable to fetch stats' })
+    } else {
+      res.set({
+        'Cache-Control': 'public, max-age=86400',
+        'Cache-Tag': 'stats'
+      }).send(stats)
+    }
+  })
+}
+
+function serveLastMonthStats(req, res) {
+  servePastDaysStats(30, req, res)
+}
+
+function serveLastWeekStats(req, res) {
+  servePastDaysStats(7, req, res)
+}
+
+function serveLastDayStats(req, res) {
+  servePastDaysStats(1, req, res)
+}
+
+function serveStats() {
+  const app = express.Router()
+
+  app.get('/', serveArbitraryStats)
+  app.get('/last-month', serveLastMonthStats)
+  app.get('/last-week', serveLastWeekStats)
+  app.get('/last-day', serveLastDayStats)
+
+  return app
+}
+
+module.exports = serveStats

yarn.lock

@@ -1004,10 +1004,6 @@ builtins@^1.0.3:
   version "1.0.3"
   resolved "https://registry.yarnpkg.com/builtins/-/builtins-1.0.3.tgz#cb94faeb61c8696451db36534e1422f94f0aee88"
 
-byte-length@^0.1.1:
-  version "0.1.1"
-  resolved "https://registry.yarnpkg.com/byte-length/-/byte-length-0.1.1.tgz#e9b4774dbce7c59764bf5be87c302789a88738c3"
-
 bytes@2.5.0:
   version "2.5.0"
   resolved "https://registry.yarnpkg.com/bytes/-/bytes-2.5.0.tgz#4c9423ea2d252c270c41b2bdefeff9bb6b62c06a"
@@ -2538,13 +2534,6 @@ htmlparser2@~3.3.0:
     domutils "1.1"
     readable-stream "1.0"
 
-http-client@^4.3.1:
-  version "4.3.1"
-  resolved "https://registry.yarnpkg.com/http-client/-/http-client-4.3.1.tgz#cf82fa1b1ef993c078a82144fe677a8671bc7cf2"
-  dependencies:
-    byte-length "^0.1.1"
-    query-string "^4.1.0"
-
 http-errors@~1.6.1, http-errors@~1.6.2:
   version "1.6.2"
   resolved "https://registry.yarnpkg.com/http-errors/-/http-errors-1.6.2.tgz#0a002cc85707192a7e7946ceedc11155f60ec736"