Improve stats workers

- More accurate expiration times for stats
- Reduce duplication between stats and logs worker
- Expire hostname stats after one week

server/CloudflareAPI.js

@@ -1,5 +1,7 @@
 require('isomorphic-fetch')
 const invariant = require('invariant')
+const gunzip = require('gunzip-maybe')
+const ndjson = require('ndjson')
 
 const CloudflareAPIURL = 'https://api.cloudflare.com'
 const CloudflareEmail = process.env.CLOUDFLARE_EMAIL
@@ -32,7 +34,33 @@ function getJSON(path, headers) {
   })
 }
 
+function getZones(domain) {
+  return getJSON(`/zones?name=${domain}`)
+}
+
+function getZoneAnalyticsDashboard(zoneId, since) {
+  return getJSON(`/zones/${zoneId}/analytics/dashboard?since=${since}&continuous=true`)
+}
+
+function getJSONStream(path, headers) {
+  const acceptGzipHeaders = Object.assign({}, headers, { 'Accept-Encoding': 'gzip' })
+
+  return get(path, acceptGzipHeaders).then(function (res) {
+    return res.body.pipe(gunzip())
+  }).then(function (stream) {
+    return stream.pipe(ndjson.parse())
+  })
+}
+
+function getLogs(zoneId, startTime, endTime) {
+  return getJSONStream(`/zones/${zoneId}/logs/requests?start=${startTime}&end=${endTime}`)
+}
+
 module.exports = {
   get,
-  getJSON
+  getJSON,
+  getZones,
+  getZoneAnalyticsDashboard,
+  getJSONStream,
+  getLogs
 }