From 2e3c9ff52643208615c4ce7c4860f54c68fd41f8 Mon Sep 17 00:00:00 2001 From: Michael Jackson Date: Tue, 9 Jul 2019 16:48:00 -0700 Subject: [PATCH] Consolidate stats module --- modules/actions/serveStats.js | 2 +- modules/{ => actions}/utils/cloudflare.js | 24 +--- modules/actions/utils/getStats.js | 42 +++++++ modules/utils/stats.js | 145 ---------------------- 4 files changed, 45 insertions(+), 168 deletions(-) rename modules/{ => actions}/utils/cloudflare.js (74%) create mode 100644 modules/actions/utils/getStats.js delete mode 100644 modules/utils/stats.js diff --git a/modules/actions/serveStats.js b/modules/actions/serveStats.js index d8ae517..a51e2e5 100644 --- a/modules/actions/serveStats.js +++ b/modules/actions/serveStats.js @@ -1,6 +1,6 @@ import { subDays, startOfDay, startOfSecond } from 'date-fns'; -import { getStats } from '../utils/stats'; +import getStats from './utils/getStats.js'; export default function serveStats(req, res) { let since, until; diff --git a/modules/utils/cloudflare.js b/modules/actions/utils/cloudflare.js similarity index 74% rename from modules/utils/cloudflare.js rename to modules/actions/utils/cloudflare.js index 1a12262..1854de3 100644 --- a/modules/utils/cloudflare.js +++ b/modules/actions/utils/cloudflare.js @@ -1,7 +1,5 @@ import fetch from 'isomorphic-fetch'; import invariant from 'invariant'; -import gunzip from 'gunzip-maybe'; -import ndjson from 'ndjson'; const cloudflareURL = 'https://api.cloudflare.com/client/v4'; const cloudflareEmail = process.env.CLOUDFLARE_EMAIL; @@ -14,7 +12,7 @@ invariant( invariant(cloudflareKey, 'Missing the $CLOUDFLARE_KEY environment variable'); -export function get(path, headers) { +function get(path, headers) { return fetch(`${cloudflareURL}${path}`, { headers: Object.assign({}, headers, { 'X-Auth-Email': cloudflareEmail, @@ -23,7 +21,7 @@ export function get(path, headers) { }); } -export function getJSON(path, headers) { +function getJSON(path, headers) { return get(path, headers) .then(res => { return res.json(); @@ -72,21 +70,3 @@ export function getZoneAnalyticsDashboard(zones, since, until) { }) ).then(results => results.reduce(reduceResults)); } - -export function getJSONStream(path, headers) { - const gzipHeaders = Object.assign({}, headers, { - 'Accept-Encoding': 'gzip' - }); - - return get(path, gzipHeaders) - .then(res => res.body.pipe(gunzip())) - .then(stream => stream.pipe(ndjson.parse())); -} - -export function getLogs(zoneId, startTime, endTime, fieldsArray) { - const fields = fieldsArray.join(','); - - return getJSONStream( - `/zones/${zoneId}/logs/received?start=${startTime}&end=${endTime}&fields=${fields}` - ); -} diff --git a/modules/actions/utils/getStats.js b/modules/actions/utils/getStats.js new file mode 100644 index 0000000..8fc82c5 --- /dev/null +++ b/modules/actions/utils/getStats.js @@ -0,0 +1,42 @@ +import * as cloudflare from './cloudflare.js'; + +function extractPublicInfo(data) { + return { + since: data.since, + until: data.until, + requests: { + all: data.requests.all, + cached: data.requests.cached, + country: data.requests.country, + status: data.requests.http_status + }, + bandwidth: { + all: data.bandwidth.all, + cached: data.bandwidth.cached, + country: data.bandwidth.country + }, + threats: { + all: data.threats.all, + country: data.threats.country + }, + uniques: { + all: data.uniques.all + } + }; +} + +const DomainNames = ['unpkg.com', 'npmcdn.com']; + +export default async function getStats(since, until) { + const zones = await cloudflare.getZones(DomainNames); + const dashboard = await cloudflare.getZoneAnalyticsDashboard( + zones, + since, + until + ); + + return { + timeseries: dashboard.timeseries.map(extractPublicInfo), + totals: extractPublicInfo(dashboard.totals) + }; +} diff --git a/modules/utils/stats.js b/modules/utils/stats.js deleted file mode 100644 index 623987a..0000000 --- a/modules/utils/stats.js +++ /dev/null @@ -1,145 +0,0 @@ -// import data from './data'; -import * as cloudflare from './cloudflare'; - -export function createDayKey(date) { - return `${date.getUTCFullYear()}-${date.getUTCMonth()}-${date.getUTCDate()}`; -} - -export function createHourKey(date) { - return `${createDayKey(date)}-${date.getUTCHours()}`; -} - -export function createMinuteKey(date) { - return `${createHourKey(date)}-${date.getUTCMinutes()}`; -} - -// function createScoresMap(array) { -// const map = {}; - -// for (let i = 0; i < array.length; i += 2) { -// map[array[i]] = parseInt(array[i + 1], 10); -// } - -// return map; -// } - -// function getScoresMap(key, n = 100) { -// return new Promise((resolve, reject) => { -// data.zrevrange(key, 0, n, 'withscores', (error, value) => { -// if (error) { -// reject(error); -// } else { -// resolve(createScoresMap(value)); -// } -// }); -// }); -// } - -// function getPackageRequests(date, n = 100) { -// return getScoresMap(`stats-packageRequests-${createDayKey(date)}`, n); -// } - -// function getPackageBandwidth(date, n = 100) { -// return getScoresMap(`stats-packageBytes-${createDayKey(date)}`, n); -// } - -// function getProtocolRequests(date) { -// return getScoresMap(`stats-protocolRequests-${createDayKey(date)}`); -// } - -// function addDailyMetricsToTimeseries(timeseries) { -// const since = new Date(timeseries.since); - -// return Promise.all([ -// getPackageRequests(since), -// getPackageBandwidth(since), -// getProtocolRequests(since) -// ]).then(results => { -// timeseries.requests.package = results[0]; -// timeseries.bandwidth.package = results[1]; -// timeseries.requests.protocol = results[2]; -// return timeseries; -// }); -// } - -// function sumMaps(maps) { -// return maps.reduce((memo, map) => { -// Object.keys(map).forEach(key => { -// memo[key] = (memo[key] || 0) + map[key]; -// }); - -// return memo; -// }, {}); -// } - -// function addDailyMetrics(result) { -// return Promise.all(result.timeseries.map(addDailyMetricsToTimeseries)).then( -// () => { -// result.totals.requests.package = sumMaps( -// result.timeseries.map(timeseries => { -// return timeseries.requests.package; -// }) -// ); - -// result.totals.bandwidth.package = sumMaps( -// result.timeseries.map(timeseries => timeseries.bandwidth.package) -// ); - -// result.totals.requests.protocol = sumMaps( -// result.timeseries.map(timeseries => timeseries.requests.protocol) -// ); - -// return result; -// } -// ); -// } - -function extractPublicInfo(data) { - return { - since: data.since, - until: data.until, - requests: { - all: data.requests.all, - cached: data.requests.cached, - country: data.requests.country, - status: data.requests.http_status - }, - bandwidth: { - all: data.bandwidth.all, - cached: data.bandwidth.cached, - country: data.bandwidth.country - }, - threats: { - all: data.threats.all, - country: data.threats.country - }, - uniques: { - all: data.uniques.all - } - }; -} - -const DomainNames = ['unpkg.com', 'npmcdn.com']; - -function fetchStats(since, until) { - return cloudflare.getZones(DomainNames).then(zones => { - return cloudflare - .getZoneAnalyticsDashboard(zones, since, until) - .then(dashboard => { - return { - timeseries: dashboard.timeseries.map(extractPublicInfo), - totals: extractPublicInfo(dashboard.totals) - }; - }); - }); -} - -// const oneMinute = 1000 * 60; -// const oneHour = oneMinute * 60; -// const oneDay = oneHour * 24; - -export function getStats(since, until) { - const promise = fetchStats(since, until); - // return until - since > oneDay ? promise.then(addDailyMetrics) : promise; - return promise; -}