const db = require("./utils/data"); const CloudflareAPI = require("./CloudflareAPI"); const BlacklistAPI = require("./BlacklistAPI"); function prunePackages(packagesMap) { return Promise.all( Object.keys(packagesMap).map(packageName => BlacklistAPI.includesPackage(packageName).then(blacklisted => { if (blacklisted) { delete packagesMap[packageName]; } }) ) ).then(() => packagesMap); } function createDayKey(date) { return `${date.getUTCFullYear()}-${date.getUTCMonth()}-${date.getUTCDate()}`; } function createHourKey(date) { return `${createDayKey(date)}-${date.getUTCHours()}`; } function createMinuteKey(date) { return `${createHourKey(date)}-${date.getUTCMinutes()}`; } function createScoresMap(array) { const map = {}; for (let i = 0; i < array.length; i += 2) { map[array[i]] = parseInt(array[i + 1], 10); } return map; } function getScoresMap(key, n = 100) { return new Promise((resolve, reject) => { db.zrevrange(key, 0, n, "withscores", (error, value) => { if (error) { reject(error); } else { resolve(createScoresMap(value)); } }); }); } function getPackageRequests(date, n = 100) { return getScoresMap(`stats-packageRequests-${createDayKey(date)}`, n).then( prunePackages ); } function getPackageBandwidth(date, n = 100) { return getScoresMap(`stats-packageBytes-${createDayKey(date)}`, n).then( prunePackages ); } function getProtocolRequests(date) { return getScoresMap(`stats-protocolRequests-${createDayKey(date)}`); } function addDailyMetricsToTimeseries(timeseries) { const since = new Date(timeseries.since); return Promise.all([ getPackageRequests(since), getPackageBandwidth(since), getProtocolRequests(since) ]).then(results => { timeseries.requests.package = results[0]; timeseries.bandwidth.package = results[1]; timeseries.requests.protocol = results[2]; return timeseries; }); } function sumMaps(maps) { return maps.reduce((memo, map) => { Object.keys(map).forEach(key => { memo[key] = (memo[key] || 0) + map[key]; }); return memo; }, {}); } function addDailyMetrics(result) { return Promise.all(result.timeseries.map(addDailyMetricsToTimeseries)).then( () => { result.totals.requests.package = sumMaps( result.timeseries.map(timeseries => { return timeseries.requests.package; }) ); result.totals.bandwidth.package = sumMaps( result.timeseries.map(timeseries => timeseries.bandwidth.package) ); result.totals.requests.protocol = sumMaps( result.timeseries.map(timeseries => timeseries.requests.protocol) ); return result; } ); } function extractPublicInfo(data) { return { since: data.since, until: data.until, requests: { all: data.requests.all, cached: data.requests.cached, country: data.requests.country, status: data.requests.http_status }, bandwidth: { all: data.bandwidth.all, cached: data.bandwidth.cached, country: data.bandwidth.country }, threats: { all: data.threats.all, country: data.threats.country }, uniques: { all: data.uniques.all } }; } const DomainNames = ["unpkg.com", "npmcdn.com"]; function fetchStats(since, until) { return CloudflareAPI.getZones(DomainNames).then(zones => { return CloudflareAPI.getZoneAnalyticsDashboard(zones, since, until).then( dashboard => { return { timeseries: dashboard.timeseries.map(extractPublicInfo), totals: extractPublicInfo(dashboard.totals) }; } ); }); } const oneMinute = 1000 * 60; const oneHour = oneMinute * 60; const oneDay = oneHour * 24; function getStats(since, until) { const promise = fetchStats(since, until); return until - since > oneDay ? promise.then(addDailyMetrics) : promise; } module.exports = { createDayKey, createHourKey, createMinuteKey, getStats };