2018-02-18 02:00:56 +00:00
|
|
|
const parseURL = require("url").parse;
|
|
|
|
const startOfDay = require("date-fns/start_of_day");
|
2018-06-02 05:15:41 +00:00
|
|
|
const startOfMinute = require("date-fns/start_of_minute");
|
2018-02-18 02:00:56 +00:00
|
|
|
const addDays = require("date-fns/add_days");
|
2018-04-04 05:32:32 +00:00
|
|
|
|
|
|
|
const db = require("./utils/redis");
|
2018-05-26 00:30:53 +00:00
|
|
|
const isValidPackageName = require("./utils/isValidPackageName");
|
2018-02-18 02:00:56 +00:00
|
|
|
const parsePackageURL = require("./utils/parsePackageURL");
|
2018-04-04 05:32:32 +00:00
|
|
|
|
2018-02-18 02:00:56 +00:00
|
|
|
const CloudflareAPI = require("./CloudflareAPI");
|
|
|
|
const StatsAPI = require("./StatsAPI");
|
2017-11-08 19:07:48 +00:00
|
|
|
|
2017-05-23 22:00:09 +00:00
|
|
|
/**
|
|
|
|
* Domains we want to analyze.
|
|
|
|
*/
|
2018-04-04 05:32:32 +00:00
|
|
|
const domainNames = [
|
2017-11-25 21:25:01 +00:00
|
|
|
"unpkg.com"
|
2018-02-18 04:21:19 +00:00
|
|
|
//"npmcdn.com" // We don't have log data on npmcdn.com yet :/
|
2018-02-18 02:00:56 +00:00
|
|
|
];
|
2017-05-23 22:00:09 +00:00
|
|
|
|
|
|
|
/**
|
|
|
|
* The window of time to download in a single fetch.
|
|
|
|
*/
|
2018-06-02 05:15:41 +00:00
|
|
|
const logWindowSeconds = 60;
|
|
|
|
|
|
|
|
/**
|
|
|
|
* The minimum time to wait between fetches.
|
|
|
|
*/
|
|
|
|
const minInterval = 15000;
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2017-08-17 18:24:40 +00:00
|
|
|
function getSeconds(date) {
|
2018-02-18 02:00:56 +00:00
|
|
|
return Math.floor(date.getTime() / 1000);
|
2017-08-12 16:18:54 +00:00
|
|
|
}
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2017-08-12 16:18:54 +00:00
|
|
|
function stringifySeconds(seconds) {
|
2018-06-02 05:15:41 +00:00
|
|
|
return new Date(seconds * 1000).toISOString().replace(/\.0+Z$/, "Z");
|
2017-08-12 16:18:54 +00:00
|
|
|
}
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
function toSeconds(ms) {
|
|
|
|
return Math.floor(ms / 1000);
|
2017-05-20 05:41:24 +00:00
|
|
|
}
|
|
|
|
|
2018-02-18 02:00:56 +00:00
|
|
|
const oneSecond = 1000;
|
|
|
|
const oneMinute = oneSecond * 60;
|
|
|
|
const oneHour = oneMinute * 60;
|
2018-06-02 05:15:41 +00:00
|
|
|
const oneDay = oneHour * 24;
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2017-08-12 16:18:54 +00:00
|
|
|
function computeCounters(stream) {
|
2017-11-08 18:14:46 +00:00
|
|
|
return new Promise((resolve, reject) => {
|
2018-02-18 02:00:56 +00:00
|
|
|
const counters = {};
|
|
|
|
const expireat = {};
|
2018-06-02 05:15:41 +00:00
|
|
|
let totalEntries = 0;
|
2017-05-24 16:25:00 +00:00
|
|
|
|
2017-08-17 18:24:40 +00:00
|
|
|
function incr(key, member, by, expiry) {
|
2018-02-18 02:00:56 +00:00
|
|
|
counters[key] = counters[key] || {};
|
|
|
|
counters[key][member] = (counters[key][member] || 0) + by;
|
|
|
|
expireat[key] = expiry;
|
2017-05-24 16:25:00 +00:00
|
|
|
}
|
2017-05-23 22:00:09 +00:00
|
|
|
|
|
|
|
stream
|
2017-11-25 21:25:01 +00:00
|
|
|
.on("error", reject)
|
2018-06-02 05:15:41 +00:00
|
|
|
.on("data", entry => {
|
|
|
|
totalEntries += 1;
|
|
|
|
|
|
|
|
const date = new Date(Math.round(entry.EdgeStartTimestamp / 1000000));
|
2017-08-17 18:24:40 +00:00
|
|
|
|
2018-02-18 02:00:56 +00:00
|
|
|
const nextDay = startOfDay(addDays(date, 1));
|
|
|
|
const sevenDaysLater = getSeconds(addDays(nextDay, 7));
|
|
|
|
const thirtyDaysLater = getSeconds(addDays(nextDay, 30));
|
|
|
|
const dayKey = StatsAPI.createDayKey(date);
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
if (entry.EdgeResponseStatus === 200) {
|
2017-08-17 18:24:40 +00:00
|
|
|
// Q: How many requests do we serve for a package per day?
|
|
|
|
// Q: How many bytes do we serve for a package per day?
|
2018-06-02 05:15:41 +00:00
|
|
|
const url = parsePackageURL(entry.ClientRequestURI);
|
2018-02-18 02:00:56 +00:00
|
|
|
const packageName = url && url.packageName;
|
2017-05-29 05:41:01 +00:00
|
|
|
|
2018-05-26 00:30:53 +00:00
|
|
|
if (packageName && isValidPackageName(packageName)) {
|
2018-02-18 02:00:56 +00:00
|
|
|
incr(
|
|
|
|
`stats-packageRequests-${dayKey}`,
|
|
|
|
packageName,
|
|
|
|
1,
|
|
|
|
thirtyDaysLater
|
|
|
|
);
|
|
|
|
incr(
|
|
|
|
`stats-packageBytes-${dayKey}`,
|
|
|
|
packageName,
|
2018-06-02 05:15:41 +00:00
|
|
|
entry.EdgeResponseBytes,
|
2018-02-18 02:00:56 +00:00
|
|
|
thirtyDaysLater
|
|
|
|
);
|
2017-08-17 18:24:40 +00:00
|
|
|
}
|
2017-05-24 19:07:29 +00:00
|
|
|
}
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2017-08-17 18:24:40 +00:00
|
|
|
// Q: How many requests per day do we receive via a protocol?
|
2018-06-02 05:15:41 +00:00
|
|
|
const protocol = entry.ClientRequestProtocol;
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
if (protocol) {
|
2018-02-18 02:00:56 +00:00
|
|
|
incr(
|
|
|
|
`stats-protocolRequests-${dayKey}`,
|
|
|
|
protocol,
|
|
|
|
1,
|
|
|
|
thirtyDaysLater
|
|
|
|
);
|
2018-06-02 05:15:41 +00:00
|
|
|
}
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2017-05-24 19:07:29 +00:00
|
|
|
// Q: How many requests do we receive from a hostname per day?
|
|
|
|
// Q: How many bytes do we serve to a hostname per day?
|
2018-06-02 05:15:41 +00:00
|
|
|
const referer = entry.ClientRequestReferer;
|
2018-02-18 02:00:56 +00:00
|
|
|
const hostname = referer && parseURL(referer).hostname;
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2017-05-24 19:07:29 +00:00
|
|
|
if (hostname) {
|
2018-02-18 02:00:56 +00:00
|
|
|
incr(`stats-hostnameRequests-${dayKey}`, hostname, 1, sevenDaysLater);
|
|
|
|
incr(
|
|
|
|
`stats-hostnameBytes-${dayKey}`,
|
|
|
|
hostname,
|
2018-06-02 05:15:41 +00:00
|
|
|
entry.EdgeResponseBytes,
|
2018-02-18 02:00:56 +00:00
|
|
|
sevenDaysLater
|
|
|
|
);
|
2017-05-24 19:07:29 +00:00
|
|
|
}
|
2017-05-23 22:00:09 +00:00
|
|
|
})
|
2018-06-02 05:15:41 +00:00
|
|
|
.on("end", () => {
|
|
|
|
resolve({ counters, expireat, totalEntries });
|
2018-02-18 02:00:56 +00:00
|
|
|
});
|
|
|
|
});
|
2017-08-12 16:18:54 +00:00
|
|
|
}
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2017-08-12 16:18:54 +00:00
|
|
|
function processLogs(stream) {
|
2018-06-02 05:15:41 +00:00
|
|
|
return computeCounters(stream).then(
|
|
|
|
({ counters, expireat, totalEntries }) => {
|
|
|
|
Object.keys(counters).forEach(key => {
|
|
|
|
const values = counters[key];
|
|
|
|
|
|
|
|
Object.keys(values).forEach(member => {
|
|
|
|
db.zincrby(key, values[member], member);
|
|
|
|
});
|
2017-05-24 16:25:00 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
if (expireat[key]) {
|
|
|
|
db.expireat(key, expireat[key]);
|
|
|
|
}
|
2018-02-18 02:00:56 +00:00
|
|
|
});
|
2017-05-29 05:41:01 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
return totalEntries;
|
|
|
|
}
|
|
|
|
);
|
2017-08-12 16:18:54 +00:00
|
|
|
}
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2017-08-12 16:18:54 +00:00
|
|
|
function ingestLogs(zone, startSeconds, endSeconds) {
|
2018-06-02 05:15:41 +00:00
|
|
|
const startFetchTime = Date.now();
|
|
|
|
const fields = [
|
|
|
|
"EdgeStartTimestamp",
|
|
|
|
"EdgeResponseStatus",
|
|
|
|
"EdgeResponseBytes",
|
|
|
|
"ClientRequestProtocol",
|
|
|
|
"ClientRequestURI",
|
|
|
|
"ClientRequestReferer"
|
|
|
|
];
|
|
|
|
|
|
|
|
return CloudflareAPI.getLogs(
|
|
|
|
zone.id,
|
|
|
|
stringifySeconds(startSeconds),
|
|
|
|
stringifySeconds(endSeconds),
|
|
|
|
fields
|
|
|
|
).then(stream => {
|
|
|
|
const endFetchTime = Date.now();
|
|
|
|
|
2017-05-20 05:41:24 +00:00
|
|
|
console.log(
|
2018-06-02 05:15:41 +00:00
|
|
|
"info: Fetched logs for %s from %s to %s (%dms)",
|
2017-05-20 05:41:24 +00:00
|
|
|
zone.name,
|
|
|
|
stringifySeconds(startSeconds),
|
2018-06-02 05:15:41 +00:00
|
|
|
stringifySeconds(endSeconds),
|
|
|
|
endFetchTime - startFetchTime
|
2018-02-18 02:00:56 +00:00
|
|
|
);
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
const startProcessTime = Date.now();
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
return processLogs(stream).then(totalEntries => {
|
|
|
|
const endProcessTime = Date.now();
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
console.log(
|
|
|
|
"info: Processed %d log entries for %s (%dms)",
|
|
|
|
totalEntries,
|
|
|
|
zone.name,
|
|
|
|
endProcessTime - startProcessTime
|
|
|
|
);
|
|
|
|
});
|
2018-02-18 02:00:56 +00:00
|
|
|
});
|
2017-08-12 16:18:54 +00:00
|
|
|
}
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2017-08-12 16:18:54 +00:00
|
|
|
function startZone(zone) {
|
2018-06-02 05:15:41 +00:00
|
|
|
const suffix = zone.name.replace(".", "-");
|
|
|
|
const startSecondsKey = `ingestLogs-start-${suffix}`;
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2017-08-12 16:18:54 +00:00
|
|
|
function takeATurn() {
|
2018-06-02 05:15:41 +00:00
|
|
|
const now = Date.now();
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
// Cloudflare keeps logs around for 7 days.
|
|
|
|
// https://support.cloudflare.com/hc/en-us/articles/216672448-Enterprise-Log-Share-Logpull-REST-API
|
|
|
|
const minSeconds = toSeconds(startOfMinute(now - oneDay * 5));
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
db.get(startSecondsKey, (error, value) => {
|
|
|
|
let startSeconds = value && parseInt(value, 10);
|
2017-05-20 05:41:24 +00:00
|
|
|
|
|
|
|
if (startSeconds == null) {
|
2018-02-18 02:00:56 +00:00
|
|
|
startSeconds = minSeconds;
|
2017-05-20 05:41:24 +00:00
|
|
|
} else if (startSeconds < minSeconds) {
|
|
|
|
console.warn(
|
2017-11-25 21:25:01 +00:00
|
|
|
"warning: Dropped logs for %s from %s to %s!",
|
2017-05-20 05:41:24 +00:00
|
|
|
zone.name,
|
|
|
|
stringifySeconds(startSeconds),
|
|
|
|
stringifySeconds(minSeconds)
|
2018-02-18 02:00:56 +00:00
|
|
|
);
|
2017-05-20 05:41:24 +00:00
|
|
|
|
2018-02-18 02:00:56 +00:00
|
|
|
startSeconds = minSeconds;
|
2017-05-20 05:41:24 +00:00
|
|
|
}
|
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
const endSeconds = startSeconds + logWindowSeconds;
|
|
|
|
|
2017-05-23 22:00:09 +00:00
|
|
|
// The log for a request is typically available within thirty (30) minutes
|
|
|
|
// of the request taking place under normal conditions. We deliver logs
|
|
|
|
// ordered by the time that the logs were created, i.e. the timestamp of
|
|
|
|
// the request when it was received by the edge. Given the order of
|
|
|
|
// delivery, we recommend waiting a full thirty minutes to ingest a full
|
|
|
|
// set of logs. This will help ensure that any congestion in the log
|
|
|
|
// pipeline has passed and a full set of logs can be ingested.
|
|
|
|
// https://support.cloudflare.com/hc/en-us/articles/216672448-Enterprise-Log-Share-REST-API
|
2018-02-18 02:00:56 +00:00
|
|
|
const maxSeconds = toSeconds(now - oneMinute * 30);
|
2017-05-23 22:00:09 +00:00
|
|
|
|
2018-06-02 05:15:41 +00:00
|
|
|
if (endSeconds < maxSeconds) {
|
2017-11-08 16:57:15 +00:00
|
|
|
ingestLogs(zone, startSeconds, endSeconds).then(
|
2018-06-02 05:15:41 +00:00
|
|
|
() => {
|
2018-02-18 02:00:56 +00:00
|
|
|
db.set(startSecondsKey, endSeconds);
|
2018-06-02 05:15:41 +00:00
|
|
|
setTimeout(takeATurn, minInterval);
|
2017-11-08 16:57:15 +00:00
|
|
|
},
|
2018-06-02 05:15:41 +00:00
|
|
|
error => {
|
2018-02-18 02:00:56 +00:00
|
|
|
console.error(error.stack);
|
|
|
|
process.exit(1);
|
2017-11-08 16:57:15 +00:00
|
|
|
}
|
2018-02-18 02:00:56 +00:00
|
|
|
);
|
2017-05-20 05:41:24 +00:00
|
|
|
} else {
|
2018-02-18 02:00:56 +00:00
|
|
|
setTimeout(takeATurn, (startSeconds - maxSeconds) * 1000);
|
2017-05-20 05:41:24 +00:00
|
|
|
}
|
2018-02-18 02:00:56 +00:00
|
|
|
});
|
2017-05-20 05:41:24 +00:00
|
|
|
}
|
|
|
|
|
2018-02-18 02:00:56 +00:00
|
|
|
takeATurn();
|
2017-05-20 05:41:24 +00:00
|
|
|
}
|
|
|
|
|
2018-04-04 05:32:32 +00:00
|
|
|
Promise.all(domainNames.map(CloudflareAPI.getZones)).then(results => {
|
2018-06-02 05:15:41 +00:00
|
|
|
const zones = results.reduce((memo, zones) => memo.concat(zones));
|
2018-02-18 02:00:56 +00:00
|
|
|
zones.forEach(startZone);
|
|
|
|
});
|