Experimental port to Firebase hosting

modules/functions/index.js

@@ -0,0 +1,15 @@
+import { https } from 'firebase-functions';
+
+// import serveAuth from './serveAuth';
+import serveAutoIndexPage from './serveAutoIndexPage';
+import serveNpmPackageFile from './serveNpmPackageFile';
+import servePublicKey from './servePublicKey';
+import serveStats from './serveStats';
+
+export default {
+  // serveAuth: https.onRequest(serveAuth),
+  serveAutoIndexPage: https.onRequest(serveAutoIndexPage),
+  serveNpmPackageFile: https.onRequest(serveNpmPackageFile),
+  servePublicKey: https.onRequest(servePublicKey),
+  serveStats: https.onRequest(serveStats)
+};

modules/functions/ingestLogs.js

@@ -0,0 +1,192 @@
+import url from 'url';
+import { startOfDay, addDays } from 'date-fns';
+
+import data from '../utils/data';
+import isValidPackageName from '../utils/isValidPackageName';
+import parsePackageURL from '../utils/parsePackageURL';
+import * as cloudflare from '../utils/cloudflare';
+import * as stats from '../utils/stats';
+
+/**
+ * Domains we want to analyze.
+ */
+const domainNames = [
+  'unpkg.com'
+  //"npmcdn.com" // We don't have log data on npmcdn.com yet :/
+];
+
+let cachedZones;
+
+function getSeconds(date) {
+  return Math.floor(date.getTime() / 1000);
+}
+
+function stringifySeconds(seconds) {
+  return new Date(seconds * 1000).toISOString().replace(/\.0+Z$/, 'Z');
+}
+
+function toSeconds(ms) {
+  return Math.floor(ms / 1000);
+}
+
+function computeCounters(stream) {
+  return new Promise((resolve, reject) => {
+    const counters = {};
+    const expireat = {};
+    let totalEntries = 0;
+
+    function incr(key, member, by, expiry) {
+      counters[key] = counters[key] || {};
+      counters[key][member] = (counters[key][member] || 0) + by;
+      expireat[key] = expiry;
+    }
+
+    stream
+      .on('error', reject)
+      .on('data', entry => {
+        totalEntries += 1;
+
+        const date = new Date(Math.round(entry.EdgeStartTimestamp / 1000000));
+
+        const nextDay = startOfDay(addDays(date, 1));
+        const sevenDaysLater = getSeconds(addDays(nextDay, 7));
+        const thirtyDaysLater = getSeconds(addDays(nextDay, 30));
+        const dayKey = stats.createDayKey(date);
+
+        if (entry.EdgeResponseStatus === 200) {
+          // Q: How many requests do we serve for a package per day?
+          // Q: How many bytes do we serve for a package per day?
+          const parsed = parsePackageURL(entry.ClientRequestURI);
+          const packageName = parsed && parsed.packageName;
+
+          if (packageName && isValidPackageName(packageName)) {
+            incr(
+              `stats-packageRequests-${dayKey}`,
+              packageName,
+              1,
+              thirtyDaysLater
+            );
+            incr(
+              `stats-packageBytes-${dayKey}`,
+              packageName,
+              entry.EdgeResponseBytes,
+              thirtyDaysLater
+            );
+          }
+        }
+
+        // Q: How many requests per day do we receive via a protocol?
+        const protocol = entry.ClientRequestProtocol;
+
+        if (protocol) {
+          incr(
+            `stats-protocolRequests-${dayKey}`,
+            protocol,
+            1,
+            thirtyDaysLater
+          );
+        }
+
+        // Q: How many requests do we receive from a hostname per day?
+        // Q: How many bytes do we serve to a hostname per day?
+        const referer = entry.ClientRequestReferer;
+        const hostname = referer && url.parse(referer).hostname;
+
+        if (hostname) {
+          incr(`stats-hostnameRequests-${dayKey}`, hostname, 1, sevenDaysLater);
+          incr(
+            `stats-hostnameBytes-${dayKey}`,
+            hostname,
+            entry.EdgeResponseBytes,
+            sevenDaysLater
+          );
+        }
+      })
+      .on('end', () => {
+        resolve({ counters, expireat, totalEntries });
+      });
+  });
+}
+
+function processLogs(stream) {
+  return computeCounters(stream).then(
+    ({ counters, expireat, totalEntries }) => {
+      Object.keys(counters).forEach(key => {
+        const values = counters[key];
+
+        Object.keys(values).forEach(member => {
+          data.zincrby(key, values[member], member);
+        });
+
+        if (expireat[key]) {
+          data.expireat(key, expireat[key]);
+        }
+      });
+
+      return totalEntries;
+    }
+  );
+}
+
+function ingestLogsForZone(zone, startDate, endDate) {
+  const startSeconds = toSeconds(startDate);
+  const endSeconds = toSeconds(endDate);
+
+  const startFetchTime = Date.now();
+  const fields = [
+    'EdgeStartTimestamp',
+    'EdgeResponseStatus',
+    'EdgeResponseBytes',
+    'ClientRequestProtocol',
+    'ClientRequestURI',
+    'ClientRequestReferer'
+  ];
+
+  return cloudflare
+    .getLogs(
+      zone.id,
+      stringifySeconds(startSeconds),
+      stringifySeconds(endSeconds),
+      fields
+    )
+    .then(stream => {
+      const endFetchTime = Date.now();
+
+      console.log(
+        'Fetched logs for %s from %s to %s (%dms)',
+        zone.name,
+        stringifySeconds(startSeconds),
+        stringifySeconds(endSeconds),
+        endFetchTime - startFetchTime
+      );
+
+      const startProcessTime = Date.now();
+
+      return processLogs(stream).then(totalEntries => {
+        const endProcessTime = Date.now();
+
+        console.log(
+          'Processed %d log entries for %s (%dms)',
+          totalEntries,
+          zone.name,
+          endProcessTime - startProcessTime
+        );
+      });
+    });
+}
+
+function getZones(domainNames) {
+  return Promise.all(domainNames.map(cloudflare.getZones)).then(results =>
+    results.reduce((memo, zones) => memo.concat(zones))
+  );
+}
+
+export default function ingestLogs(startDate, endDate) {
+  return Promise.resolve(cachedZones || getZones(domainNames)).then(zones => {
+    if (!cachedZones) cachedZones = zones;
+
+    return Promise.all(
+      zones.map(zone => ingestLogsForZone(zone, startDate, endDate))
+    );
+  });
+}

modules/functions/serveAuth.js

@@ -0,0 +1,15 @@
+import express from 'express';
+import cors from 'cors';
+
+import userToken from '../middleware/userToken';
+import showAuth from '../actions/showAuth';
+
+const app = express();
+
+app.disable('x-powered-by');
+
+app.use(cors());
+app.use(userToken);
+app.use(showAuth);
+
+export default app;

modules/functions/serveAutoIndexPage.js

@@ -0,0 +1,27 @@
+import express from 'express';
+import cors from 'cors';
+
+// import checkBlacklist from '../middleware/checkBlacklist';
+import fetchPackage from '../middleware/fetchPackage';
+import findFile from '../middleware/findFile';
+import redirectLegacyURLs from '../middleware/redirectLegacyURLs';
+import validatePackageURL from '../middleware/validatePackageURL';
+import validatePackageName from '../middleware/validatePackageName';
+import validateQuery from '../middleware/validateQuery';
+import serveAutoIndexPage from '../actions/serveAutoIndexPage';
+
+const app = express();
+
+app.disable('x-powered-by');
+
+app.use(cors());
+app.use(redirectLegacyURLs);
+app.use(validatePackageURL);
+app.use(validatePackageName);
+app.use(validateQuery);
+// app.use(checkBlacklist);
+app.use(fetchPackage);
+app.use(findFile);
+app.use(serveAutoIndexPage);
+
+export default app;

modules/functions/serveNpmPackageFile.js

@@ -0,0 +1,27 @@
+import express from 'express';
+import cors from 'cors';
+
+// import checkBlacklist from '../middleware/checkBlacklist';
+import fetchPackage from '../middleware/fetchPackage';
+import findFile from '../middleware/findFile';
+import redirectLegacyURLs from '../middleware/redirectLegacyURLs';
+import validatePackageURL from '../middleware/validatePackageURL';
+import validatePackageName from '../middleware/validatePackageName';
+import validateQuery from '../middleware/validateQuery';
+import serveFile from '../actions/serveFile';
+
+const app = express();
+
+app.disable('x-powered-by');
+
+app.use(cors());
+app.use(redirectLegacyURLs);
+app.use(validatePackageURL);
+app.use(validatePackageName);
+app.use(validateQuery);
+// app.use(checkBlacklist);
+app.use(fetchPackage);
+app.use(findFile);
+app.use(serveFile);
+
+export default app;

modules/functions/servePublicKey.js

@@ -0,0 +1,13 @@
+import express from 'express';
+import cors from 'cors';
+
+import showPublicKey from '../actions/showPublicKey';
+
+const app = express();
+
+app.disable('x-powered-by');
+
+app.use(cors());
+app.use(showPublicKey);
+
+export default app;

modules/functions/serveStats.js

@@ -0,0 +1,13 @@
+import express from 'express';
+import cors from 'cors';
+
+import showStats from '../actions/showStats';
+
+const app = express();
+
+app.disable('x-powered-by');
+
+app.use(cors());
+app.use(showStats);
+
+export default app;