Remove Firebase hosting, back to Express

.firebaserc

@@ -1,5 +0,0 @@
-{
-  "projects": {
-    "default": "unpkg-staging"
-  }
-}

.gitignore

@@ -4,9 +4,8 @@ firebase-debug.log*
 npm-debug.log*
 
 /.env
-/functions/*.js
-/functions/node_modules/
 /node_modules/
 /public/_client/
 /secret_key
+/server.js
 /tokens/

firebase.json

@@ -1,13 +0,0 @@
-{
-  "hosting": {
-    "public": "public",
-    "ignore": ["firebase.json", "**/.*", "**/node_modules/**"],
-    "rewrites": [
-      { "source": "/api/auth", "function": "serveAuth" },
-      { "source": "/api/public-key", "function": "servePublicKey" },
-      { "source": "/api/stats", "function": "serveStats" },
-      { "source": "/", "function": "serveMainPage" },
-      { "source": "**", "function": "serveNpmPackageFile" }
-    ]
-  }
-}

functions/package.json

@@ -1,49 +0,0 @@
-{
-  "private": true,
-  "name": "functions",
-  "description": "Cloud Functions for Firebase",
-  "scripts": {
-    "start": "firebase serve --only functions",
-    "shell": "firebase functions:shell",
-    "deploy": "firebase deploy --only functions",
-    "logs": "firebase functions:log"
-  },
-  "dependencies": {
-    "@babel/core": "^7.2.2",
-    "@babel/plugin-proposal-class-properties": "^7.2.3",
-    "@babel/plugin-syntax-dynamic-import": "^7.2.0",
-    "@babel/plugin-syntax-export-default-from": "^7.2.0",
-    "@babel/plugin-syntax-export-namespace-from": "^7.2.0",
-    "@babel/plugin-syntax-import-meta": "^7.2.0",
-    "@emotion/core": "^10.0.6",
-    "cheerio": "^1.0.0-rc.2",
-    "cors": "^2.8.5",
-    "date-fns": "^1.30.1",
-    "etag": "^1.8.1",
-    "express": "^4.16.4",
-    "firebase-admin": "^6.0.0",
-    "firebase-functions": "^2.1.0",
-    "gunzip-maybe": "^1.4.1",
-    "invariant": "^2.2.4",
-    "isomorphic-fetch": "^2.2.1",
-    "jsonwebtoken": "^8.4.0",
-    "lru-cache": "^5.1.1",
-    "mime": "^2.4.0",
-    "morgan": "^1.9.1",
-    "ndjson": "^1.5.0",
-    "pretty-bytes": "^5.1.0",
-    "prop-types": "^15.6.2",
-    "react": "^16.7.0",
-    "react-dom": "^16.7.0",
-    "semver": "^5.6.0",
-    "sort-by": "^1.2.0",
-    "sri-toolbox": "^0.2.0",
-    "tar-stream": "^1.6.2",
-    "validate-npm-package-name": "^3.0.0",
-    "warning": "^4.0.2",
-    "whatwg-url": "^7.0.0"
-  },
-  "engines": {
-    "node": "8"
-  }
-}

modules/actions/serveAuth.js

@@ -0,0 +1,3 @@
+export default function serveAuth(req, res) {
+  res.send({ auth: req.user });
+}

modules/actions/serveFile.js

@@ -1,8 +1,8 @@
 import serveAutoIndexPage from './serveAutoIndexPage';
 import serveHTMLModule from './serveHTMLModule';
 import serveJavaScriptModule from './serveJavaScriptModule';
-import serveStaticFile from './serveStaticFile';
 import serveMetadata from './serveMetadata';
+import serveStaticFile from './serveStaticFile';
 
 /**
  * Send the file, JSON metadata, or HTML directory listing.

modules/actions/servePublicKey.js

@@ -0,0 +1,5 @@
+import { publicKey } from '../utils/secret';
+
+export default function servePublicKey(req, res) {
+  res.send({ publicKey });
+}

modules/actions/serveStats.js

@@ -2,7 +2,7 @@ import { subDays, startOfDay, startOfSecond } from 'date-fns';
 
 import { getStats } from '../utils/stats';
 
-export default function showStats(req, res) {
+export default function serveStats(req, res) {
   let since, until;
   if (req.query.period) {
     switch (req.query.period) {

modules/actions/showAuth.js

@@ -1,3 +0,0 @@
-export default function showAuth(req, res) {
-  res.send({ auth: req.user });
-}

modules/actions/showPublicKey.js

@@ -1,5 +0,0 @@
-import { secretKey } from '../config';
-
-export default function showPublicKey(req, res) {
-  res.send({ publicKey: secretKey.public });
-}

modules/config.js

@@ -1,10 +0,0 @@
-import invariant from 'invariant';
-
-export const npmRegistryURL =
-  process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org';
-
-export const origin = process.env.ORIGIN || 'https://unpkg.com';
-
-export const secretKey = process.env.SECRET_KEY;
-
-invariant(secretKey, 'Missing $SECRET_KEY environment variable');

modules/functions/index.js

@@ -1,15 +0,0 @@
-import { https } from 'firebase-functions';
-
-// import serveAuth from './serveAuth';
-import serveMainPage from './serveMainPage';
-import serveNpmPackageFile from './serveNpmPackageFile';
-import servePublicKey from './servePublicKey';
-import serveStats from './serveStats';
-
-export default {
-  // serveAuth: https.onRequest(serveAuth),
-  serveMainPage: https.onRequest(serveMainPage),
-  serveNpmPackageFile: https.onRequest(serveNpmPackageFile),
-  servePublicKey: https.onRequest(servePublicKey),
-  serveStats: https.onRequest(serveStats)
-};

modules/functions/serveAuth.js

@@ -1,15 +0,0 @@
-import cors from 'cors';
-import express from 'express';
-
-import logging from '../middleware/logging';
-import userToken from '../middleware/userToken';
-import showAuth from '../actions/showAuth';
-
-const app = express.Router();
-
-app.use(logging);
-app.use(cors());
-app.use(userToken);
-app.use(showAuth);
-
-export default app;

modules/functions/serveMainPage.js

@@ -1,11 +0,0 @@
-import express from 'express';
-
-import logging from '../middleware/logging';
-import serveMainPage from '../actions/serveMainPage';
-
-const app = express.Router();
-
-app.use(logging);
-app.use(serveMainPage);
-
-export default app;

modules/functions/serveNpmPackageFile.js

@@ -1,25 +0,0 @@
-import cors from 'cors';
-import express from 'express';
-
-import fetchPackage from '../middleware/fetchPackage';
-import findFile from '../middleware/findFile';
-import logging from '../middleware/logging';
-import redirectLegacyURLs from '../middleware/redirectLegacyURLs';
-import validatePackageURL from '../middleware/validatePackageURL';
-import validatePackageName from '../middleware/validatePackageName';
-import validateQuery from '../middleware/validateQuery';
-import serveFile from '../actions/serveFile';
-
-const app = express.Router();
-
-app.use(logging);
-app.use(cors());
-app.use(redirectLegacyURLs);
-app.use(validatePackageURL);
-app.use(validatePackageName);
-app.use(validateQuery);
-app.use(fetchPackage);
-app.use(findFile);
-app.use(serveFile);
-
-export default app;

modules/functions/servePublicKey.js

@@ -1,13 +0,0 @@
-import cors from 'cors';
-import express from 'express';
-
-import logging from '../middleware/logging';
-import showPublicKey from '../actions/showPublicKey';
-
-const app = express.Router();
-
-app.use(logging);
-app.use(cors());
-app.use(showPublicKey);
-
-export default app;

modules/functions/serveStats.js

@@ -1,13 +0,0 @@
-import cors from 'cors';
-import express from 'express';
-
-import logging from '../middleware/logging';
-import showStats from '../actions/showStats';
-
-const app = express.Router();
-
-app.use(logging);
-app.use(cors());
-app.use(showStats);
-
-export default app;

modules/middleware/cors.js

@@ -0,0 +1,5 @@
+import corsMiddleware from 'cors';
+
+const cors = corsMiddleware();
+
+export default cors;

modules/middleware/logger.js

@@ -1,6 +1,6 @@
 import morgan from 'morgan';
 
-const logging = morgan(
+const logger = morgan(
   process.env.NODE_ENV === 'development'
     ? 'dev'
     : ':date[clf] - :method :url :status :res[content-length] - :response-time ms',
@@ -12,4 +12,4 @@ const logging = morgan(
   }
 );
 
-export default logging;
+export default logger;

modules/middleware/staticFiles.js

@@ -0,0 +1,12 @@
+import express from 'express';
+
+const staticMiddleware = express.static('public', { maxAge: '1y' });
+
+export default function staticFiles(req, res, next) {
+  if (req.query.meta != null) {
+    // Let ?meta requests fall through.
+    return next();
+  }
+
+  staticMiddleware(req, res, next);
+}

modules/server.js

@@ -0,0 +1,57 @@
+import express from 'express';
+
+// import serveAuth from './actions/serveAuth';
+import serveFile from './actions/serveFile';
+import serveMainPage from './actions/serveMainPage';
+import servePublicKey from './actions/servePublicKey';
+import serveStats from './actions/serveStats';
+
+import cors from './middleware/cors';
+import fetchPackage from './middleware/fetchPackage';
+import findFile from './middleware/findFile';
+import logger from './middleware/logger';
+import redirectLegacyURLs from './middleware/redirectLegacyURLs';
+import staticFiles from './middleware/staticFiles';
+// import userToken from './middleware/userToken';
+import validatePackageURL from './middleware/validatePackageURL';
+import validatePackageName from './middleware/validatePackageName';
+import validateQuery from './middleware/validateQuery';
+
+import createRouter from './utils/createRouter';
+
+const port = process.env.PORT || '8080';
+
+const app = express();
+
+app.disable('x-powered-by');
+
+app.use(logger);
+app.use(staticFiles);
+
+app.get('/', serveMainPage);
+
+app.use(redirectLegacyURLs);
+app.use(cors);
+
+app.use(
+  '/api',
+  createRouter(app => {
+    // app.get('/auth', userToken, serveAuth);
+    app.get('/public-key', servePublicKey);
+    app.get('/stats', serveStats);
+  })
+);
+
+app.get(
+  '*',
+  validatePackageURL,
+  validatePackageName,
+  validateQuery,
+  fetchPackage,
+  findFile,
+  serveFile
+);
+
+app.listen(port, () => {
+  console.log('Server listening on port %s, Ctrl+C to quit', port);
+});

modules/utils/auth.js

@@ -2,7 +2,7 @@ import crypto from 'crypto';
 import jwt from 'jsonwebtoken';
 
 import data from './data';
-import { secretKey } from '../config';
+import { privateKey, publicKey } from './secret';
 
 function getCurrentSeconds() {
   return Math.floor(Date.now() / 1000);
@@ -21,18 +21,13 @@ export function createToken(scopes = {}) {
       scopes
     };
 
-    jwt.sign(
-      payload,
-      secretKey.private,
-      { algorithm: 'RS256' },
-      (error, token) => {
-        if (error) {
-          reject(error);
-        } else {
-          resolve(token);
-        }
+    jwt.sign(payload, privateKey, { algorithm: 'RS256' }, (error, token) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(token);
       }
-    );
+    });
   });
 }
 
@@ -42,7 +37,7 @@ export function verifyToken(token) {
   return new Promise((resolve, reject) => {
     const options = { algorithms: ['RS256'] };
 
-    jwt.verify(token, secretKey.public, options, (error, payload) => {
+    jwt.verify(token, publicKey, options, (error, payload) => {
       if (error) {
         reject(error);
       } else {

modules/utils/createRouter.js

@@ -0,0 +1,7 @@
+import express from 'express';
+
+export default function createRouter(configureRouter) {
+  const router = express.Router();
+  configureRouter(router);
+  return router;
+}

modules/utils/fetchNpmPackageInfo.js

@@ -1,12 +1,13 @@
 import url from 'url';
 import https from 'https';
 
-import { npmRegistryURL } from '../config';
-
 import debug from './debug';
 import bufferStream from './bufferStream';
 import agent from './registryAgent';
 
+const npmRegistryURL =
+  process.env.NPM_REGISTRY_URL || 'https://registry.npmjs.org';
+
 function parseJSON(res) {
   return bufferStream(res).then(JSON.parse);
 }

modules/utils/ingestLogs.js

@@ -1,11 +1,11 @@
 import url from 'url';
 import { startOfDay, addDays } from 'date-fns';
 
-import data from '../utils/data';
-import isValidPackageName from '../utils/isValidPackageName';
-import parsePackageURL from '../utils/parsePackageURL';
-import * as cloudflare from '../utils/cloudflare';
-import * as stats from '../utils/stats';
+import data from './data';
+import isValidPackageName from './isValidPackageName';
+import parsePackageURL from './parsePackageURL';
+import * as cloudflare from './cloudflare';
+import * as stats from './stats';
 
 /**
  * Domains we want to analyze.

modules/utils/rewriteBareModuleIdentifiers.js

@@ -1,8 +1,9 @@
 import babel from '@babel/core';
 
-import { origin } from '../config';
 import unpkgRewrite from '../plugins/unpkgRewrite';
 
+const origin = process.env.ORIGIN || 'https://unpkg.com';
+
 export default function rewriteBareModuleIdentifiers(code, packageConfig) {
   const dependencies = Object.assign(
     {},

modules/utils/secret.js

@@ -0,0 +1,8 @@
+import invariant from 'invariant';
+
+const secretKey = process.env.SECRET_KEY;
+
+invariant(secretKey, 'Missing $SECRET_KEY environment variable');
+
+export const privateKey = secretKey.private;
+export const publicKey = secretKey.public;

package.json

@@ -6,23 +6,44 @@
     "build": "rollup -c",
     "clean": "git clean -e '!/secret_key' -e '!/.env' -fdX .",
     "lint": "eslint modules",
-    "postinstall": "node ./scripts/postinstall.js",
-    "start": "firebase serve",
     "test": "jest",
     "watch": "rollup -c -w"
   },
   "dependencies": {
+    "@babel/core": "^7.2.2",
+    "@babel/plugin-proposal-class-properties": "^7.2.3",
+    "@babel/plugin-syntax-dynamic-import": "^7.2.0",
+    "@babel/plugin-syntax-export-default-from": "^7.2.0",
+    "@babel/plugin-syntax-export-namespace-from": "^7.2.0",
+    "@babel/plugin-syntax-import-meta": "^7.2.0",
     "@emotion/core": "^10.0.6",
+    "cheerio": "^1.0.0-rc.2",
+    "cors": "^2.8.5",
     "date-fns": "^1.30.1",
+    "etag": "^1.8.1",
+    "express": "^4.16.4",
+    "gunzip-maybe": "^1.4.1",
+    "invariant": "^2.2.4",
+    "isomorphic-fetch": "^2.2.1",
+    "jsonwebtoken": "^8.4.0",
+    "lru-cache": "^5.1.1",
+    "mime": "^2.4.0",
+    "morgan": "^1.9.1",
+    "ndjson": "^1.5.0",
     "pretty-bytes": "^5.1.0",
+    "prop-types": "^15.6.2",
     "react": "^16.7.0",
     "react-dom": "^16.7.0",
-    "sort-by": "^1.2.0"
+    "semver": "^5.6.0",
+    "sort-by": "^1.2.0",
+    "sri-toolbox": "^0.2.0",
+    "tar-stream": "^1.6.2",
+    "validate-npm-package-name": "^3.0.0",
+    "warning": "^4.0.2",
+    "whatwg-url": "^7.0.0"
   },
   "devDependencies": {
     "@ampproject/rollup-plugin-closure-compiler": "^0.8.5",
-    "@babel/core": "^7.2.2",
-    "@babel/plugin-proposal-class-properties": "^7.2.1",
     "@babel/preset-env": "^7.2.0",
     "@babel/preset-react": "^7.0.0",
     "babel-core": "^7.0.0-bridge.0",
@@ -33,8 +54,6 @@
     "eslint-import-resolver-node": "^0.3.2",
     "eslint-plugin-import": "^2.8.0",
     "eslint-plugin-react": "^7.12.3",
-    "invariant": "^2.2.2",
-    "isomorphic-fetch": "^2.2.1",
     "jest": "^22.4.4",
     "node-forge": "^0.7.6",
     "rollup": "^1.1.0",

rollup.config.js

@@ -11,6 +11,8 @@ const resolve = require('rollup-plugin-node-resolve');
 const url = require('rollup-plugin-url');
 
 const entryManifest = require('./plugins/entryManifest');
+const pkg = require('./package.json');
+const secretKey = require('./secretKey');
 
 const env = process.env.NODE_ENV || 'development';
 const dev = env === 'development';
@@ -59,70 +61,105 @@ const client = ['main', 'autoIndex'].map(entryName => {
   };
 });
 
-const secretKey = require('./secretKey');
-const fnsPkg = require('./functions/package.json');
-
-const fnsDeps = (dev
-  ? Object.keys(fnsPkg.dependencies).concat(
-      Object.keys(fnsPkg.devDependencies || {})
-    )
-  : Object.keys(fnsPkg.dependencies)
+const dependencies = (dev
+  ? Object.keys(pkg.dependencies).concat(Object.keys(pkg.devDependencies || {}))
+  : Object.keys(pkg.dependencies)
 ).concat('react-dom/server');
 
-const functions = [
-  {
-    external: id => true,
-    input: path.resolve(__dirname, 'modules/functions/index.js'),
-    output: { file: 'functions/index.js', format: 'cjs' },
-    plugins: [
-      babel(),
-      json(),
-      replace({
-        'process.env.NODE_ENV': JSON.stringify(env)
-      })
-    ]
-  }
-].concat(
-  [
-    // 'serveAuth',
-    'serveMainPage',
-    'serveNpmPackageFile',
-    'servePublicKey',
-    'serveStats'
-  ].map(functionName => {
-    return {
-      external: builtinModules.concat(fnsDeps),
-      input: path.resolve(__dirname, `modules/functions/${functionName}.js`),
-      output: { file: `functions/${functionName}.js`, format: 'cjs' },
-      plugins: [
-        manifest.inject({ virtualId: 'entry-manifest' }),
-        babel({ exclude: /node_modules/ }),
-        json(),
-        resolve(),
-        commonjs(),
-        url({
-          limit: 5 * 1024,
-          publicPath: '/_client/',
-          emitFiles: false
-        }),
-        replace({
-          'process.env.CLOUDFLARE_EMAIL': JSON.stringify(
-            process.env.CLOUDFLARE_EMAIL
-          ),
-          'process.env.CLOUDFLARE_KEY': JSON.stringify(
-            process.env.CLOUDFLARE_KEY
-          ),
-          'process.env.DEBUG': JSON.stringify(process.env.DEBUG),
-          'process.env.NODE_ENV': JSON.stringify(env),
-          'process.env.NPM_REGISTRY_URL': JSON.stringify(
-            process.env.NPM_REGISTRY_URL
-          ),
-          'process.env.ORIGIN': JSON.stringify(process.env.ORIGIN),
-          'process.env.SECRET_KEY': JSON.stringify(secretKey)
-        })
-      ]
-    };
-  })
-);
+const server = {
+  external: builtinModules.concat(dependencies),
+  input: path.resolve(__dirname, 'modules/server.js'),
+  output: { file: 'server.js', format: 'cjs' },
+  plugins: [
+    manifest.inject({ virtualId: 'entry-manifest' }),
+    babel({ exclude: /node_modules/ }),
+    json(),
+    resolve(),
+    commonjs(),
+    url({
+      limit: 5 * 1024,
+      publicPath: '/_client/',
+      emitFiles: false
+    }),
+    replace({
+      'process.env.CLOUDFLARE_EMAIL': JSON.stringify(
+        process.env.CLOUDFLARE_EMAIL
+      ),
+      'process.env.CLOUDFLARE_KEY': JSON.stringify(process.env.CLOUDFLARE_KEY),
+      'process.env.DEBUG': JSON.stringify(process.env.DEBUG),
+      'process.env.NODE_ENV': JSON.stringify(env),
+      'process.env.NPM_REGISTRY_URL': JSON.stringify(
+        process.env.NPM_REGISTRY_URL
+      ),
+      'process.env.ORIGIN': JSON.stringify(process.env.ORIGIN),
+      'process.env.SECRET_KEY': JSON.stringify(secretKey)
+    })
+  ]
+};
+// const fnsPkg = require('./functions/package.json');
 
-module.exports = client.concat(functions);
+// const fnsDeps = (dev
+//   ? Object.keys(fnsPkg.dependencies).concat(
+//       Object.keys(fnsPkg.devDependencies || {})
+//     )
+//   : Object.keys(fnsPkg.dependencies)
+// ).concat('react-dom/server');
+
+// const functions = [
+//   {
+//     external: id => true,
+//     input: path.resolve(__dirname, 'modules/functions/index.js'),
+//     output: { file: 'functions/index.js', format: 'cjs' },
+//     plugins: [
+//       babel(),
+//       json(),
+//       replace({
+//         'process.env.NODE_ENV': JSON.stringify(env)
+//       })
+//     ]
+//   }
+// ].concat(
+//   [
+//     // 'serveAuth',
+//     'serveMainPage',
+//     'serveNpmPackageFile',
+//     'servePublicKey',
+//     'serveStats'
+//   ].map(functionName => {
+//     return {
+//       external: builtinModules.concat(fnsDeps),
+//       input: path.resolve(__dirname, `modules/functions/${functionName}.js`),
+//       output: { file: `functions/${functionName}.js`, format: 'cjs' },
+//       plugins: [
+//         manifest.inject({ virtualId: 'entry-manifest' }),
+//         babel({ exclude: /node_modules/ }),
+//         json(),
+//         resolve(),
+//         commonjs(),
+//         url({
+//           limit: 5 * 1024,
+//           publicPath: '/_client/',
+//           emitFiles: false
+//         }),
+//         replace({
+//           'process.env.CLOUDFLARE_EMAIL': JSON.stringify(
+//             process.env.CLOUDFLARE_EMAIL
+//           ),
+//           'process.env.CLOUDFLARE_KEY': JSON.stringify(
+//             process.env.CLOUDFLARE_KEY
+//           ),
+//           'process.env.DEBUG': JSON.stringify(process.env.DEBUG),
+//           'process.env.NODE_ENV': JSON.stringify(env),
+//           'process.env.NPM_REGISTRY_URL': JSON.stringify(
+//             process.env.NPM_REGISTRY_URL
+//           ),
+//           'process.env.ORIGIN': JSON.stringify(process.env.ORIGIN),
+//           'process.env.SECRET_KEY': JSON.stringify(secretKey)
+//         })
+//       ]
+//     };
+//   })
+// );
+
+// module.exports = client.concat(functions);
+module.exports = client.concat(server);

scripts/postinstall.js

@@ -1,14 +0,0 @@
-const path = require('path');
-const execSync = require('child_process').execSync;
-
-function exec(cmd) {
-  execSync(cmd, { stdio: 'inherit', env: process.env });
-}
-
-process.chdir(path.resolve(__dirname, '../functions'));
-
-if (process.env.CI) {
-  exec('npm ci');
-} else {
-  exec('npm install');
-}