Rename server => modules

modules/utils/__tests__/.eslintrc

@@ -0,0 +1,5 @@
+{
+  "env": {
+    "jest": true
+  }
+}

modules/utils/__tests__/createSearch-test.js

@@ -0,0 +1,15 @@
+const createSearch = require("../createSearch");
+
+describe("createSearch", () => {
+  it("omits the trailing = for empty string values", () => {
+    expect(createSearch({ a: "a", b: "" })).toEqual("?a=a&b");
+  });
+
+  it("sorts keys", () => {
+    expect(createSearch({ b: "b", a: "a", c: "c" })).toEqual("?a=a&b=b&c=c");
+  });
+
+  it("returns an empty string when there are no params", () => {
+    expect(createSearch({})).toEqual("");
+  });
+});

modules/utils/__tests__/getContentType-test.js

@@ -0,0 +1,39 @@
+const getContentType = require("../getContentType");
+
+it("gets a content type of text/plain for LICENSE|README|CHANGES|AUTHORS|Makefile", () => {
+  expect(getContentType("AUTHORS")).toBe("text/plain");
+  expect(getContentType("CHANGES")).toBe("text/plain");
+  expect(getContentType("LICENSE")).toBe("text/plain");
+  expect(getContentType("Makefile")).toBe("text/plain");
+  expect(getContentType("PATENTS")).toBe("text/plain");
+  expect(getContentType("README")).toBe("text/plain");
+});
+
+it("gets a content type of text/plain for .*rc files", () => {
+  expect(getContentType(".eslintrc")).toBe("text/plain");
+  expect(getContentType(".babelrc")).toBe("text/plain");
+  expect(getContentType(".anythingrc")).toBe("text/plain");
+});
+
+it("gets a content type of text/plain for .git* files", () => {
+  expect(getContentType(".gitignore")).toBe("text/plain");
+  expect(getContentType(".gitanything")).toBe("text/plain");
+});
+
+it("gets a content type of text/plain for .*ignore files", () => {
+  expect(getContentType(".eslintignore")).toBe("text/plain");
+  expect(getContentType(".anythingignore")).toBe("text/plain");
+});
+
+it("gets a content type of text/plain for .ts files", () => {
+  expect(getContentType("app.ts")).toBe("text/plain");
+  expect(getContentType("app.d.ts")).toBe("text/plain");
+});
+
+it("gets a content type of text/plain for .flow files", () => {
+  expect(getContentType("app.js.flow")).toBe("text/plain");
+});
+
+it("gets a content type of text/plain for .lock files", () => {
+  expect(getContentType("yarn.lock")).toBe("text/plain");
+});

modules/utils/__tests__/parsePackageURL-test.js

@@ -0,0 +1,80 @@
+const parsePackageURL = require("../parsePackageURL");
+
+describe("parsePackageURL", () => {
+  it("parses plain packages", () => {
+    expect(parsePackageURL("/history@1.0.0/umd/history.min.js")).toEqual({
+      pathname: "/history@1.0.0/umd/history.min.js",
+      search: "",
+      query: {},
+      packageName: "history",
+      packageVersion: "1.0.0",
+      filename: "/umd/history.min.js"
+    });
+  });
+
+  it("parses plain packages with a hyphen in the name", () => {
+    expect(parsePackageURL("/query-string@5.0.0/index.js")).toEqual({
+      pathname: "/query-string@5.0.0/index.js",
+      search: "",
+      query: {},
+      packageName: "query-string",
+      packageVersion: "5.0.0",
+      filename: "/index.js"
+    });
+  });
+
+  it("parses plain packages with no version specified", () => {
+    expect(parsePackageURL("/query-string/index.js")).toEqual({
+      pathname: "/query-string/index.js",
+      search: "",
+      query: {},
+      packageName: "query-string",
+      packageVersion: "latest",
+      filename: "/index.js"
+    });
+  });
+
+  it("parses plain packages with version spec", () => {
+    expect(parsePackageURL("/query-string@>=4.0.0/index.js")).toEqual({
+      pathname: "/query-string@>=4.0.0/index.js",
+      search: "",
+      query: {},
+      packageName: "query-string",
+      packageVersion: ">=4.0.0",
+      filename: "/index.js"
+    });
+  });
+
+  it("parses scoped packages", () => {
+    expect(parsePackageURL("/@angular/router@4.3.3/src/index.d.ts")).toEqual({
+      pathname: "/@angular/router@4.3.3/src/index.d.ts",
+      search: "",
+      query: {},
+      packageName: "@angular/router",
+      packageVersion: "4.3.3",
+      filename: "/src/index.d.ts"
+    });
+  });
+
+  it("parses package names with a period in them", () => {
+    expect(parsePackageURL("/index.js")).toEqual({
+      pathname: "/index.js",
+      search: "",
+      query: {},
+      packageName: "index.js",
+      packageVersion: "latest",
+      filename: ""
+    });
+  });
+
+  it("parses valid query parameters", () => {
+    expect(parsePackageURL("/history?main=browser")).toEqual({
+      pathname: "/history",
+      search: "?main=browser",
+      query: { main: "browser" },
+      packageName: "history",
+      packageVersion: "latest",
+      filename: ""
+    });
+  });
+});

modules/utils/addLeadingSlash.js

@@ -0,0 +1,5 @@
+function addLeadingSlash(name) {
+  return name.charAt(0) === "/" ? name : "/" + name;
+}
+
+module.exports = addLeadingSlash;

modules/utils/bufferStream.js

@@ -0,0 +1,12 @@
+function bufferStream(stream) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+
+    stream
+      .on("error", reject)
+      .on("data", chunk => chunks.push(chunk))
+      .on("end", () => resolve(Buffer.concat(chunks)));
+  });
+}
+
+module.exports = bufferStream;

modules/utils/cache.js

@@ -0,0 +1,9 @@
+const redis = require("redis");
+
+redis.debug_mode = process.env.DEBUG_REDIS != null;
+
+const client = redis.createClient(
+  process.env.CACHE_URL || process.env.OPENREDIS_URL || "redis://localhost:6379"
+);
+
+module.exports = client;

modules/utils/createPackageURL.js

@@ -0,0 +1,11 @@
+function createPackageURL(packageName, version, pathname, search) {
+  let url = `/${packageName}`;
+
+  if (version != null) url += `@${version}`;
+  if (pathname) url += pathname;
+  if (search) url += search;
+
+  return url;
+}
+
+module.exports = createPackageURL;

modules/utils/createSearch.js

@@ -0,0 +1,16 @@
+function createSearch(query) {
+  const keys = Object.keys(query).sort();
+  const params = keys.reduce(
+    (memo, key) =>
+      memo.concat(
+        query[key] === ""
+          ? key // Omit the trailing "=" from key=
+          : `${key}=${encodeURIComponent(query[key])}`
+      ),
+    []
+  );
+
+  return params.length ? `?${params.join("&")}` : "";
+}
+
+module.exports = createSearch;

modules/utils/data.js

@@ -0,0 +1,9 @@
+const redis = require("redis");
+
+redis.debug_mode = process.env.DEBUG_REDIS != null;
+
+const client = redis.createClient(
+  process.env.DATA_URL || process.env.OPENREDIS_URL || "redis://localhost:6379"
+);
+
+module.exports = client;

modules/utils/fetchNpmPackage.js

@@ -0,0 +1,46 @@
+const url = require("url");
+const https = require("https");
+const gunzip = require("gunzip-maybe");
+const tar = require("tar-stream");
+
+const bufferStream = require("./bufferStream");
+const agent = require("./registryAgent");
+
+function fetchNpmPackage(packageConfig) {
+  return new Promise((resolve, reject) => {
+    const tarballURL = packageConfig.dist.tarball;
+
+    console.log(
+      `info: Fetching package for ${packageConfig.name} from ${tarballURL}`
+    );
+
+    const { hostname, pathname } = url.parse(tarballURL);
+    const options = {
+      agent: agent,
+      hostname: hostname,
+      path: pathname
+    };
+
+    https
+      .get(options, res => {
+        if (res.statusCode === 200) {
+          resolve(res.pipe(gunzip()).pipe(tar.extract()));
+        } else {
+          bufferStream(res).then(data => {
+            const spec = `${packageConfig.name}@${packageConfig.version}`;
+            const content = data.toString("utf-8");
+            const error = new Error(
+              `Failed to fetch tarball for ${spec}\nstatus: ${
+                res.statusCode
+              }\ndata: ${content}`
+            );
+
+            reject(error);
+          });
+        }
+      })
+      .on("error", reject);
+  });
+}
+
+module.exports = fetchNpmPackage;

modules/utils/fetchNpmPackageInfo.js

@@ -0,0 +1,58 @@
+const url = require("url");
+const https = require("https");
+
+const serverConfig = require("../serverConfig");
+const bufferStream = require("./bufferStream");
+const agent = require("./registryAgent");
+
+function parseJSON(res) {
+  return bufferStream(res).then(JSON.parse);
+}
+
+function fetchNpmPackageInfo(packageName) {
+  return new Promise((resolve, reject) => {
+    const encodedPackageName =
+      packageName.charAt(0) === "@"
+        ? `@${encodeURIComponent(packageName.substring(1))}`
+        : encodeURIComponent(packageName);
+
+    const infoURL = `${serverConfig.registryURL}/${encodedPackageName}`;
+
+    console.log(
+      `info: Fetching package info for ${packageName} from ${infoURL}`
+    );
+
+    const { hostname, pathname } = url.parse(infoURL);
+    const options = {
+      agent: agent,
+      hostname: hostname,
+      path: pathname,
+      headers: {
+        Accept: "application/json"
+      }
+    };
+
+    https
+      .get(options, res => {
+        if (res.statusCode === 200) {
+          resolve(parseJSON(res));
+        } else if (res.statusCode === 404) {
+          resolve(null);
+        } else {
+          bufferStream(res).then(data => {
+            const content = data.toString("utf-8");
+            const error = new Error(
+              `Failed to fetch info for ${packageName}\nstatus: ${
+                res.statusCode
+              }\ndata: ${content}`
+            );
+
+            reject(error);
+          });
+        }
+      })
+      .on("error", reject);
+  });
+}
+
+module.exports = fetchNpmPackageInfo;

modules/utils/getContentType.js

@@ -0,0 +1,22 @@
+const mime = require("mime");
+
+mime.define({
+  "text/plain": [
+    "authors",
+    "changes",
+    "license",
+    "makefile",
+    "patents",
+    "readme",
+    "ts",
+    "flow"
+  ]
+});
+
+const textFiles = /\/?(\.[a-z]*rc|\.git[a-z]*|\.[a-z]*ignore|\.lock)$/i;
+
+function getContentType(file) {
+  return textFiles.test(file) ? "text/plain" : mime.lookup(file);
+}
+
+module.exports = getContentType;

modules/utils/getContentTypeHeader.js

@@ -0,0 +1,5 @@
+function getContentTypeHeader(type) {
+  return type === "application/javascript" ? type + "; charset=utf-8" : type;
+}
+
+module.exports = getContentTypeHeader;

modules/utils/getIntegrity.js

@@ -0,0 +1,7 @@
+const SRIToolbox = require("sri-toolbox");
+
+function getIntegrity(data) {
+  return SRIToolbox.generate({ algorithms: ["sha384"] }, data);
+}
+
+module.exports = getIntegrity;

modules/utils/getNpmPackageInfo.js

@@ -0,0 +1,40 @@
+const cache = require("./cache");
+const fetchNpmPackageInfo = require("./fetchNpmPackageInfo");
+
+const notFound = "PackageNotFound";
+
+function getNpmPackageInfo(packageName) {
+  return new Promise((resolve, reject) => {
+    const key = `npmPackageInfo-${packageName}`;
+
+    cache.get(key, (error, value) => {
+      if (error) {
+        reject(error);
+      } else if (value != null) {
+        resolve(value === notFound ? null : JSON.parse(value));
+      } else {
+        fetchNpmPackageInfo(packageName).then(value => {
+          if (value == null) {
+            resolve(null);
+
+            // Cache 404s for 5 minutes. This prevents us from making
+            // unnecessary requests to the registry for bad package names.
+            // In the worst case, a brand new package's info will be
+            // available within 5 minutes.
+            cache.setex(key, 300, notFound);
+          } else {
+            resolve(value);
+
+            // Cache valid package info for 1 minute. In the worst case,
+            // new versions won't be available for 1 minute.
+            cache.setnx(key, JSON.stringify(value), (error, reply) => {
+              if (reply === 1) cache.expire(key, 60);
+            });
+          }
+        }, reject);
+      }
+    });
+  });
+}
+
+module.exports = getNpmPackageInfo;

modules/utils/incrementCounter.js

@@ -0,0 +1,15 @@
+const db = require("./data");
+
+function incrementCounter(counter, key, by = 1) {
+  return new Promise((resolve, reject) => {
+    db.hincrby(counter, key, by, (error, value) => {
+      if (error) {
+        reject(error);
+      } else {
+        resolve(value);
+      }
+    });
+  });
+}
+
+module.exports = incrementCounter;

modules/utils/isValidPackageName.js

@@ -0,0 +1,7 @@
+const validateNpmPackageName = require("validate-npm-package-name");
+
+function isValidPackageName(packageName) {
+  return validateNpmPackageName(packageName).errors == null;
+}
+
+module.exports = isValidPackageName;

modules/utils/parsePackageURL.js

@@ -0,0 +1,41 @@
+const url = require("url");
+
+const packageURLFormat = /^\/((?:@[^/@]+\/)?[^/@]+)(?:@([^/]+))?(\/.*)?$/;
+
+function decodeParam(param) {
+  if (param) {
+    try {
+      return decodeURIComponent(param);
+    } catch (error) {
+      // Ignore invalid params.
+    }
+  }
+
+  return "";
+}
+
+function parsePackageURL(originalURL) {
+  const { pathname, search, query } = url.parse(originalURL, true);
+  const match = packageURLFormat.exec(pathname);
+
+  // Disallow invalid URL formats.
+  if (match == null) {
+    return null;
+  }
+
+  const packageName = match[1];
+  const packageVersion = decodeParam(match[2]) || "latest";
+  const filename = decodeParam(match[3]);
+
+  return {
+    // If the URL is /@scope/name@version/file.js?main=browser:
+    pathname, // /@scope/name@version/path.js
+    search: search || "", // ?main=browser
+    query, // { main: 'browser' }
+    packageName, // @scope/name
+    packageVersion, // version
+    filename // /file.js
+  };
+}
+
+module.exports = parsePackageURL;

modules/utils/registryAgent.js

@@ -0,0 +1,7 @@
+const https = require("https");
+
+const agent = new https.Agent({
+  keepAlive: true
+});
+
+module.exports = agent;

modules/utils/renderPage.js

@@ -0,0 +1,11 @@
+const React = require("react");
+const ReactDOMServer = require("react-dom/server");
+
+const doctype = "<!DOCTYPE html>";
+
+function renderPage(page, props) {
+  const element = React.createElement(page, props);
+  return doctype + ReactDOMServer.renderToStaticMarkup(element);
+}
+
+module.exports = renderPage;