Finer-grained caching of data from npm

modules/middleware/fetchPackage.js

@@ -1,48 +1,20 @@
-import semver from 'semver';
-
 import addLeadingSlash from '../utils/addLeadingSlash.js';
 import createPackageURL from '../utils/createPackageURL.js';
 import createSearch from '../utils/createSearch.js';
-import { getPackageInfo as getNpmPackageInfo } from '../utils/npm.js';
-
-function tagRedirect(req, res) {
-  const version = req.packageInfo['dist-tags'][req.packageVersion];
+import { getPackageConfig, resolveVersion } from '../utils/npm.js';
 
+function semverRedirect(req, res, newVersion) {
   res
     .set({
       'Cache-Control': 'public, s-maxage=600, max-age=60', // 10 mins on CDN, 1 min on clients
-      'Cache-Tag': 'redirect, tag-redirect'
+      'Cache-Tag': 'redirect, semver-redirect'
     })
     .redirect(
       302,
-      createPackageURL(req.packageName, version, req.filename, req.search)
+      createPackageURL(req.packageName, newVersion, req.filename, req.search)
     );
 }
 
-function semverRedirect(req, res) {
-  const maxVersion = semver.maxSatisfying(
-    Object.keys(req.packageInfo.versions),
-    req.packageVersion
-  );
-
-  if (maxVersion) {
-    res
-      .set({
-        'Cache-Control': 'public, s-maxage=600, max-age=60', // 10 mins on CDN, 1 min on clients
-        'Cache-Tag': 'redirect, semver-redirect'
-      })
-      .redirect(
-        302,
-        createPackageURL(req.packageName, maxVersion, req.filename, req.search)
-      );
-  } else {
-    res
-      .status(404)
-      .type('text')
-      .send(`Cannot find package ${req.packageSpec}`);
-  }
-}
-
 function filenameRedirect(req, res) {
   let filename;
   if (req.query.module != null) {
@@ -110,40 +82,35 @@ function filenameRedirect(req, res) {
 }
 
 /**
- * Fetch the package metadata and tarball from npm. Redirect to the exact
- * version if the request targets a tag or uses a semver version, or to the
- * exact filename if the request omits the filename.
+ * Fetch the package config. Redirect to the exact version if the request
+ * targets a tag or uses semver, or to the exact filename if the request
+ * omits the filename.
  */
 export default async function fetchPackage(req, res, next) {
-  let packageInfo;
-  try {
-    packageInfo = await getNpmPackageInfo(req.packageName);
-  } catch (error) {
-    console.error(error);
+  const version = await resolveVersion(req.packageName, req.packageVersion);
 
-    return res
-      .status(500)
-      .type('text')
-      .send(`Cannot get info for package "${req.packageName}"`);
-  }
-
-  if (packageInfo == null || packageInfo.versions == null) {
+  if (!version) {
     return res
       .status(404)
       .type('text')
-      .send(`Cannot find package "${req.packageName}"`);
+      .send(`Cannot find package ${req.packageSpec}`);
   }
 
-  req.packageInfo = packageInfo;
-  req.packageConfig = req.packageInfo.versions[req.packageVersion];
+  if (version !== req.packageVersion) {
+    return semverRedirect(req, res, version);
+  }
+
+  req.packageConfig = await getPackageConfig(
+    req.packageName,
+    req.packageVersion
+  );
 
   if (!req.packageConfig) {
-    // Redirect to a fully-resolved version.
-    if (req.packageVersion in req.packageInfo['dist-tags']) {
-      return tagRedirect(req, res);
-    } else {
-      return semverRedirect(req, res);
-    }
+    // TODO: Log why.
+    return res
+      .status(500)
+      .type('text')
+      .send(`Cannot get config for package ${req.packageSpec}`);
   }
 
   if (!req.filename) {

modules/middleware/findFile.js

@@ -1,9 +1,11 @@
 import path from 'path';
+import gunzip from 'gunzip-maybe';
+import tar from 'tar-stream';
 
 import addLeadingSlash from '../utils/addLeadingSlash.js';
 import createPackageURL from '../utils/createPackageURL.js';
 import createSearch from '../utils/createSearch.js';
-import { fetchPackage as fetchNpmPackage } from '../utils/npm.js';
+import { getPackage } from '../utils/npm.js';
 import getIntegrity from '../utils/getIntegrity.js';
 import getContentType from '../utils/getContentType.js';
 
@@ -54,7 +56,7 @@ function stripLeadingSegment(name) {
  * Follows node's resolution algorithm.
  * https://nodejs.org/api/modules.html#modules_all_together
  */
-function searchEntries(tarballStream, entryName, wantsIndex) {
+function searchEntries(stream, entryName, wantsIndex) {
   return new Promise((resolve, reject) => {
     const jsEntryName = `${entryName}.js`;
     const jsonEntryName = `${entryName}.json`;
@@ -66,7 +68,9 @@ function searchEntries(tarballStream, entryName, wantsIndex) {
       foundEntry = entries[''] = { name: '', type: 'directory' };
     }
 
-    tarballStream
+    stream
+      .pipe(gunzip())
+      .pipe(tar.extract())
       .on('error', reject)
       .on('entry', (header, stream, next) => {
         const entry = {
@@ -173,9 +177,9 @@ export default async function findFile(req, res, next) {
     .replace(trailingSlash, '')
     .replace(leadingSlash, '');
 
-  const tarballStream = await fetchNpmPackage(req.packageConfig);
+  const stream = await getPackage(req.packageName, req.packageVersion);
   const { entries, foundEntry } = await searchEntries(
-    tarballStream,
+    stream,
     entryName,
     wantsIndex
   );