Add _search endpoint

server/createApp.js

@@ -10,6 +10,8 @@ const fetchFile = require('./middleware/fetchFile')
 const serveFile = require('./middleware/serveFile')
 const serveStats = require('./middleware/serveStats')
 
+const createSearchServer = require('./createSearchServer')
+
 morgan.token('fwd', function (req) {
   return req.get('x-forwarded-for').replace(/\s/g, '')
 })
@@ -48,6 +50,8 @@ function createApp() {
 
   app.use('/_stats', serveStats())
 
+  app.use('/_search', createSearchServer())
+
   app.use('/',
     packageURL,
     checkBlacklist(PackageBlacklist),

server/createSearchServer.js

@@ -0,0 +1,87 @@
+const express = require('express')
+const getAssetPaths = require('./npm/getAssetPaths')
+const npmSearchIndex = require('./npm/searchIndex')
+
+function enhanceHit(hit) {
+  return new Promise(function (resolve, reject) {
+    const assetPaths = getAssetPaths(hit.name, hit.version)
+
+    if (assetPaths) {
+      // TODO: Double check the package metadata to ensure the files
+      // haven't moved from the paths in the index?
+      hit.assets = assetPaths.map(function (path) {
+        return `https://unpkg.com/${hit.name}@${hit.version}${path}`
+      })
+
+      resolve(hit)
+    } else {
+      resolve(hit)
+    }
+  })
+}
+
+function byRelevanceDescending(a, b) {
+  // Hits that have assets are more relevant.
+  return a.assets ? (b.assets ? 0 : -1) : (b.assets ? 1 : 0)
+}
+
+function createSearchServer() {
+  const app = express()
+
+  app.get('/', function (req, res) {
+    const { query, page = 0 } = req.query
+    const hitsPerPage = 20
+
+    if (!query)
+      return res.status(403).send({ error: 'Missing ?query parameter' })
+
+    const params = {
+      typoTolerance: 'min',
+      attributesToRetrieve: [
+        'name',
+        'version',
+        'description',
+        'owner'
+      ],
+      attributesToHighlight: null,
+      restrictSearchableAttributes: [
+        'name',
+        'description'
+      ],
+      hitsPerPage,
+      page
+    }
+
+    npmSearchIndex.search(query, params, function (error, value) {
+      if (error) {
+        console.error(error)
+        res.status(500).send({ error: 'There was an error executing the search' })
+      } else {
+        Promise.all(
+          value.hits.map(enhanceHit)
+        ).then(function (hits) {
+          hits.sort(byRelevanceDescending)
+
+          const totalHits = value.nbHits
+          const totalPages = value.nbPages
+
+          res.send({
+            query,
+            page,
+            hitsPerPage,
+            totalHits,
+            totalPages,
+            hits
+          })
+        }, function (error) {
+          console.error(error)
+          res.status(500).send({ error: 'There was an error executing the search' })
+        })
+      }
+    })
+  })
+
+  return app
+}
+
+module.exports = createSearchServer

server/npm/assetPathsIndex.js

@@ -0,0 +1,144 @@
+/**
+ * A hand-built index of paths in npm packages that use globals. The index is
+ * not meant to contain *all* the global paths that a given package publishes,
+ * but rather those that are probably most useful for someone who wants to put
+ * a library on a page quickly in development.
+ *
+ * Each entry in the index is keyed by package name and contains a list of
+ * [ versionRange, ...paths ] for that package. The list is traversed in order,
+ * so version ranges that come sooner will match before those that come later.
+ * The range `null` is a catch-all.
+ */
+module.exports = {
+
+  'angular': [
+    [ '>=1.2.27', '/angular.min.js' ],
+    [ null, '/lib/angular.min.js' ]
+  ],
+
+  'animate.css': [
+    [ null, '/animate.min.css' ]
+  ],
+
+  'babel-standalone': [
+    [ null, '/babel.min.js' ]
+  ],
+
+  'backbone': [
+    [ null, '/backbone-min.js' ]
+  ],
+
+  'bootstrap': [
+    [ null, '/dist/css/bootstrap.min.css', '/dist/js/bootstrap.min.js' ]
+  ],
+
+  'bulma': [
+    [ null, '/css/bulma.css' ]
+  ],
+
+  'core.js': [
+    [ null, '/dist/core.min.js' ]
+  ],
+
+  'd3': [
+    [ null, '/build/d3.min.js' ]
+  ],
+
+  'ember-source': [
+    [ null, '/dist/ember.min.js' ]
+  ],
+
+  'foundation-sites': [
+    [ null, '/dist/css/foundation.min.css', '/dist/js/foundation.min.js' ]
+  ],
+
+  'gsap': [
+    [ null, '/TweenMax.js' ]
+  ],
+
+  'handlebars': [
+    [ null, '/dist/handlebars.min.js' ]
+  ],
+
+  'jquery': [
+    [ null, '/dist/jquery.min.js' ]
+  ],
+
+  'fastclick': [
+    [ null, '/lib/fastclick.js' ]
+  ],
+
+  'lodash': [
+    [ '<3', '/dist/lodash.min.js' ],
+    [ null, '/lodash.min.js' ]
+  ],
+
+  'masonry-layout': [
+    [ null, '/dist/masonry.pkgd.min.js' ]
+  ],
+
+  'materialize-css': [
+    [ null, '/dist/css/materialize.min.css' ]
+  ],
+
+  'react': [
+    [ '>=16.0.0-alpha.7', '/umd/react.production.min.js' ],
+    [ null, '/dist/react.min.js' ]
+  ],
+
+  'react-dom': [
+    [ '>=16.0.0-alpha.7', '/umd/react-dom.production.min.js' ],
+    [ null, '/dist/react-dom.min.js' ]
+  ],
+
+  'react-router': [
+    [ '>=4.0.0', '/umd/react-router.min.js' ],
+    [ null, '/umd/ReactRouter.min.js' ]
+  ],
+
+  'redux': [
+    [ null, '/dist/redux.min.js' ]
+  ],
+
+  'redux-saga': [
+    [ null, '/dist/redux-saga.min.js' ]
+  ],
+
+  'redux-thunk': [
+    [ null, '/dist/redux-thunk.min.js' ]
+  ],
+
+  'snapsvg': [
+    [ null, '/snap.svg-min.js' ]
+  ],
+
+  'systemjs': [
+    [ null, '/dist/system.js' ]
+  ],
+
+  'three': [
+    [ '<=0.77.0', '/three.min.js' ],
+    [ null, '/build/three.min.js' ]
+  ],
+
+  'underscore': [
+    [ null, '/underscore-min.js' ]
+  ],
+
+  'vue': [
+    [ null, '/dist/vue.min.js' ]
+  ],
+
+  'zepto': [
+    [ null, '/dist/zepto.min.js' ]
+  ],
+
+  'zingchart': [
+    [ null, '/client/zingchart.min.js' ]
+  ],
+
+  'zone.js': [
+    [ null, '/dist/zone.js' ]
+  ]
+
+}

server/npm/getAssetPaths.js

@@ -0,0 +1,20 @@
+const assetPathsIndex = require('./assetPathsIndex')
+
+function getAssetPaths(packageName, version) {
+  const entries = assetPathsIndex[packageName]
+
+  if (entries) {
+    const matchingEntry = entries.find(function (entry) {
+      const range = entry[0]
+
+      if (range == null || semver.satisfies(version, range))
+        return entry
+    })
+
+    return matchingEntry.slice(1)
+  }
+
+  return null
+}
+
+module.exports = getAssetPaths

server/npm/searchIndex.js

@@ -0,0 +1,22 @@
+const algolia = require('algoliasearch')
+const invariant = require('invariant')
+
+const AlgoliaNpmSearchAppId = process.env.ALGOLIA_NPM_SEARCH_APP_ID
+const AlgoliaNpmSearchApiKey = process.env.ALGOLIA_NPM_SEARCH_API_KEY
+
+invariant(
+  AlgoliaNpmSearchAppId,
+  'Missing $ALGOLIA_NPM_SEARCH_APP_ID environment variable'
+)
+
+invariant(
+  AlgoliaNpmSearchApiKey,
+  'Missing $ALGOLIA_NPM_SEARCH_API_KEY environment variable'
+)
+
+const index = algolia(
+  AlgoliaNpmSearchAppId,
+  AlgoliaNpmSearchApiKey
+).initIndex('npm-search')
+
+module.exports = index