diff --git a/package.json b/package.json index 0133ca0..55f97fa 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "test": "node scripts/test.js --env=jsdom" }, "dependencies": { + "algoliasearch": "^3.24.3", "babel-plugin-unpkg-rewrite": "^3.1.0", "cors": "^2.8.1", "countries-list": "^1.3.2", diff --git a/server/createApp.js b/server/createApp.js index 9644629..e3e03cd 100644 --- a/server/createApp.js +++ b/server/createApp.js @@ -10,6 +10,8 @@ const fetchFile = require('./middleware/fetchFile') const serveFile = require('./middleware/serveFile') const serveStats = require('./middleware/serveStats') +const createSearchServer = require('./createSearchServer') + morgan.token('fwd', function (req) { return req.get('x-forwarded-for').replace(/\s/g, '') }) @@ -48,6 +50,8 @@ function createApp() { app.use('/_stats', serveStats()) + app.use('/_search', createSearchServer()) + app.use('/', packageURL, checkBlacklist(PackageBlacklist), diff --git a/server/createSearchServer.js b/server/createSearchServer.js new file mode 100644 index 0000000..b3ee71d --- /dev/null +++ b/server/createSearchServer.js @@ -0,0 +1,87 @@ +const express = require('express') +const getAssetPaths = require('./npm/getAssetPaths') +const npmSearchIndex = require('./npm/searchIndex') + +function enhanceHit(hit) { + return new Promise(function (resolve, reject) { + const assetPaths = getAssetPaths(hit.name, hit.version) + + if (assetPaths) { + // TODO: Double check the package metadata to ensure the files + // haven't moved from the paths in the index? + hit.assets = assetPaths.map(function (path) { + return `https://unpkg.com/${hit.name}@${hit.version}${path}` + }) + + resolve(hit) + } else { + resolve(hit) + } + }) +} + +function byRelevanceDescending(a, b) { + // Hits that have assets are more relevant. + return a.assets ? (b.assets ? 0 : -1) : (b.assets ? 1 : 0) +} + +function createSearchServer() { + const app = express() + + app.get('/', function (req, res) { + const { query, page = 0 } = req.query + const hitsPerPage = 20 + + if (!query) + return res.status(403).send({ error: 'Missing ?query parameter' }) + + const params = { + typoTolerance: 'min', + attributesToRetrieve: [ + 'name', + 'version', + 'description', + 'owner' + ], + attributesToHighlight: null, + restrictSearchableAttributes: [ + 'name', + 'description' + ], + hitsPerPage, + page + } + + npmSearchIndex.search(query, params, function (error, value) { + if (error) { + console.error(error) + res.status(500).send({ error: 'There was an error executing the search' }) + } else { + Promise.all( + value.hits.map(enhanceHit) + ).then(function (hits) { + hits.sort(byRelevanceDescending) + + const totalHits = value.nbHits + const totalPages = value.nbPages + + res.send({ + query, + page, + hitsPerPage, + totalHits, + totalPages, + hits + }) + }, function (error) { + console.error(error) + res.status(500).send({ error: 'There was an error executing the search' }) + }) + } + }) + }) + + return app +} + +module.exports = createSearchServer diff --git a/server/npm/assetPathsIndex.js b/server/npm/assetPathsIndex.js new file mode 100644 index 0000000..83064ab --- /dev/null +++ b/server/npm/assetPathsIndex.js @@ -0,0 +1,144 @@ +/** + * A hand-built index of paths in npm packages that use globals. The index is + * not meant to contain *all* the global paths that a given package publishes, + * but rather those that are probably most useful for someone who wants to put + * a library on a page quickly in development. + * + * Each entry in the index is keyed by package name and contains a list of + * [ versionRange, ...paths ] for that package. The list is traversed in order, + * so version ranges that come sooner will match before those that come later. + * The range `null` is a catch-all. + */ +module.exports = { + + 'angular': [ + [ '>=1.2.27', '/angular.min.js' ], + [ null, '/lib/angular.min.js' ] + ], + + 'animate.css': [ + [ null, '/animate.min.css' ] + ], + + 'babel-standalone': [ + [ null, '/babel.min.js' ] + ], + + 'backbone': [ + [ null, '/backbone-min.js' ] + ], + + 'bootstrap': [ + [ null, '/dist/css/bootstrap.min.css', '/dist/js/bootstrap.min.js' ] + ], + + 'bulma': [ + [ null, '/css/bulma.css' ] + ], + + 'core.js': [ + [ null, '/dist/core.min.js' ] + ], + + 'd3': [ + [ null, '/build/d3.min.js' ] + ], + + 'ember-source': [ + [ null, '/dist/ember.min.js' ] + ], + + 'foundation-sites': [ + [ null, '/dist/css/foundation.min.css', '/dist/js/foundation.min.js' ] + ], + + 'gsap': [ + [ null, '/TweenMax.js' ] + ], + + 'handlebars': [ + [ null, '/dist/handlebars.min.js' ] + ], + + 'jquery': [ + [ null, '/dist/jquery.min.js' ] + ], + + 'fastclick': [ + [ null, '/lib/fastclick.js' ] + ], + + 'lodash': [ + [ '<3', '/dist/lodash.min.js' ], + [ null, '/lodash.min.js' ] + ], + + 'masonry-layout': [ + [ null, '/dist/masonry.pkgd.min.js' ] + ], + + 'materialize-css': [ + [ null, '/dist/css/materialize.min.css' ] + ], + + 'react': [ + [ '>=16.0.0-alpha.7', '/umd/react.production.min.js' ], + [ null, '/dist/react.min.js' ] + ], + + 'react-dom': [ + [ '>=16.0.0-alpha.7', '/umd/react-dom.production.min.js' ], + [ null, '/dist/react-dom.min.js' ] + ], + + 'react-router': [ + [ '>=4.0.0', '/umd/react-router.min.js' ], + [ null, '/umd/ReactRouter.min.js' ] + ], + + 'redux': [ + [ null, '/dist/redux.min.js' ] + ], + + 'redux-saga': [ + [ null, '/dist/redux-saga.min.js' ] + ], + + 'redux-thunk': [ + [ null, '/dist/redux-thunk.min.js' ] + ], + + 'snapsvg': [ + [ null, '/snap.svg-min.js' ] + ], + + 'systemjs': [ + [ null, '/dist/system.js' ] + ], + + 'three': [ + [ '<=0.77.0', '/three.min.js' ], + [ null, '/build/three.min.js' ] + ], + + 'underscore': [ + [ null, '/underscore-min.js' ] + ], + + 'vue': [ + [ null, '/dist/vue.min.js' ] + ], + + 'zepto': [ + [ null, '/dist/zepto.min.js' ] + ], + + 'zingchart': [ + [ null, '/client/zingchart.min.js' ] + ], + + 'zone.js': [ + [ null, '/dist/zone.js' ] + ] + +} diff --git a/server/npm/getAssetPaths.js b/server/npm/getAssetPaths.js new file mode 100644 index 0000000..c0d869d --- /dev/null +++ b/server/npm/getAssetPaths.js @@ -0,0 +1,20 @@ +const assetPathsIndex = require('./assetPathsIndex') + +function getAssetPaths(packageName, version) { + const entries = assetPathsIndex[packageName] + + if (entries) { + const matchingEntry = entries.find(function (entry) { + const range = entry[0] + + if (range == null || semver.satisfies(version, range)) + return entry + }) + + return matchingEntry.slice(1) + } + + return null +} + +module.exports = getAssetPaths diff --git a/server/npm/searchIndex.js b/server/npm/searchIndex.js new file mode 100644 index 0000000..6278fc6 --- /dev/null +++ b/server/npm/searchIndex.js @@ -0,0 +1,22 @@ +const algolia = require('algoliasearch') +const invariant = require('invariant') + +const AlgoliaNpmSearchAppId = process.env.ALGOLIA_NPM_SEARCH_APP_ID +const AlgoliaNpmSearchApiKey = process.env.ALGOLIA_NPM_SEARCH_API_KEY + +invariant( + AlgoliaNpmSearchAppId, + 'Missing $ALGOLIA_NPM_SEARCH_APP_ID environment variable' +) + +invariant( + AlgoliaNpmSearchApiKey, + 'Missing $ALGOLIA_NPM_SEARCH_API_KEY environment variable' +) + +const index = algolia( + AlgoliaNpmSearchAppId, + AlgoliaNpmSearchApiKey +).initIndex('npm-search') + +module.exports = index diff --git a/yarn.lock b/yarn.lock index 106660e..e3bad96 100644 --- a/yarn.lock +++ b/yarn.lock @@ -41,6 +41,10 @@ acorn@^5.1.1: version "5.1.1" resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.1.1.tgz#53fe161111f912ab999ee887a90a0bc52822fd75" +agentkeepalive@^2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-2.2.0.tgz#c5d1bd4b129008f1163f236f86e5faea2026e2ef" + ajv-keywords@^1.0.0: version "1.5.1" resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-1.5.1.tgz#314dd0a4b3368fad3dfcdc54ede6171b886daf3c" @@ -52,6 +56,26 @@ ajv@^4.7.0, ajv@^4.9.1: co "^4.6.0" json-stable-stringify "^1.0.1" +algoliasearch@^3.24.3: + version "3.24.3" + resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-3.24.3.tgz#0b5da4242baa2e4bf9359bd20c47266ec0e501f4" + dependencies: + agentkeepalive "^2.2.0" + debug "^2.6.8" + envify "^4.0.0" + es6-promise "^4.1.0" + events "^1.1.0" + foreach "^2.0.5" + global "^4.3.2" + inherits "^2.0.1" + isarray "^2.0.1" + load-script "^1.0.0" + object-keys "^1.0.11" + querystring-es3 "^0.2.1" + reduce "^1.0.1" + semver "^5.1.0" + tunnel-agent "^0.6.0" + align-text@^0.1.1, align-text@^0.1.3: version "0.1.4" resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117" @@ -1635,6 +1659,10 @@ dom-serializer@0: domelementtype "~1.1.1" entities "~1.1.1" +dom-walk@^0.1.0: + version "0.1.1" + resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.1.tgz#672226dc74c8f799ad35307df936aba11acd6018" + domain-browser@^1.1.1: version "1.1.7" resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.1.7.tgz#867aa4b093faa05f1de08c06f4d7b21fdf8698bc" @@ -1737,6 +1765,13 @@ entities@~1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0" +envify@^4.0.0: + version "4.1.0" + resolved "https://registry.yarnpkg.com/envify/-/envify-4.1.0.tgz#f39ad3db9d6801b4e6b478b61028d3f0b6819f7e" + dependencies: + esprima "^4.0.0" + through "~2.3.4" + errno@^0.1.3, errno@^0.1.4: version "0.1.4" resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.4.tgz#b896e23a9e5e8ba33871fc996abd3635fc9a1c7d" @@ -1775,6 +1810,10 @@ es6-map@^0.1.3: es6-symbol "~3.1.1" event-emitter "~0.3.5" +es6-promise@^4.1.0: + version "4.1.1" + resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.1.1.tgz#8811e90915d9a0dba36274f0b242dbda78f9c92a" + es6-set@~0.1.5: version "0.1.5" resolved "https://registry.yarnpkg.com/es6-set/-/es6-set-0.1.5.tgz#d2b3ec5d4d800ced818db538d28974db0a73ccb1" @@ -1998,7 +2037,7 @@ eventemitter3@1.x.x: version "1.2.0" resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-1.2.0.tgz#1c86991d816ad1e504750e73874224ecf3bec508" -events@^1.0.0: +events@^1.0.0, events@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" @@ -2224,6 +2263,10 @@ for-own@^0.1.4: dependencies: for-in "^1.0.1" +foreach@^2.0.5: + version "2.0.5" + resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99" + forever-agent@~0.6.1: version "0.6.1" resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91" @@ -2355,6 +2398,13 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.0.5: once "^1.3.0" path-is-absolute "^1.0.0" +global@^4.3.2: + version "4.3.2" + resolved "https://registry.yarnpkg.com/global/-/global-4.3.2.tgz#e76989268a6c74c38908b1305b10fc0e394e9d0f" + dependencies: + min-document "^2.19.0" + process "~0.5.1" + globals@^9.14.0, globals@^9.18.0: version "9.18.0" resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a" @@ -2844,6 +2894,10 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" +isarray@^2.0.1: + version "2.0.2" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.2.tgz#5aa99638daf2248b10b9598b763a045688ece3ee" + isexe@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10" @@ -3266,6 +3320,10 @@ load-json-file@^1.0.0: pinkie-promise "^2.0.0" strip-bom "^2.0.0" +load-script@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/load-script/-/load-script-1.0.0.tgz#0491939e0bee5643ee494a7e3da3d2bac70c6ca4" + loader-utils@0.2.x, loader-utils@^0.2.11, loader-utils@^0.2.16, loader-utils@^0.2.3, loader-utils@^0.2.7, loader-utils@~0.2.2, loader-utils@~0.2.5: version "0.2.17" resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-0.2.17.tgz#f86e6374d43205a6e6c60e9196f17c0299bfb348" @@ -3514,6 +3572,12 @@ mime@^1.3.4, mime@^1.3.6: version "1.3.6" resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0" +min-document@^2.19.0: + version "2.19.0" + resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685" + dependencies: + dom-walk "^0.1.0" + minimatch@3.0.3: version "3.0.3" resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774" @@ -3736,6 +3800,10 @@ object-assign@4.1.1, object-assign@^4, object-assign@^4.0.1, object-assign@^4.1. version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" +object-keys@^1.0.11, object-keys@~1.0.0: + version "1.0.11" + resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d" + object.omit@^2.0.0: version "2.0.1" resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa" @@ -4266,6 +4334,10 @@ process@^0.11.0: version "0.11.10" resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182" +process@~0.5.1: + version "0.5.2" + resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf" + progress@^1.1.8: version "1.1.8" resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be" @@ -4340,7 +4412,7 @@ query-string@^4.1.0: object-assign "^4.1.0" strict-uri-encode "^1.0.0" -querystring-es3@^0.2.0: +querystring-es3@^0.2.0, querystring-es3@^0.2.1: version "0.2.1" resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73" @@ -4556,6 +4628,12 @@ reduce-function-call@^1.0.1: dependencies: balanced-match "^0.4.2" +reduce@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/reduce/-/reduce-1.0.1.tgz#14fa2e5ff1fc560703a020cbb5fbaab691565804" + dependencies: + object-keys "~1.0.0" + regenerate@^1.2.1: version "1.3.2" resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260" @@ -5190,7 +5268,7 @@ through2@^2.0.2, through2@^2.0.3: readable-stream "^2.1.5" xtend "~4.0.1" -through@^2.3.6, through@~2.3.6: +through@^2.3.6, through@~2.3.4, through@~2.3.6: version "2.3.8" resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"