186526
/
unpkg
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Add _search endpoint

This commit is contained in:
MICHAEL JACKSON 2017-09-13 07:56:46 -07:00
parent d72d4ee6f3
commit de1195d8e9
7 changed files with 359 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
package.json
View File

@ -9,6 +9,7 @@
"test": "node scripts/test.js --env=jsdom"
},
"dependencies": {
"algoliasearch": "^3.24.3",
"babel-plugin-unpkg-rewrite": "^3.1.0",
"cors": "^2.8.1",
"countries-list": "^1.3.2",

4
server/createApp.js
View File

@ -10,6 +10,8 @@ const fetchFile = require('./middleware/fetchFile')
const serveFile = require('./middleware/serveFile')
const serveStats = require('./middleware/serveStats')
const createSearchServer = require('./createSearchServer')
morgan.token('fwd', function (req) {
return req.get('x-forwarded-for').replace(/\s/g, '')
})
@ -48,6 +50,8 @@ function createApp() {
app.use('/_stats', serveStats())
app.use('/_search', createSearchServer())
app.use('/',
packageURL,
checkBlacklist(PackageBlacklist),

87
server/createSearchServer.js Normal file
View File

@ -0,0 +1,87 @@
const express = require('express')
const getAssetPaths = require('./npm/getAssetPaths')
const npmSearchIndex = require('./npm/searchIndex')
function enhanceHit(hit) {
return new Promise(function (resolve, reject) {
const assetPaths = getAssetPaths(hit.name, hit.version)
if (assetPaths) {
// TODO: Double check the package metadata to ensure the files
// haven't moved from the paths in the index?
hit.assets = assetPaths.map(function (path) {
return `https://unpkg.com/${hit.name}@${hit.version}${path}`
})
resolve(hit)
} else {
resolve(hit)
}
})
}
function byRelevanceDescending(a, b) {
// Hits that have assets are more relevant.
return a.assets ? (b.assets ? 0 : -1) : (b.assets ? 1 : 0)
}
function createSearchServer() {
const app = express()
app.get('/', function (req, res) {
const { query, page = 0 } = req.query
const hitsPerPage = 20
if (!query)
return res.status(403).send({ error: 'Missing ?query parameter' })
const params = {
typoTolerance: 'min',
attributesToRetrieve: [
'name',
'version',
'description',
'owner'
],
attributesToHighlight: null,
restrictSearchableAttributes: [
'name',
'description'
],
hitsPerPage,
page
}
npmSearchIndex.search(query, params, function (error, value) {
if (error) {
console.error(error)
res.status(500).send({ error: 'There was an error executing the search' })
} else {
Promise.all(
value.hits.map(enhanceHit)
).then(function (hits) {
hits.sort(byRelevanceDescending)
const totalHits = value.nbHits
const totalPages = value.nbPages
res.send({
query,
page,
hitsPerPage,
totalHits,
totalPages,
hits
})
}, function (error) {
console.error(error)
res.status(500).send({ error: 'There was an error executing the search' })
})
}
})
})
return app
}
module.exports = createSearchServer

144
server/npm/assetPathsIndex.js Normal file
View File

@ -0,0 +1,144 @@
/**
* A hand-built index of paths in npm packages that use globals. The index is
* not meant to contain *all* the global paths that a given package publishes,
* but rather those that are probably most useful for someone who wants to put
* a library on a page quickly in development.
*
* Each entry in the index is keyed by package name and contains a list of
* [ versionRange, ...paths ] for that package. The list is traversed in order,
* so version ranges that come sooner will match before those that come later.
* The range `null` is a catch-all.
*/
module.exports = {
'angular': [
[ '>=1.2.27', '/angular.min.js' ],
[ null, '/lib/angular.min.js' ]
],
'animate.css': [
[ null, '/animate.min.css' ]
],
'babel-standalone': [
[ null, '/babel.min.js' ]
],
'backbone': [
[ null, '/backbone-min.js' ]
],
'bootstrap': [
[ null, '/dist/css/bootstrap.min.css', '/dist/js/bootstrap.min.js' ]
],
'bulma': [
[ null, '/css/bulma.css' ]
],
'core.js': [
[ null, '/dist/core.min.js' ]
],
'd3': [
[ null, '/build/d3.min.js' ]
],
'ember-source': [
[ null, '/dist/ember.min.js' ]
],
'foundation-sites': [
[ null, '/dist/css/foundation.min.css', '/dist/js/foundation.min.js' ]
],
'gsap': [
[ null, '/TweenMax.js' ]
],
'handlebars': [
[ null, '/dist/handlebars.min.js' ]
],
'jquery': [
[ null, '/dist/jquery.min.js' ]
],
'fastclick': [
[ null, '/lib/fastclick.js' ]
],
'lodash': [
[ '<3', '/dist/lodash.min.js' ],
[ null, '/lodash.min.js' ]
],
'masonry-layout': [
[ null, '/dist/masonry.pkgd.min.js' ]
],
'materialize-css': [
[ null, '/dist/css/materialize.min.css' ]
],
'react': [
[ '>=16.0.0-alpha.7', '/umd/react.production.min.js' ],
[ null, '/dist/react.min.js' ]
],
'react-dom': [
[ '>=16.0.0-alpha.7', '/umd/react-dom.production.min.js' ],
[ null, '/dist/react-dom.min.js' ]
],
'react-router': [
[ '>=4.0.0', '/umd/react-router.min.js' ],
[ null, '/umd/ReactRouter.min.js' ]
],
'redux': [
[ null, '/dist/redux.min.js' ]
],
'redux-saga': [
[ null, '/dist/redux-saga.min.js' ]
],
'redux-thunk': [
[ null, '/dist/redux-thunk.min.js' ]
],
'snapsvg': [
[ null, '/snap.svg-min.js' ]
],
'systemjs': [
[ null, '/dist/system.js' ]
],
'three': [
[ '<=0.77.0', '/three.min.js' ],
[ null, '/build/three.min.js' ]
],
'underscore': [
[ null, '/underscore-min.js' ]
],
'vue': [
[ null, '/dist/vue.min.js' ]
],
'zepto': [
[ null, '/dist/zepto.min.js' ]
],
'zingchart': [
[ null, '/client/zingchart.min.js' ]
],
'zone.js': [
[ null, '/dist/zone.js' ]
]
}

20
server/npm/getAssetPaths.js Normal file
View File

@ -0,0 +1,20 @@
const assetPathsIndex = require('./assetPathsIndex')
function getAssetPaths(packageName, version) {
const entries = assetPathsIndex[packageName]
if (entries) {
const matchingEntry = entries.find(function (entry) {
const range = entry[0]
if (range == null || semver.satisfies(version, range))
return entry
})
return matchingEntry.slice(1)
}
return null
}
module.exports = getAssetPaths

22
server/npm/searchIndex.js Normal file
View File

@ -0,0 +1,22 @@
const algolia = require('algoliasearch')
const invariant = require('invariant')
const AlgoliaNpmSearchAppId = process.env.ALGOLIA_NPM_SEARCH_APP_ID
const AlgoliaNpmSearchApiKey = process.env.ALGOLIA_NPM_SEARCH_API_KEY
invariant(
AlgoliaNpmSearchAppId,
'Missing $ALGOLIA_NPM_SEARCH_APP_ID environment variable'
)
invariant(
AlgoliaNpmSearchApiKey,
'Missing $ALGOLIA_NPM_SEARCH_API_KEY environment variable'
)
const index = algolia(
AlgoliaNpmSearchAppId,
AlgoliaNpmSearchApiKey
).initIndex('npm-search')
module.exports = index

84
yarn.lock
View File

@ -41,6 +41,10 @@ acorn@^5.1.1:
version "5.1.1"
resolved "https://registry.yarnpkg.com/acorn/-/acorn-5.1.1.tgz#53fe161111f912ab999ee887a90a0bc52822fd75"
agentkeepalive@^2.2.0:
version "2.2.0"
resolved "https://registry.yarnpkg.com/agentkeepalive/-/agentkeepalive-2.2.0.tgz#c5d1bd4b129008f1163f236f86e5faea2026e2ef"
ajv-keywords@^1.0.0:
version "1.5.1"
resolved "https://registry.yarnpkg.com/ajv-keywords/-/ajv-keywords-1.5.1.tgz#314dd0a4b3368fad3dfcdc54ede6171b886daf3c"
@ -52,6 +56,26 @@ ajv@^4.7.0, ajv@^4.9.1:
co "^4.6.0"
json-stable-stringify "^1.0.1"
algoliasearch@^3.24.3:
version "3.24.3"
resolved "https://registry.yarnpkg.com/algoliasearch/-/algoliasearch-3.24.3.tgz#0b5da4242baa2e4bf9359bd20c47266ec0e501f4"
dependencies:
agentkeepalive "^2.2.0"
debug "^2.6.8"
envify "^4.0.0"
es6-promise "^4.1.0"
events "^1.1.0"
foreach "^2.0.5"
global "^4.3.2"
inherits "^2.0.1"
isarray "^2.0.1"
load-script "^1.0.0"
object-keys "^1.0.11"
querystring-es3 "^0.2.1"
reduce "^1.0.1"
semver "^5.1.0"
tunnel-agent "^0.6.0"
align-text@^0.1.1, align-text@^0.1.3:
version "0.1.4"
resolved "https://registry.yarnpkg.com/align-text/-/align-text-0.1.4.tgz#0cd90a561093f35d0a99256c22b7069433fad117"
@ -1635,6 +1659,10 @@ dom-serializer@0:
domelementtype "~1.1.1"
entities "~1.1.1"
dom-walk@^0.1.0:
version "0.1.1"
resolved "https://registry.yarnpkg.com/dom-walk/-/dom-walk-0.1.1.tgz#672226dc74c8f799ad35307df936aba11acd6018"
domain-browser@^1.1.1:
version "1.1.7"
resolved "https://registry.yarnpkg.com/domain-browser/-/domain-browser-1.1.7.tgz#867aa4b093faa05f1de08c06f4d7b21fdf8698bc"
@ -1737,6 +1765,13 @@ entities@~1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.1.tgz#6e5c2d0a5621b5dadaecef80b90edfb5cd7772f0"
envify@^4.0.0:
version "4.1.0"
resolved "https://registry.yarnpkg.com/envify/-/envify-4.1.0.tgz#f39ad3db9d6801b4e6b478b61028d3f0b6819f7e"
dependencies:
esprima "^4.0.0"
through "~2.3.4"
errno@^0.1.3, errno@^0.1.4:
version "0.1.4"
resolved "https://registry.yarnpkg.com/errno/-/errno-0.1.4.tgz#b896e23a9e5e8ba33871fc996abd3635fc9a1c7d"
@ -1775,6 +1810,10 @@ es6-map@^0.1.3:
es6-symbol "~3.1.1"
event-emitter "~0.3.5"
es6-promise@^4.1.0:
version "4.1.1"
resolved "https://registry.yarnpkg.com/es6-promise/-/es6-promise-4.1.1.tgz#8811e90915d9a0dba36274f0b242dbda78f9c92a"
es6-set@~0.1.5:
version "0.1.5"
resolved "https://registry.yarnpkg.com/es6-set/-/es6-set-0.1.5.tgz#d2b3ec5d4d800ced818db538d28974db0a73ccb1"
@ -1998,7 +2037,7 @@ eventemitter3@1.x.x:
version "1.2.0"
resolved "https://registry.yarnpkg.com/eventemitter3/-/eventemitter3-1.2.0.tgz#1c86991d816ad1e504750e73874224ecf3bec508"
events@^1.0.0:
events@^1.0.0, events@^1.1.0:
version "1.1.1"
resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924"
@ -2224,6 +2263,10 @@ for-own@^0.1.4:
dependencies:
for-in "^1.0.1"
foreach@^2.0.5:
version "2.0.5"
resolved "https://registry.yarnpkg.com/foreach/-/foreach-2.0.5.tgz#0bee005018aeb260d0a3af3ae658dd0136ec1b99"
forever-agent@~0.6.1:
version "0.6.1"
resolved "https://registry.yarnpkg.com/forever-agent/-/forever-agent-0.6.1.tgz#fbc71f0c41adeb37f96c577ad1ed42d8fdacca91"
@ -2355,6 +2398,13 @@ glob@^7.0.0, glob@^7.0.3, glob@^7.0.5:
once "^1.3.0"
path-is-absolute "^1.0.0"
global@^4.3.2:
version "4.3.2"
resolved "https://registry.yarnpkg.com/global/-/global-4.3.2.tgz#e76989268a6c74c38908b1305b10fc0e394e9d0f"
dependencies:
min-document "^2.19.0"
process "~0.5.1"
globals@^9.14.0, globals@^9.18.0:
version "9.18.0"
resolved "https://registry.yarnpkg.com/globals/-/globals-9.18.0.tgz#aa3896b3e69b487f17e31ed2143d69a8e30c2d8a"
@ -2844,6 +2894,10 @@ isarray@1.0.0, isarray@^1.0.0, isarray@~1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
isarray@^2.0.1:
version "2.0.2"
resolved "https://registry.yarnpkg.com/isarray/-/isarray-2.0.2.tgz#5aa99638daf2248b10b9598b763a045688ece3ee"
isexe@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/isexe/-/isexe-2.0.0.tgz#e8fbf374dc556ff8947a10dcb0572d633f2cfa10"
@ -3266,6 +3320,10 @@ load-json-file@^1.0.0:
pinkie-promise "^2.0.0"
strip-bom "^2.0.0"
load-script@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/load-script/-/load-script-1.0.0.tgz#0491939e0bee5643ee494a7e3da3d2bac70c6ca4"
loader-utils@0.2.x, loader-utils@^0.2.11, loader-utils@^0.2.16, loader-utils@^0.2.3, loader-utils@^0.2.7, loader-utils@~0.2.2, loader-utils@~0.2.5:
version "0.2.17"
resolved "https://registry.yarnpkg.com/loader-utils/-/loader-utils-0.2.17.tgz#f86e6374d43205a6e6c60e9196f17c0299bfb348"
@ -3514,6 +3572,12 @@ mime@^1.3.4, mime@^1.3.6:
version "1.3.6"
resolved "https://registry.yarnpkg.com/mime/-/mime-1.3.6.tgz#591d84d3653a6b0b4a3b9df8de5aa8108e72e5e0"
min-document@^2.19.0:
version "2.19.0"
resolved "https://registry.yarnpkg.com/min-document/-/min-document-2.19.0.tgz#7bd282e3f5842ed295bb748cdd9f1ffa2c824685"
dependencies:
dom-walk "^0.1.0"
minimatch@3.0.3:
version "3.0.3"
resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.3.tgz#2a4e4090b96b2db06a9d7df01055a62a77c9b774"
@ -3736,6 +3800,10 @@ object-assign@4.1.1, object-assign@^4, object-assign@^4.0.1, object-assign@^4.1.
version "4.1.1"
resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863"
object-keys@^1.0.11, object-keys@~1.0.0:
version "1.0.11"
resolved "https://registry.yarnpkg.com/object-keys/-/object-keys-1.0.11.tgz#c54601778ad560f1142ce0e01bcca8b56d13426d"
object.omit@^2.0.0:
version "2.0.1"
resolved "https://registry.yarnpkg.com/object.omit/-/object.omit-2.0.1.tgz#1a9c744829f39dbb858c76ca3579ae2a54ebd1fa"
@ -4266,6 +4334,10 @@ process@^0.11.0:
version "0.11.10"
resolved "https://registry.yarnpkg.com/process/-/process-0.11.10.tgz#7332300e840161bda3e69a1d1d91a7d4bc16f182"
process@~0.5.1:
version "0.5.2"
resolved "https://registry.yarnpkg.com/process/-/process-0.5.2.tgz#1638d8a8e34c2f440a91db95ab9aeb677fc185cf"
progress@^1.1.8:
version "1.1.8"
resolved "https://registry.yarnpkg.com/progress/-/progress-1.1.8.tgz#e260c78f6161cdd9b0e56cc3e0a85de17c7a57be"
@ -4340,7 +4412,7 @@ query-string@^4.1.0:
object-assign "^4.1.0"
strict-uri-encode "^1.0.0"
querystring-es3@^0.2.0:
querystring-es3@^0.2.0, querystring-es3@^0.2.1:
version "0.2.1"
resolved "https://registry.yarnpkg.com/querystring-es3/-/querystring-es3-0.2.1.tgz#9ec61f79049875707d69414596fd907a4d711e73"
@ -4556,6 +4628,12 @@ reduce-function-call@^1.0.1:
dependencies:
balanced-match "^0.4.2"
reduce@^1.0.1:
version "1.0.1"
resolved "https://registry.yarnpkg.com/reduce/-/reduce-1.0.1.tgz#14fa2e5ff1fc560703a020cbb5fbaab691565804"
dependencies:
object-keys "~1.0.0"
regenerate@^1.2.1:
version "1.3.2"
resolved "https://registry.yarnpkg.com/regenerate/-/regenerate-1.3.2.tgz#d1941c67bad437e1be76433add5b385f95b19260"
@ -5190,7 +5268,7 @@ through2@^2.0.2, through2@^2.0.3:
readable-stream "^2.1.5"
xtend "~4.0.1"
through@^2.3.6, through@~2.3.6:
through@^2.3.6, through@~2.3.4, through@~2.3.6:
version "2.3.8"
resolved "https://registry.yarnpkg.com/through/-/through-2.3.8.tgz#0dd4c9ffaabc357960b1b724115d7e0e86a2e1f5"