186526/unpkg
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

New "browse" UI

Browse Source 
Also, separated out browse, ?meta, and ?module request handlers.

Fixes #82
This commit is contained in:
Michael Jackson
2019-07-24 17:55:13 -07:00
parent ea35b3c4b0
commit 34baab07ab
57 changed files with 2431 additions and 686 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
modules/actions/serveAutoIndexPage.js → modules/actions/serveBrowsePage.js
View File

@ -1,38 +1,36 @@
import { renderToString, renderToStaticMarkup } from 'react-dom/server';
import AutoIndexApp from '../client/autoIndex/App.js';
import MainTemplate from './utils/MainTemplate.js';
import getScripts from './utils/getScripts.js';
import { createElement, createHTML } from './utils/markupHelpers.js';
import BrowseApp from '../client/browse/App.js';
import MainTemplate from '../templates/MainTemplate.js';
import { getAvailableVersions } from '../utils/npm.js';
import getScripts from '../utils/getScripts.js';
import { createElement, createHTML } from '../utils/markup.js';
const doctype = '<!DOCTYPE html>';
const globalURLs =
process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'staging'
? {
'@emotion/core': '/@emotion/core@10.0.6/dist/core.umd.min.js',
react: '/react@16.7.0/umd/react.production.min.js',
'react-dom': '/react-dom@16.7.0/umd/react-dom.production.min.js'
react: '/react@16.8.6/umd/react.production.min.js',
'react-dom': '/react-dom@16.8.6/umd/react-dom.production.min.js'
}
: {
'@emotion/core': '/@emotion/core@10.0.6/dist/core.umd.min.js',
react: '/react@16.7.0/umd/react.development.js',
'react-dom': '/react-dom@16.7.0/umd/react-dom.development.js'
react: '/react@16.8.6/umd/react.development.js',
'react-dom': '/react-dom@16.8.6/umd/react-dom.development.js'
};
export default async function serveAutoIndexPage(req, res) {
export default async function serveBrowsePage(req, res) {
const availableVersions = await getAvailableVersions(req.packageName);
const data = {
packageName: req.packageName,
packageVersion: req.packageVersion,
availableVersions: availableVersions,
filename: req.filename,
entry: req.entry,
entries: req.entries
target: req.browseTarget
};
const content = createHTML(renderToString(createElement(AutoIndexApp, data)));
const elements = getScripts('autoIndex', 'iife', globalURLs);
const content = createHTML(renderToString(createElement(BrowseApp, data)));
const elements = getScripts('browse', 'iife', globalURLs);
const html =
doctype +
@ -49,7 +47,7 @@ export default async function serveAutoIndexPage(req, res) {
res
.set({
'Cache-Control': 'public, max-age=14400', // 4 hours
'Cache-Tag': 'auto-index'
'Cache-Tag': 'browse'
})
.send(html);
}

81
modules/actions/serveDirectoryBrowser.js Normal file
View File

@ -0,0 +1,81 @@
import path from 'path';
import gunzip from 'gunzip-maybe';
import tar from 'tar-stream';
import bufferStream from '../utils/bufferStream.js';
import getContentType from '../utils/getContentType.js';
import getIntegrity from '../utils/getIntegrity.js';
import { getPackage } from '../utils/npm.js';
import serveBrowsePage from './serveBrowsePage.js';
async function findMatchingEntries(stream, filename) {
// filename = /some/dir/name
return new Promise((accept, reject) => {
const entries = {};
stream
.pipe(gunzip())
.pipe(tar.extract())
.on('error', reject)
.on('entry', async (header, stream, next) => {
const entry = {
// Most packages have header names that look like `package/index.js`
// so we shorten that to just `/index.js` here. A few packages use a
// prefix other than `package/`. e.g. the firebase package uses the
// `firebase_npm/` prefix. So we just strip the first dir name.
path: header.name.replace(/^[^/]+/, ''),
type: header.type
};
// Dynamically create "directory" entries for all subdirectories
// in this entry's path. Some tarballs omit directory entries for
// some reason, so this is the "brute force" method.
let dir = path.dirname(entry.path);
while (dir !== '/') {
if (!entries[dir] && path.dirname(dir) === filename) {
entries[dir] = { path: dir, type: 'directory' };
}
dir = path.dirname(dir);
}
// Ignore non-files and files that aren't in this directory.
if (entry.type !== 'file' || path.dirname(entry.path) !== filename) {
stream.resume();
stream.on('end', next);
return;
}
const content = await bufferStream(stream);
entry.contentType = getContentType(entry.path);
entry.integrity = getIntegrity(content);
entry.size = content.length;
entries[entry.path] = entry;
next();
})
.on('finish', () => {
accept(entries);
});
});
}
export default async function serveDirectoryBrowser(req, res) {
const stream = await getPackage(req.packageName, req.packageVersion);
const filename = req.filename.slice(0, -1) || '/';
const entries = await findMatchingEntries(stream, filename);
if (Object.keys(entries).length === 0) {
return res.status(404).send(`Not found: ${req.packageSpec}${req.filename}`);
}
req.browseTarget = {
path: filename,
type: 'directory',
details: entries
};
serveBrowsePage(req, res);
}

97
modules/actions/serveDirectoryMetadata.js Normal file
View File

@ -0,0 +1,97 @@
import path from 'path';
import gunzip from 'gunzip-maybe';
import tar from 'tar-stream';
import bufferStream from '../utils/bufferStream.js';
import getContentType from '../utils/getContentType.js';
import getIntegrity from '../utils/getIntegrity.js';
import { getPackage } from '../utils/npm.js';
async function findMatchingEntries(stream, filename) {
// filename = /some/dir/name
return new Promise((accept, reject) => {
const entries = {};
entries[filename] = { path: filename, type: 'directory' };
stream
.pipe(gunzip())
.pipe(tar.extract())
.on('error', reject)
.on('entry', async (header, stream, next) => {
const entry = {
// Most packages have header names that look like `package/index.js`
// so we shorten that to just `/index.js` here. A few packages use a
// prefix other than `package/`. e.g. the firebase package uses the
// `firebase_npm/` prefix. So we just strip the first dir name.
path: header.name.replace(/^[^/]+/, ''),
type: header.type
};
// Dynamically create "directory" entries for all subdirectories
// in this entry's path. Some tarballs omit directory entries for
// some reason, so this is the "brute force" method.
let dir = path.dirname(entry.path);
while (dir !== '/') {
if (!entries[dir] && dir.startsWith(filename)) {
entries[dir] = { path: dir, type: 'directory' };
}
dir = path.dirname(dir);
}
// Ignore non-files and files that don't match the prefix.
if (entry.type !== 'file' || !entry.path.startsWith(filename)) {
stream.resume();
stream.on('end', next);
return;
}
const content = await bufferStream(stream);
entry.contentType = getContentType(entry.path);
entry.integrity = getIntegrity(content);
entry.lastModified = header.mtime.toUTCString();
entry.size = content.length;
entries[entry.path] = entry;
next();
})
.on('finish', () => {
accept(entries);
});
});
}
function getMatchingEntries(entry, entries) {
return Object.keys(entries)
.filter(key => entry.path !== key && path.dirname(key) === entry.path)
.map(key => entries[key]);
}
function getMetadata(entry, entries) {
const metadata = { path: entry.path, type: entry.type };
if (entry.type === 'file') {
metadata.contentType = entry.contentType;
metadata.integrity = entry.integrity;
metadata.lastModified = entry.lastModified;
metadata.size = entry.size;
} else if (entry.type === 'directory') {
metadata.files = getMatchingEntries(entry, entries).map(e =>
getMetadata(e, entries)
);
}
return metadata;
}
export default async function serveDirectoryMetadata(req, res) {
const stream = await getPackage(req.packageName, req.packageVersion);
const filename = req.filename.slice(0, -1) || '/';
const entries = await findMatchingEntries(stream, filename);
const metadata = getMetadata(entries[filename], entries);
res.send(metadata);
}

37
modules/actions/serveFile.js
View File

@ -1,23 +1,24 @@
import serveAutoIndexPage from './serveAutoIndexPage.js';
import serveMetadata from './serveMetadata.js';
import serveModule from './serveModule.js';
import serveStaticFile from './serveStaticFile.js';
import path from 'path';
import etag from 'etag';
import getContentTypeHeader from '../utils/getContentTypeHeader.js';
/**
* Send the file, JSON metadata, or HTML directory listing.
*/
export default function serveFile(req, res) {
if (req.query.meta != null) {
return serveMetadata(req, res);
const tags = ['file'];
const ext = path.extname(req.entry.path).substr(1);
if (ext) {
tags.push(`${ext}-file`);
}
if (req.entry.type === 'directory') {
return serveAutoIndexPage(req, res);
}
if (req.query.module != null) {
return serveModule(req, res);
}
serveStaticFile(req, res);
res
.set({
'Content-Type': getContentTypeHeader(req.entry.contentType),
'Content-Length': req.entry.size,
'Cache-Control': 'public, max-age=31536000', // 1 year
'Last-Modified': req.entry.lastModified,
ETag: etag(req.entry.content),
'Cache-Tag': tags.join(', ')
})
.send(req.entry.content);
}

84
modules/actions/serveFileBrowser.js Normal file
View File

@ -0,0 +1,84 @@
import gunzip from 'gunzip-maybe';
import tar from 'tar-stream';
import bufferStream from '../utils/bufferStream.js';
import createDataURI from '../utils/createDataURI.js';
import getContentType from '../utils/getContentType.js';
import getIntegrity from '../utils/getIntegrity.js';
import { getPackage } from '../utils/npm.js';
import getHighlights from '../utils/getHighlights.js';
import getLanguageName from '../utils/getLanguageName.js';
import serveBrowsePage from './serveBrowsePage.js';
async function findEntry(stream, filename) {
// filename = /some/file/name.js
return new Promise((accept, reject) => {
let foundEntry = null;
stream
.pipe(gunzip())
.pipe(tar.extract())
.on('error', reject)
.on('entry', async (header, stream, next) => {
const entry = {
// Most packages have header names that look like `package/index.js`
// so we shorten that to just `/index.js` here. A few packages use a
// prefix other than `package/`. e.g. the firebase package uses the
// `firebase_npm/` prefix. So we just strip the first dir name.
path: header.name.replace(/^[^/]+/, ''),
type: header.type
};
// Ignore non-files and files that don't match the name.
if (entry.type !== 'file' || entry.path !== filename) {
stream.resume();
stream.on('end', next);
return;
}
entry.content = await bufferStream(stream);
foundEntry = entry;
next();
})
.on('finish', () => {
accept(foundEntry);
});
});
}
export default async function serveFileBrowser(req, res) {
const stream = await getPackage(req.packageName, req.packageVersion);
const entry = await findEntry(stream, req.filename);
if (!entry) {
return res.status(404).send(`Not found: ${req.packageSpec}${req.filename}`);
}
const details = {
contentType: getContentType(entry.path),
integrity: getIntegrity(entry.content),
language: getLanguageName(entry.path),
size: entry.content.length
};
if (/^image\//.test(details.contentType)) {
details.uri = createDataURI(details.contentType, entry.content);
details.highlights = null;
} else {
details.uri = null;
details.highlights = getHighlights(
entry.content.toString('utf8'),
entry.path
);
}
req.browseTarget = {
path: req.filename,
type: 'file',
details
};
serveBrowsePage(req, res);
}

61
modules/actions/serveFileMetadata.js Normal file
View File

@ -0,0 +1,61 @@
import gunzip from 'gunzip-maybe';
import tar from 'tar-stream';
import bufferStream from '../utils/bufferStream.js';
import getContentType from '../utils/getContentType.js';
import getIntegrity from '../utils/getIntegrity.js';
import { getPackage } from '../utils/npm.js';
async function findEntry(stream, filename) {
// filename = /some/file/name.js
return new Promise((accept, reject) => {
let foundEntry = null;
stream
.pipe(gunzip())
.pipe(tar.extract())
.on('error', reject)
.on('entry', async (header, stream, next) => {
const entry = {
// Most packages have header names that look like `package/index.js`
// so we shorten that to just `/index.js` here. A few packages use a
// prefix other than `package/`. e.g. the firebase package uses the
// `firebase_npm/` prefix. So we just strip the first dir name.
path: header.name.replace(/^[^/]+/, ''),
type: header.type
};
// Ignore non-files and files that don't match the name.
if (entry.type !== 'file' || entry.path !== filename) {
stream.resume();
stream.on('end', next);
return;
}
const content = await bufferStream(stream);
entry.contentType = getContentType(entry.path);
entry.integrity = getIntegrity(content);
entry.lastModified = header.mtime.toUTCString();
entry.size = content.length;
foundEntry = entry;
next();
})
.on('finish', () => {
accept(foundEntry);
});
});
}
export default async function serveFileMetadata(req, res) {
const stream = await getPackage(req.packageName, req.packageVersion);
const entry = await findEntry(stream, req.filename);
if (!entry) {
// TODO: 404
}
res.send(entry);
}

15
modules/actions/serveMainPage.js
View File

@ -1,23 +1,22 @@
import { renderToString, renderToStaticMarkup } from 'react-dom/server';
import MainApp from '../client/main/App.js';
import MainTemplate from './utils/MainTemplate.js';
import getScripts from './utils/getScripts.js';
import { createElement, createHTML } from './utils/markupHelpers.js';
import MainTemplate from '../templates/MainTemplate.js';
import getScripts from '../utils/getScripts.js';
import { createElement, createHTML } from '../utils/markup.js';
const doctype = '<!DOCTYPE html>';
const globalURLs =
process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'staging'
? {
'@emotion/core': '/@emotion/core@10.0.6/dist/core.umd.min.js',
react: '/react@16.7.0/umd/react.production.min.js',
'react-dom': '/react-dom@16.7.0/umd/react-dom.production.min.js'
react: '/react@16.8.6/umd/react.production.min.js',
'react-dom': '/react-dom@16.8.6/umd/react-dom.production.min.js'
}
: {
'@emotion/core': '/@emotion/core@10.0.6/dist/core.umd.min.js',
react: '/react@16.7.0/umd/react.development.js',
'react-dom': '/react-dom@16.7.0/umd/react-dom.development.js'
react: '/react@16.8.6/umd/react.development.js',
'react-dom': '/react-dom@16.8.6/umd/react-dom.development.js'
};
export default function serveMainPage(req, res) {

42
modules/actions/serveMetadata.js
View File

@ -1,42 +0,0 @@
import path from 'path';
function getMatchingEntries(entry, entries) {
const dirname = entry.name || '.';
return Object.keys(entries)
.filter(name => entry.name !== name && path.dirname(name) === dirname)
.map(name => entries[name]);
}
const leadingSlashes = /^\/*/;
function getMetadata(entry, entries) {
const metadata = {
path: entry.name.replace(leadingSlashes, '/'),
type: entry.type
};
if (entry.type === 'file') {
metadata.contentType = entry.contentType;
metadata.integrity = entry.integrity;
metadata.lastModified = entry.lastModified;
metadata.size = entry.size;
} else if (entry.type === 'directory') {
metadata.files = getMatchingEntries(entry, entries).map(e =>
getMetadata(e, entries)
);
}
return metadata;
}
export default function serveMetadata(req, res) {
const metadata = getMetadata(req.entry, req.entries);
res
.set({
'Cache-Control': 'public, max-age=31536000', // 1 year
'Cache-Tag': 'meta'
})
.send(metadata);
}

2
modules/actions/serveModule.js
View File

@ -1,7 +1,7 @@
import serveHTMLModule from './serveHTMLModule.js';
import serveJavaScriptModule from './serveJavaScriptModule.js';
export default function serveModule(req, res) {
export default async function serveModule(req, res) {
if (req.entry.contentType === 'application/javascript') {
return serveJavaScriptModule(req, res);
}

24
modules/actions/serveStaticFile.js
View File

@ -1,24 +0,0 @@
import path from 'path';
import etag from 'etag';
import getContentTypeHeader from '../utils/getContentTypeHeader.js';
export default function serveStaticFile(req, res) {
const tags = ['file'];
const ext = path.extname(req.entry.name).substr(1);
if (ext) {
tags.push(`${ext}-file`);
}
res
.set({
'Content-Length': req.entry.size,
'Content-Type': getContentTypeHeader(req.entry.contentType),
'Cache-Control': 'public, max-age=31536000', // 1 year
'Last-Modified': req.entry.lastModified,
ETag: etag(req.entry.content),
'Cache-Tag': tags.join(', ')
})
.send(req.entry.content);
}

2
modules/actions/serveStats.js
View File

@ -1,6 +1,6 @@
import { subDays, startOfDay } from 'date-fns';
import getStats from './utils/getStats.js';
import getStats from '../utils/getStats.js';
export default function serveStats(req, res) {
let since, until;

74
modules/actions/utils/MainTemplate.js
View File

@ -1,74 +0,0 @@
import PropTypes from 'prop-types';
import {
createElement as e,
createHTML as h,
createScript as x
} from './markupHelpers.js';
const promiseShim =
'window.Promise || document.write(\'\\x3Cscript src="/es6-promise@4.2.5/dist/es6-promise.min.js">\\x3C/script>\\x3Cscript>ES6Promise.polyfill()\\x3C/script>\')';
const fetchShim =
'window.fetch || document.write(\'\\x3Cscript src="/whatwg-fetch@3.0.0/dist/fetch.umd.js">\\x3C/script>\')';
export default function MainTemplate({
title = 'UNPKG',
description = 'The CDN for everything on npm',
favicon = '/favicon.ico',
data,
content = h(''),
elements = []
}) {
return e(
'html',
{ lang: 'en' },
e(
'head',
null,
// Global site tag (gtag.js) - Google Analytics
e('script', {
async: true,
src: 'https://www.googletagmanager.com/gtag/js?id=UA-140352188-1'
}),
x(`window.dataLayer = window.dataLayer || [];
function gtag(){dataLayer.push(arguments);}
gtag('js', new Date());
gtag('config', 'UA-140352188-1');`),
e('meta', { charSet: 'utf-8' }),
e('meta', { httpEquiv: 'X-UA-Compatible', content: 'IE=edge,chrome=1' }),
description && e('meta', { name: 'description', content: description }),
e('meta', {
name: 'viewport',
content: 'width=device-width,initial-scale=1,maximum-scale=1'
}),
e('meta', { name: 'timestamp', content: new Date().toISOString() }),
favicon && e('link', { rel: 'shortcut icon', href: favicon }),
e('title', null, title),
x(promiseShim),
x(fetchShim),
data && x(`window.__DATA__ = ${JSON.stringify(data)}`)
),
e(
'body',
null,
e('div', { id: 'root', dangerouslySetInnerHTML: content }),
...elements
)
);
}
if (process.env.NODE_ENV !== 'production') {
const htmlType = PropTypes.shape({
__html: PropTypes.string
});
MainTemplate.propTypes = {
title: PropTypes.string,
description: PropTypes.string,
favicon: PropTypes.string,
data: PropTypes.any,
content: htmlType,
elements: PropTypes.arrayOf(PropTypes.node)
};
}

74
modules/actions/utils/cloudflare.js
View File

@ -1,74 +0,0 @@
import fetch from 'isomorphic-fetch';
const cloudflareURL = 'https://api.cloudflare.com/client/v4';
const cloudflareEmail = process.env.CLOUDFLARE_EMAIL;
const cloudflareKey = process.env.CLOUDFLARE_KEY;
if (process.env.NODE_ENV !== 'production') {
if (!cloudflareEmail) {
throw new Error('Missing the $CLOUDFLARE_EMAIL environment variable');
}
if (!cloudflareKey) {
throw new Error('Missing the $CLOUDFLARE_KEY environment variable');
}
}
function get(path, headers) {
return fetch(`${cloudflareURL}${path}`, {
headers: Object.assign({}, headers, {
'X-Auth-Email': cloudflareEmail,
'X-Auth-Key': cloudflareKey
})
});
}
function getJSON(path, headers) {
return get(path, headers)
.then(res => {
return res.json();
})
.then(data => {
if (!data.success) {
console.error(`cloudflare.getJSON failed at ${path}`);
console.error(data);
throw new Error('Failed to getJSON from Cloudflare');
}
return data.result;
});
}
export function getZones(domains) {
return Promise.all(
(Array.isArray(domains) ? domains : [domains]).map(domain =>
getJSON(`/zones?name=${domain}`)
)
).then(results => results.reduce((memo, zones) => memo.concat(zones)));
}
function reduceResults(target, values) {
Object.keys(values).forEach(key => {
const value = values[key];
if (typeof value === 'object' && value) {
target[key] = reduceResults(target[key] || {}, value);
} else if (typeof value === 'number') {
target[key] = (target[key] || 0) + values[key];
}
});
return target;
}
export function getZoneAnalyticsDashboard(zones, since, until) {
return Promise.all(
(Array.isArray(zones) ? zones : [zones]).map(zone => {
return getJSON(
`/zones/${
zone.id
}/analytics/dashboard?since=${since.toISOString()}&until=${until.toISOString()}`
);
})
).then(results => results.reduce(reduceResults));
}

42
modules/actions/utils/getScripts.js
View File

@ -1,42 +0,0 @@
// Virtual module id; see rollup.config.js
// eslint-disable-next-line import/no-unresolved
import entryManifest from 'entry-manifest';
import { createElement, createScript } from './markupHelpers.js';
function getEntryPoint(name, format) {
let entryPoints;
entryManifest.forEach(manifest => {
if (name in manifest) {
entryPoints = manifest[name];
}
});
if (entryPoints) {
return entryPoints.find(e => e.format === format);
}
return null;
}
function getGlobalScripts(entryPoint, globalURLs) {
return entryPoint.globalImports.map(id => {
if (process.env.NODE_ENV !== 'production') {
if (!globalURLs[id]) {
throw new Error('Missing global URL for id "%s"', id);
}
}
return createElement('script', { src: globalURLs[id] });
});
}
export default function getScripts(entryName, format, globalURLs) {
const entryPoint = getEntryPoint(entryName, format);
if (!entryPoint) return [];
return getGlobalScripts(entryPoint, globalURLs).concat(
createScript(entryPoint.code)
);
}

38
modules/actions/utils/getStats.js
View File

@ -1,38 +0,0 @@
import { getZones, getZoneAnalyticsDashboard } from './cloudflare.js';
function extractPublicInfo(data) {
return {
since: data.since,
until: data.until,
requests: {
all: data.requests.all,
cached: data.requests.cached,
country: data.requests.country,
status: data.requests.http_status
},
bandwidth: {
all: data.bandwidth.all,
cached: data.bandwidth.cached,
country: data.bandwidth.country
},
threats: {
all: data.threats.all,
country: data.threats.country
},
uniques: {
all: data.uniques.all
}
};
}
const DomainNames = ['unpkg.com', 'npmcdn.com'];
export default async function getStats(since, until) {
const zones = await getZones(DomainNames);
const dashboard = await getZoneAnalyticsDashboard(zones, since, until);
return {
timeseries: dashboard.timeseries.map(extractPublicInfo),
totals: extractPublicInfo(dashboard.totals)
};
}

13
modules/actions/utils/markupHelpers.js
View File

@ -1,13 +0,0 @@
import { createElement } from 'react';
export { createElement };
export function createHTML(code) {
return { __html: code };
}
export function createScript(script) {
return createElement('script', {
dangerouslySetInnerHTML: createHTML(script)
});
}