From 0c8c70022c12c9726278416e6b94929b57951fe6 Mon Sep 17 00:00:00 2001 From: Alexei Mochalov Date: Fri, 11 Sep 2020 10:30:40 +0500 Subject: [PATCH] Embed customization & related fixes (#1795) * utils/pipeThru * Data: more memoization * Preview: refactor, partial glacier support, display helper * partial glacier support * Preview: fix json loader * Preview/loaders: more error handling * embed scoping * embed: hideRootLink * embed: padding override example * handle consecutive slashes in s3 paths * search: handle more syntax errors --- .../app/components/BreadCrumbs/BreadCrumbs.js | 6 +- catalog/app/components/Preview/Display.js | 129 +++++++++ catalog/app/components/Preview/Preview.js | 36 --- catalog/app/components/Preview/index.js | 5 +- catalog/app/components/Preview/load.js | 39 +++ catalog/app/components/Preview/loaders/Csv.js | 23 +- .../app/components/Preview/loaders/Excel.js | 12 +- catalog/app/components/Preview/loaders/Fcs.js | 17 +- .../app/components/Preview/loaders/Html.js | 25 +- .../app/components/Preview/loaders/Image.js | 6 +- .../app/components/Preview/loaders/Json.js | 96 +++---- .../components/Preview/loaders/Markdown.js | 122 +++++---- .../components/Preview/loaders/Notebook.js | 12 +- .../app/components/Preview/loaders/Parquet.js | 17 +- catalog/app/components/Preview/loaders/Pdf.js | 21 +- .../app/components/Preview/loaders/Text.js | 36 +-- catalog/app/components/Preview/loaders/Vcf.js | 18 +- .../components/Preview/loaders/fallback.js | 9 + .../app/components/Preview/loaders/index.js | 14 - .../app/components/Preview/loaders/utils.js | 237 +++++++++-------- catalog/app/components/Preview/render.js | 4 + catalog/app/components/Preview/types.js | 9 +- .../components/SearchResults/SearchResults.js | 174 ++++++------ catalog/app/constants/embed-routes.js | 2 +- catalog/app/constants/routes.js | 2 +- catalog/app/containers/Bucket/File.js | 119 +++------ catalog/app/containers/Bucket/FilePreview.js | 107 -------- catalog/app/containers/Bucket/FileView.js | 134 ++++++++++ catalog/app/containers/Bucket/Listing.js | 4 +- catalog/app/containers/Bucket/Overview.js | 98 ++----- catalog/app/containers/Bucket/PackageTree.js | 250 +++++++++--------- catalog/app/containers/Bucket/Summary.js | 86 +----- .../app/containers/Bucket/renderPreview.js | 33 +++ catalog/app/containers/Bucket/requests.js | 4 +- catalog/app/embed/AppBar.js | 50 +++- catalog/app/embed/Dir.js | 62 ++--- catalog/app/embed/Embed.js | 5 + catalog/app/embed/File.js | 142 +++------- catalog/app/embed/customization-example.json | 17 +- catalog/app/embed/debug-harness.js | 8 +- catalog/app/embed/getCrumbs.js | 25 ++ catalog/app/utils/Data.js | 13 +- catalog/app/utils/pipeThru.js | 3 + catalog/app/utils/s3paths.js | 11 +- catalog/app/utils/search.js | 6 +- 45 files changed, 1178 insertions(+), 1070 deletions(-) create mode 100644 catalog/app/components/Preview/Display.js delete mode 100644 catalog/app/components/Preview/Preview.js create mode 100644 catalog/app/components/Preview/load.js create mode 100644 catalog/app/components/Preview/loaders/fallback.js delete mode 100644 catalog/app/components/Preview/loaders/index.js create mode 100644 catalog/app/components/Preview/render.js delete mode 100644 catalog/app/containers/Bucket/FilePreview.js create mode 100644 catalog/app/containers/Bucket/FileView.js create mode 100644 catalog/app/containers/Bucket/renderPreview.js create mode 100644 catalog/app/embed/getCrumbs.js create mode 100644 catalog/app/utils/pipeThru.js diff --git a/catalog/app/components/BreadCrumbs/BreadCrumbs.js b/catalog/app/components/BreadCrumbs/BreadCrumbs.js index 6c72fdfc2e3..0498cf08733 100644 --- a/catalog/app/components/BreadCrumbs/BreadCrumbs.js +++ b/catalog/app/components/BreadCrumbs/BreadCrumbs.js @@ -3,12 +3,15 @@ import * as React from 'react' import Link from 'utils/StyledLink' import tagged from 'utils/tagged' +const EMPTY = {''} + export const Crumb = tagged([ 'Segment', // { label, to } 'Sep', // value ]) -export const Segment = ({ label, to }) => (to ? {label} : label) +export const Segment = ({ label, to }) => + to ? {label || EMPTY} : label || EMPTY export const render = (items) => items.map( @@ -25,6 +28,7 @@ export function copyWithoutSpaces(e) { document .getSelection() .toString() + .replace('', '') .replace(/\s*\/\s*/g, '/'), ) e.preventDefault() diff --git a/catalog/app/components/Preview/Display.js b/catalog/app/components/Preview/Display.js new file mode 100644 index 00000000000..431f80b1809 --- /dev/null +++ b/catalog/app/components/Preview/Display.js @@ -0,0 +1,129 @@ +import * as R from 'ramda' +import * as React from 'react' +import * as M from '@material-ui/core' + +import * as AWS from 'utils/AWS' +import AsyncResult from 'utils/AsyncResult' +import * as Config from 'utils/Config' +import StyledLink from 'utils/StyledLink' +import pipeThru from 'utils/pipeThru' + +import render from './render' +import { PreviewError } from './types' + +const defaultProgress = () => + +const defaultMessage = ({ heading, body, action }) => ( + <> + {!!heading && ( + + {heading} + + )} + {!!body && ( + + {body} + + )} + {!!action && action} + +) + +const defaultAction = ({ label, ...rest }) => ( + + {label} + +) + +export default function PreviewDisplay({ + data, + noDownload, + renderContents = R.identity, + renderProgress = defaultProgress, + renderMessage = defaultMessage, + renderAction = defaultAction, +}) { + const cfg = Config.use() + const noDl = noDownload != null ? noDownload : cfg.noDownload + return pipeThru(data)( + AsyncResult.case({ + _: renderProgress, + Ok: R.pipe(render, renderContents), + Err: PreviewError.case({ + Deleted: () => + renderMessage({ + heading: 'Delete Marker', + body: ( + <> + Selected version of the object is a{' '} + + delete marker + + + ), + }), + Archived: () => + renderMessage({ + heading: 'Object Archived', + body: 'Preview not available', + }), + InvalidVersion: () => + renderMessage({ + heading: 'Invalid Version', + body: 'Invalid version id specified', + }), + Forbidden: () => + renderMessage({ + heading: 'Access Denied', + body: 'Preview not available', + }), + Gated: ({ load }) => + renderMessage({ + heading: 'Object is Too Large', + body: 'Large files are not previewed by default', + action: !!load && renderAction({ label: 'Load preview', onClick: load }), + }), + TooLarge: ({ handle }) => + renderMessage({ + heading: 'Object is Too Large', + body: 'Object is too large to preview', + action: + !noDl && + AWS.Signer.withDownloadUrl(handle, (href) => + renderAction({ label: 'Download and view in Browser', href }), + ), + }), + Unsupported: ({ handle }) => + renderMessage({ + heading: 'Preview Not Available', + action: + !noDl && + AWS.Signer.withDownloadUrl(handle, (href) => + renderAction({ label: 'Download and view in Browser', href }), + ), + }), + DoesNotExist: () => + renderMessage({ + heading: 'No Such Object', + body: 'Object does not exist', + }), + MalformedJson: ({ message }) => + renderMessage({ + heading: 'Malformed JSON', + body: message, + }), + Unexpected: ({ retry }) => + renderMessage({ + heading: 'Unexpected Error', + body: 'Something went wrong while loading preview', + action: !!retry && renderAction({ label: 'Retry', onClick: retry }), + }), + }), + }), + ) +} + +export const bind = (props) => (data) => diff --git a/catalog/app/components/Preview/Preview.js b/catalog/app/components/Preview/Preview.js deleted file mode 100644 index b54a303b6f1..00000000000 --- a/catalog/app/components/Preview/Preview.js +++ /dev/null @@ -1,36 +0,0 @@ -import AsyncResult from 'utils/AsyncResult' - -import * as loaders from './loaders' -import * as renderers from './renderers' -import { PreviewData, PreviewError } from './types' - -export { PreviewData, PreviewError } - -const fallback = { - detect: () => true, - load: (handle, callback) => - callback(AsyncResult.Err(PreviewError.Unsupported({ handle }))), -} - -const loaderChain = [ - loaders.Csv, - loaders.Excel, - loaders.Fcs, - loaders.Json, - loaders.Markdown, - loaders.Notebook, - loaders.Parquet, - loaders.Pdf, - loaders.Vcf, - loaders.Html, - loaders.Text, - loaders.Image, - fallback, -] - -const chooseLoader = (key) => loaderChain.find((L) => L.detect(key)) - -export const load = (handle, callback) => - chooseLoader(handle.logicalKey || handle.key).load(handle, callback) - -export const render = PreviewData.case(renderers) diff --git a/catalog/app/components/Preview/index.js b/catalog/app/components/Preview/index.js index a9e6b72042c..d73eb6fbd54 100644 --- a/catalog/app/components/Preview/index.js +++ b/catalog/app/components/Preview/index.js @@ -1 +1,4 @@ -export * from './Preview' +export { default as Display, bind as display } from './Display' +export { default as render } from './render' +export { default as load } from './load' +export { PreviewData, PreviewError } from './types' diff --git a/catalog/app/components/Preview/load.js b/catalog/app/components/Preview/load.js new file mode 100644 index 00000000000..e3d3c37bfbf --- /dev/null +++ b/catalog/app/components/Preview/load.js @@ -0,0 +1,39 @@ +import * as React from 'react' + +import * as Csv from './loaders/Csv' +import * as Excel from './loaders/Excel' +import * as Fcs from './loaders/Fcs' +import * as Html from './loaders/Html' +import * as Image from './loaders/Image' +import * as Json from './loaders/Json' +import * as Markdown from './loaders/Markdown' +import * as Notebook from './loaders/Notebook' +import * as Parquet from './loaders/Parquet' +import * as Pdf from './loaders/Pdf' +import * as Text from './loaders/Text' +import * as Vcf from './loaders/Vcf' +import * as fallback from './loaders/fallback' + +const loaderChain = [ + Csv, + Excel, + Fcs, + Json, + Markdown, + Notebook, + Parquet, + Pdf, + Vcf, + Html, + Text, + Image, + fallback, +] + +export function Load({ handle, children }) { + const key = handle.logicalKey || handle.key + const { Loader } = React.useMemo(() => loaderChain.find((L) => L.detect(key)), [key]) + return +} + +export default (handle, children) => diff --git a/catalog/app/components/Preview/loaders/Csv.js b/catalog/app/components/Preview/loaders/Csv.js index 504adea55a3..e7c51e314a6 100644 --- a/catalog/app/components/Preview/loaders/Csv.js +++ b/catalog/app/components/Preview/loaders/Csv.js @@ -1,23 +1,24 @@ import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' export const detect = R.pipe(utils.stripCompression, utils.extIn(['.csv', '.tsv'])) -const fetcher = utils.previewFetcher('csv', (json) => - AsyncResult.Ok( +const isTsv = R.pipe(utils.stripCompression, utils.extIs('.tsv')) + +export const Loader = function CsvLoader({ handle, children }) { + const data = utils.usePreview({ + type: 'csv', + handle, + query: isTsv(handle.key) ? { sep: '\t' } : undefined, + }) + const processed = utils.useProcessing(data.result, (json) => PreviewData.DataFrame({ preview: json.html, note: json.info.note, warnings: json.info.warnings, }), - ), -) - -const isTsv = R.pipe(utils.stripCompression, utils.extIs('.tsv')) - -export const load = (handle, callback) => - fetcher(handle, callback, isTsv(handle.key) ? { query: { sep: '\t' } } : undefined) + ) + return children(utils.useErrorHandling(processed, { handle, retry: data.fetch })) +} diff --git a/catalog/app/components/Preview/loaders/Excel.js b/catalog/app/components/Preview/loaders/Excel.js index 360def1886e..5f9e390fa1b 100644 --- a/catalog/app/components/Preview/loaders/Excel.js +++ b/catalog/app/components/Preview/loaders/Excel.js @@ -1,18 +1,18 @@ import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' export const detect = R.pipe(utils.stripCompression, utils.extIn(['.xls', '.xlsx'])) -export const load = utils.previewFetcher('excel', (json) => - AsyncResult.Ok( +export const Loader = function ExcelLoader({ handle, children }) { + const data = utils.usePreview({ type: 'excel', handle }) + const processed = utils.useProcessing(data.result, (json) => PreviewData.DataFrame({ preview: json.html, note: json.info.note, warnings: json.info.warnings, }), - ), -) + ) + return children(utils.useErrorHandling(processed, { handle, retry: data.fetch })) +} diff --git a/catalog/app/components/Preview/loaders/Fcs.js b/catalog/app/components/Preview/loaders/Fcs.js index cded978da5e..5245b5ca1c7 100644 --- a/catalog/app/components/Preview/loaders/Fcs.js +++ b/catalog/app/components/Preview/loaders/Fcs.js @@ -1,22 +1,19 @@ import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' export const detect = R.pipe(utils.stripCompression, utils.extIs('.fcs')) -export const load = utils.previewFetcher( - 'fcs', - R.pipe( - ({ html, info }) => ({ +export const Loader = function FcsLoader({ handle, children }) { + const data = utils.usePreview({ type: 'fcs', handle }) + const processed = utils.useProcessing(data.result, ({ html, info }) => + PreviewData.Fcs({ preview: html, metadata: info.metadata, note: info.note, warnings: info.warnings, }), - PreviewData.Fcs, - AsyncResult.Ok, - ), -) + ) + return children(utils.useErrorHandling(processed, { handle, retry: data.fetch })) +} diff --git a/catalog/app/components/Preview/loaders/Html.js b/catalog/app/components/Preview/loaders/Html.js index 0d420bca332..007f1b55320 100644 --- a/catalog/app/components/Preview/loaders/Html.js +++ b/catalog/app/components/Preview/loaders/Html.js @@ -2,6 +2,7 @@ import * as React from 'react' import * as AWS from 'utils/AWS' import AsyncResult from 'utils/AsyncResult' import { useIsInStack } from 'utils/BucketConfig' +import useMemoEq from 'utils/useMemoEq' import { PreviewData } from '../types' @@ -10,26 +11,20 @@ import * as utils from './utils' export const detect = utils.extIn(['.htm', '.html']) -const IFrameLoader = ({ handle, children }) => { +function IFrameLoader({ handle, children }) { const sign = AWS.Signer.useS3Signer() - const src = React.useMemo(() => sign(handle, { ResponseContentType: 'text/html' }), [ - handle.bucket, - handle.key, - handle.version, - sign, - ]) - return children(AsyncResult.Ok(AsyncResult.Ok(PreviewData.IFrame({ src })))) + const src = useMemoEq([handle, sign], () => + sign(handle, { ResponseContentType: 'text/html' }), + ) + // TODO: issue a head request to ensure existence and get storage class + return children(AsyncResult.Ok(PreviewData.IFrame({ src }))) } -const HtmlLoader = ({ handle, children }) => { +export const Loader = function HtmlLoader({ handle, children }) { const isInStack = useIsInStack() return isInStack(handle.bucket) ? ( - {children} + ) : ( - Text.load(handle, children) + ) } - -export const load = (handle, callback) => ( - {callback} -) diff --git a/catalog/app/components/Preview/loaders/Image.js b/catalog/app/components/Preview/loaders/Image.js index e357da39b67..3cdc7554361 100644 --- a/catalog/app/components/Preview/loaders/Image.js +++ b/catalog/app/components/Preview/loaders/Image.js @@ -7,5 +7,7 @@ import * as utils from './utils' export const detect = utils.extIn(SUPPORTED_EXTENSIONS) -export const load = (handle, callback) => - callback(AsyncResult.Ok(AsyncResult.Ok(PreviewData.Image({ handle })))) +// TODO: issue a head request to ensure existance and get storage class +export const Loader = function ImageLoader({ handle, children }) { + return children(AsyncResult.Ok(PreviewData.Image({ handle }))) +} diff --git a/catalog/app/components/Preview/loaders/Json.js b/catalog/app/components/Preview/loaders/Json.js index 7dc7f0f47f0..1a8deca2b32 100644 --- a/catalog/app/components/Preview/loaders/Json.js +++ b/catalog/app/components/Preview/loaders/Json.js @@ -1,6 +1,7 @@ import * as R from 'ramda' +import * as React from 'react' -import AsyncResult from 'utils/AsyncResult' +import * as AWS from 'utils/AWS' import * as Resource from 'utils/Resource' import { PreviewData, PreviewError } from '../types' @@ -12,18 +13,23 @@ const SCHEMA_RE = /"\$schema":\s*"https:\/\/vega\.github\.io\/schema\/([\w-]+)\/ const map = (fn) => R.ifElse(Array.isArray, R.map(fn), fn) -const signVegaSpec = ({ signer, handle }) => - R.evolve({ - data: map( - R.evolve({ - url: (url) => - signer.signResource({ - ptr: Resource.parse(url), - ctx: { type: Resource.ContextType.Vega(), handle }, - }), - }), - ), - }) +function useVegaSpecSigner(handle) { + const sign = AWS.Signer.useResourceSigner() + return React.useCallback( + R.evolve({ + data: map( + R.evolve({ + url: (url) => + sign({ + ptr: Resource.parse(url), + ctx: { type: Resource.ContextType.Vega(), handle }, + }), + }), + ), + }), + [sign, handle], + ) +} const detectSchema = (txt) => { const m = txt.match(SCHEMA_RE) @@ -33,40 +39,38 @@ const detectSchema = (txt) => { return { library, version } } -const vegaFetcher = utils.objectGetter((r, { handle, signer }) => { - try { - const contents = r.Body.toString('utf-8') - const spec = JSON.parse(contents) - return PreviewData.Vega({ spec: signVegaSpec({ signer, handle })(spec) }) - } catch (e) { - if (e instanceof SyntaxError) { - throw PreviewError.MalformedJson({ handle, originalError: e }) - } - throw PreviewError.Unexpected({ handle, originalError: e }) - } -}) - -const loadVega = (handle, callback) => - utils.withSigner((signer) => - utils.withS3((s3) => vegaFetcher({ s3, handle, signer }, callback)), - ) - -const loadText = (handle, callback) => - Text.load( - handle, - AsyncResult.case({ - Ok: callback, - _: callback, - }), - { forceLang: 'json' }, +function VegaLoader({ handle, children }) { + const signSpec = useVegaSpecSigner(handle) + const data = utils.useObjectGetter(handle) + const processed = utils.useProcessing( + data.result, + (r) => { + try { + const contents = r.Body.toString('utf-8') + const spec = JSON.parse(contents) + return PreviewData.Vega({ spec: signSpec(spec) }) + } catch (e) { + if (e instanceof SyntaxError) { + throw PreviewError.MalformedJson({ handle, message: e.message }) + } + throw e + } + }, + [signSpec, handle], ) + return children(utils.useErrorHandling(processed, { handle, retry: data.fetch })) +} export const detect = R.either(utils.extIs('.json'), R.startsWith('.quilt/')) -export const load = utils.withFirstBytes( - 256, - ({ firstBytes, contentLength, handle }, callback) => { - const schema = detectSchema(firstBytes) - return (!!schema && contentLength <= MAX_SIZE ? loadVega : loadText)(handle, callback) - }, -) +export const Loader = function JsonLoader({ handle, children }) { + return utils.useFirstBytes({ bytes: 256, handle }).case({ + Ok: ({ firstBytes, contentLength }) => + !!detectSchema(firstBytes) && contentLength <= MAX_SIZE ? ( + + ) : ( + + ), + _: children, + }) +} diff --git a/catalog/app/components/Preview/loaders/Markdown.js b/catalog/app/components/Preview/loaders/Markdown.js index 325c6079057..606aa890929 100644 --- a/catalog/app/components/Preview/loaders/Markdown.js +++ b/catalog/app/components/Preview/loaders/Markdown.js @@ -1,59 +1,89 @@ import { dirname, resolve } from 'path' import * as R from 'ramda' +import * as React from 'react' import { getRenderer } from 'components/Markdown' +import * as AWS from 'utils/AWS' +import AsyncResult from 'utils/AsyncResult' +import * as NamedRoutes from 'utils/NamedRoutes' import * as Resource from 'utils/Resource' +import pipeThru from 'utils/pipeThru' +import useMemoEq from 'utils/useMemoEq' -import { PreviewData } from '../types' +import { PreviewData, PreviewError } from '../types' import * as utils from './utils' -const signImg = ({ signer, handle }) => - R.evolve({ - src: (src) => - signer.signResource({ - ptr: Resource.parse(src), - ctx: { type: Resource.ContextType.MDImg(), handle }, - }), - }) - -const processLink = ({ urls, signer, handle }) => - R.evolve({ - href: R.pipe( - Resource.parse, - Resource.Pointer.case({ - Path: (p) => { - const hasSlash = p.endsWith('/') - const resolved = resolve(dirname(handle.key), p).slice(1) - const normalized = hasSlash ? `${resolved}/` : resolved - return hasSlash - ? urls.bucketDir(handle.bucket, normalized) - : urls.bucketFile(handle.bucket, normalized) - }, - _: (ptr) => - signer.signResource({ - ptr, - ctx: { type: Resource.ContextType.MDLink(), handle }, - }), - }), - ), - }) - -const fetch = utils.gatedS3Request( - utils.objectGetter((r, { handle, signer, urls }) => { - const contents = r.Body.toString('utf-8') - const rendered = getRenderer({ - images: true, - processImg: signImg({ signer, handle }), - processLink: processLink({ urls, signer, handle }), - })(contents) - return PreviewData.Markdown({ rendered }) - }), -) +function useImgProcessor(handle) { + const sign = AWS.Signer.useResourceSigner() + return useMemoEq([sign, handle], () => + R.evolve({ + src: (src) => + sign({ + ptr: Resource.parse(src), + ctx: { type: Resource.ContextType.MDImg(), handle }, + }), + }), + ) +} + +function useLinkProcessor(handle) { + const { urls } = NamedRoutes.use() + const sign = AWS.Signer.useResourceSigner() + return useMemoEq([sign, urls, handle], () => + R.evolve({ + href: R.pipe( + Resource.parse, + Resource.Pointer.case({ + Path: (p) => { + const hasSlash = p.endsWith('/') + const resolved = resolve(dirname(handle.key), p).slice(1) + const normalized = hasSlash ? `${resolved}/` : resolved + return hasSlash + ? urls.bucketDir(handle.bucket, normalized) + : urls.bucketFile(handle.bucket, normalized) + }, + _: (ptr) => + sign({ + ptr, + ctx: { type: Resource.ContextType.MDLink(), handle }, + }), + }), + ), + }), + ) +} export const detect = utils.extIn(['.md', '.rmd']) -export const load = (handle, callback) => - utils.withSigner((signer) => - utils.withRoutes(({ urls }) => fetch(handle, callback, { signer, urls })), +function MarkdownLoader({ gated, handle, children }) { + const processImg = useImgProcessor(handle) + const processLink = useLinkProcessor(handle) + const data = utils.useObjectGetter(handle, { noAutoFetch: gated }) + const processed = utils.useProcessing( + data.result, + (r) => { + const contents = r.Body.toString('utf-8') + const rendered = getRenderer({ images: true, processImg, processLink })(contents) + return PreviewData.Markdown({ rendered }) + }, + [processImg, processLink], + ) + const handled = utils.useErrorHandling(processed, { handle, retry: data.fetch }) + const result = + gated && AsyncResult.Init.is(handled) + ? AsyncResult.Err(PreviewError.Gated({ handle, load: data.fetch })) + : handled + return children(result) +} + +export const Loader = function GatedMarkdownLoader({ handle, children }) { + const data = utils.useGate(handle) + const handled = utils.useErrorHandling(data.result, { handle, retry: data.fetch }) + return pipeThru(handled)( + AsyncResult.case({ + _: children, + Ok: (gated) => , + }), ) +} diff --git a/catalog/app/components/Preview/loaders/Notebook.js b/catalog/app/components/Preview/loaders/Notebook.js index 2a7b9276fb5..788f9498367 100644 --- a/catalog/app/components/Preview/loaders/Notebook.js +++ b/catalog/app/components/Preview/loaders/Notebook.js @@ -1,18 +1,18 @@ import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' export const detect = R.pipe(utils.stripCompression, utils.extIs('.ipynb')) -export const load = utils.previewFetcher('ipynb', (json) => - AsyncResult.Ok( +export const Loader = function NotebookLoader({ handle, children }) { + const data = utils.usePreview({ type: 'ipynb', handle }) + const processed = utils.useProcessing(data.result, (json) => PreviewData.Notebook({ preview: json.html, note: json.info.note, warnings: json.info.warnings, }), - ), -) + ) + return children(utils.useErrorHandling(processed, { handle, retry: data.fetch })) +} diff --git a/catalog/app/components/Preview/loaders/Parquet.js b/catalog/app/components/Preview/loaders/Parquet.js index b3571fca08d..6454d15ed71 100644 --- a/catalog/app/components/Preview/loaders/Parquet.js +++ b/catalog/app/components/Preview/loaders/Parquet.js @@ -1,7 +1,5 @@ import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' @@ -14,10 +12,10 @@ export const detect = R.pipe( ]), ) -export const load = utils.previewFetcher( - 'parquet', - R.pipe( - ({ html, info }) => ({ +export const Loader = function ParquetLoader({ handle, children }) { + const data = utils.usePreview({ type: 'parquet', handle }) + const processed = utils.useProcessing(data.result, ({ html, info }) => + PreviewData.Parquet({ preview: html, createdBy: info.created_by, formatVersion: info.format_version, @@ -29,7 +27,6 @@ export const load = utils.previewFetcher( note: info.note, warnings: info.warnings, }), - PreviewData.Parquet, - AsyncResult.Ok, - ), -) + ) + return children(utils.useErrorHandling(processed, { handle, retry: data.fetch })) +} diff --git a/catalog/app/components/Preview/loaders/Pdf.js b/catalog/app/components/Preview/loaders/Pdf.js index 653d5abc185..5e02410a493 100644 --- a/catalog/app/components/Preview/loaders/Pdf.js +++ b/catalog/app/components/Preview/loaders/Pdf.js @@ -1,8 +1,5 @@ -import * as React from 'react' - import { HTTPError } from 'utils/APIConnector' import * as AWS from 'utils/AWS' -import AsyncResult from 'utils/AsyncResult' import * as Config from 'utils/Config' import * as Data from 'utils/Data' import { mkSearch } from 'utils/NamedRoutes' @@ -29,19 +26,23 @@ async function loadPdf({ endpoint, sign, handle }) { } const { page_count: pages } = JSON.parse(r.headers.get('X-Quilt-Info') || '{}') const firstPageBlob = await r.blob() - return AsyncResult.Ok(PreviewData.Pdf({ handle, pages, firstPageBlob })) + return PreviewData.Pdf({ handle, pages, firstPageBlob }) } catch (e) { - console.warn('error loading pdf preview') + if (e instanceof HTTPError && e.json && e.json.error === 'Forbidden') { + if (e.json.text && e.json.text.match(utils.GLACIER_ERROR_RE)) { + throw PreviewError.Archived({ handle }) + } + throw PreviewError.Forbidden({ handle }) + } + console.warn('error loading pdf preview', { ...e }) console.error(e) - throw PreviewError.Unexpected({ handle, originalError: e }) + throw e } } -function PdfLoader({ handle, callback }) { +export const Loader = function PdfLoader({ handle, children }) { const endpoint = Config.use().binaryApiGatewayEndpoint const sign = AWS.Signer.useS3Signer() const data = Data.use(loadPdf, { endpoint, sign, handle }) - return callback(data.result, { fetch: data.fetch }) + return children(utils.useErrorHandling(data.result, { handle, retry: data.fetch })) } - -export const load = (handle, callback) => diff --git a/catalog/app/components/Preview/loaders/Text.js b/catalog/app/components/Preview/loaders/Text.js index 90953a1f2a0..25eb84b166f 100644 --- a/catalog/app/components/Preview/loaders/Text.js +++ b/catalog/app/components/Preview/loaders/Text.js @@ -3,8 +3,6 @@ import { basename } from 'path' import hljs from 'highlight.js' import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' @@ -59,18 +57,22 @@ const getLang = R.pipe(findLang, ([lang] = []) => lang) const hl = (lang) => (contents) => hljs.highlight(lang, contents).value -const fetcher = utils.previewFetcher( - 'txt', - ({ info: { data, note, warnings } }, { handle, forceLang }) => { - const head = data.head.join('\n') - const tail = data.tail.join('\n') - const lang = forceLang || getLang(handle.logicalKey || handle.key) - const highlighted = R.map(hl(lang), { head, tail }) - return AsyncResult.Ok( - PreviewData.Text({ head, tail, lang, highlighted, note, warnings }), - ) - }, -) - -export const load = (handle, callback, extra) => - fetcher(handle, callback, { query: { max_bytes: MAX_BYTES }, ...extra }) +export const Loader = function TextLoader({ handle, forceLang, children }) { + const { result, fetch } = utils.usePreview({ + type: 'txt', + handle, + query: { max_bytes: MAX_BYTES }, + }) + const processed = utils.useProcessing( + result, + ({ info: { data, note, warnings } }) => { + const head = data.head.join('\n') + const tail = data.tail.join('\n') + const lang = forceLang || getLang(handle.logicalKey || handle.key) + const highlighted = R.map(hl(lang), { head, tail }) + return PreviewData.Text({ head, tail, lang, highlighted, note, warnings }) + }, + [forceLang, handle.logicalKey, handle.key], + ) + return children(utils.useErrorHandling(processed, { handle, retry: fetch })) +} diff --git a/catalog/app/components/Preview/loaders/Vcf.js b/catalog/app/components/Preview/loaders/Vcf.js index 33253ff9d1e..3836b66c5ea 100644 --- a/catalog/app/components/Preview/loaders/Vcf.js +++ b/catalog/app/components/Preview/loaders/Vcf.js @@ -1,15 +1,14 @@ import * as R from 'ramda' -import AsyncResult from 'utils/AsyncResult' - import { PreviewData } from '../types' import * as utils from './utils' export const detect = R.pipe(utils.stripCompression, utils.extIs('.vcf')) -export const load = utils.previewFetcher( - 'vcf', - R.pipe( +export const Loader = function VcfLoader({ handle, children }) { + const { result, fetch } = utils.usePreview({ type: 'vcf', handle }) + const processed = utils.useProcessing( + result, ({ info: { data: { meta, header, data }, @@ -17,8 +16,7 @@ export const load = utils.previewFetcher( note, warnings, }, - }) => ({ meta, header, data, variants, note, warnings }), - PreviewData.Vcf, - AsyncResult.Ok, - ), -) + }) => PreviewData.Vcf({ meta, header, data, variants, note, warnings }), + ) + return children(utils.useErrorHandling(processed, { handle, retry: fetch })) +} diff --git a/catalog/app/components/Preview/loaders/fallback.js b/catalog/app/components/Preview/loaders/fallback.js new file mode 100644 index 00000000000..eb153a4dd4e --- /dev/null +++ b/catalog/app/components/Preview/loaders/fallback.js @@ -0,0 +1,9 @@ +import AsyncResult from 'utils/AsyncResult' + +import { PreviewError } from '../types' + +export const detect = () => true + +export const Loader = function FallbackLoader({ handle, children }) { + return children(AsyncResult.Err(PreviewError.Unsupported({ handle }))) +} diff --git a/catalog/app/components/Preview/loaders/index.js b/catalog/app/components/Preview/loaders/index.js deleted file mode 100644 index be423449483..00000000000 --- a/catalog/app/components/Preview/loaders/index.js +++ /dev/null @@ -1,14 +0,0 @@ -import * as Csv from './Csv' -import * as Excel from './Excel' -import * as Fcs from './Fcs' -import * as Image from './Image' -import * as Html from './Html' -import * as Json from './Json' -import * as Markdown from './Markdown' -import * as Notebook from './Notebook' -import * as Parquet from './Parquet' -import * as Pdf from './Pdf' -import * as Text from './Text' -import * as Vcf from './Vcf' - -export { Csv, Excel, Fcs, Image, Html, Json, Markdown, Notebook, Parquet, Pdf, Text, Vcf } diff --git a/catalog/app/components/Preview/loaders/utils.js b/catalog/app/components/Preview/loaders/utils.js index f26f0d8ff1e..0a41f9a96bc 100644 --- a/catalog/app/components/Preview/loaders/utils.js +++ b/catalog/app/components/Preview/loaders/utils.js @@ -1,13 +1,14 @@ import { extname } from 'path' import * as R from 'ramda' -import * as React from 'react' import * as AWS from 'utils/AWS' import AsyncResult from 'utils/AsyncResult' import * as Config from 'utils/Config' -import Data from 'utils/Data' -import * as NamedRoutes from 'utils/NamedRoutes' +import * as Data from 'utils/Data' +import { mkSearch } from 'utils/NamedRoutes' +import pipeThru from 'utils/pipeThru' +import useMemoEq from 'utils/useMemoEq' import { PreviewError } from '../types' @@ -18,6 +19,8 @@ export const SIZE_THRESHOLDS = [ export const COMPRESSION_TYPES = { gz: '.gz', bz2: '.bz2' } +export const GLACIER_ERROR_RE = /InvalidObjectState<\/Code>The operation is not valid for the object's storage class<\/Message>/ + // eslint-disable-next-line consistent-return export const getCompression = (key) => { // eslint-disable-next-line no-restricted-syntax @@ -35,28 +38,40 @@ export const extIs = (ext) => (key) => extname(key).toLowerCase() === ext export const extIn = (exts) => (key) => exts.includes(extname(key).toLowerCase()) -export const withSigner = (callback) => {callback} - -export const withS3 = (callback) => {callback} - -export const withRoutes = (callback) => ( - {callback} -) - -export const withData = (props, callback) => {callback} - +// TODO: make it more general-purpose "head"? const gate = async ({ s3, handle }) => { let length + const req = s3.headObject({ + Bucket: handle.bucket, + Key: handle.key, + VersionId: handle.version, + }) try { - const head = await s3 - .headObject({ - Bucket: handle.bucket, - Key: handle.key, - VersionId: handle.version, - }) - .promise() + const head = await req.promise() length = head.ContentLength + if (head.DeleteMarker) throw PreviewError.Deleted({ handle }) + if (head.StorageClass === 'GLACIER' || head.StorageClass === 'DEEP_ARCHIVE') { + throw PreviewError.Archived({ handle }) + } } catch (e) { + if (PreviewError.is(e)) throw e + // TODO: should it be 'status'? + if (e.code === 405 && handle.version != null) { + // assume delete marker when 405 and version is defined, + // since GET and HEAD methods are not allowed on delete markers + // (https://github.com/boto/botocore/issues/674) + throw PreviewError.Deleted({ handle }) + } + if (e.code === 'BadRequest' && handle.version != null) { + // assume invalid version when 400 and version is defined + throw PreviewError.InvalidVersion({ handle }) + } + if ( + e.code === 'NotFound' && + req.response.httpResponse.headers['x-amz-delete-marker'] === 'true' + ) { + throw PreviewError.Deleted({ handle }) + } if (['NoSuchKey', 'NotFound'].includes(e.name)) { throw PreviewError.DoesNotExist({ handle }) } @@ -64,27 +79,18 @@ const gate = async ({ s3, handle }) => { console.error('Error loading preview') // eslint-disable-next-line no-console console.error(e) - throw PreviewError.Unexpected({ handle, originalError: e }) + throw e } if (length > SIZE_THRESHOLDS[1]) { throw PreviewError.TooLarge({ handle }) } - return { handle, gated: length > SIZE_THRESHOLDS[0] } + return length > SIZE_THRESHOLDS[0] } -export const gatedS3Request = (fetcher) => (handle, callback, extraParams) => - withS3((s3) => - withData( - { fetch: gate, params: { s3, handle } }, - AsyncResult.case({ - Ok: ({ gated }) => - fetcher({ s3, handle, gated, ...extraParams }, (r, ...args) => - callback(AsyncResult.Ok(r), ...args), - ), - _: callback, - }), - ), - ) +export function useGate(handle) { + const s3 = AWS.S3.use() + return Data.use(gate, { s3, handle }) +} const parseRange = (range) => { if (!range) return undefined @@ -93,7 +99,7 @@ const parseRange = (range) => { return Number(m[1]) } -const getFirstBytes = (bytes) => async ({ s3, handle }) => { +const getFirstBytes = async ({ s3, bytes, handle }) => { try { const res = await s3 .getObject({ @@ -107,102 +113,105 @@ const getFirstBytes = (bytes) => async ({ s3, handle }) => { const contentLength = parseRange(res.ContentRange) || 0 return { firstBytes, contentLength } } catch (e) { - if (['NoSuchKey', 'NotFound'].includes(e.name)) { + if (['NoSuchKey', 'NotFound'].includes(e.code)) { throw PreviewError.DoesNotExist({ handle }) } + if (e.code === 'InvalidObjectState') { + throw PreviewError.Archived({ handle }) + } + if (e.code === 'InvalidArgument' && e.message === 'Invalid version id specified') { + throw PreviewError.InvalidVersion({ handle }) + } // eslint-disable-next-line no-console console.error('Error loading preview') // eslint-disable-next-line no-console console.error(e) - throw PreviewError.Unexpected({ handle, originalError: e }) + throw e } } -export const withFirstBytes = (bytes, fetcher) => { - const fetch = getFirstBytes(bytes) - - return (handle, callback) => - withS3((s3) => - withData( - { fetch, params: { s3, handle } }, - AsyncResult.case({ - Ok: ({ firstBytes, contentLength }) => - fetcher({ s3, handle, firstBytes, contentLength }, (r, ...args) => - callback(AsyncResult.Ok(r), ...args), - ), - _: callback, - }), - ), - ) +export function useFirstBytes({ bytes, handle }) { + const s3 = AWS.S3.use() + return Data.use(getFirstBytes, { s3, bytes, handle }) } -export const objectGetter = (process) => { - const fetch = ({ s3, handle, ...extra }) => - s3 - .getObject({ - Bucket: handle.bucket, - Key: handle.key, - VersionId: handle.version, - }) - .promise() - .then((r) => process(r, { s3, handle, ...extra })) - - return ({ s3, handle, gated, ...extra }, callback) => - withData({ fetch, params: { s3, handle, ...extra }, noAutoFetch: gated }, callback) +const getObject = ({ s3, handle }) => + s3 + .getObject({ + Bucket: handle.bucket, + Key: handle.key, + VersionId: handle.version, + }) + .promise() + .catch((e) => { + if (['NoSuchKey', 'NotFound'].includes(e.code)) { + throw PreviewError.DoesNotExist({ handle }) + } + if (e.code === 'InvalidObjectState') { + throw PreviewError.Archived({ handle }) + } + if (e.code === 'InvalidArgument' && e.message === 'Invalid version id specified') { + throw PreviewError.InvalidVersion({ handle }) + } + throw e + }) + +export function useObjectGetter(handle, opts) { + const s3 = AWS.S3.use() + return Data.use(getObject, { s3, handle }, opts) } -const previewUrl = (endpoint, query) => - `${endpoint}/preview${NamedRoutes.mkSearch(query)}` - -class PreviewFetchError extends Error {} - -const fetchPreview = async ({ endpoint, type, handle, signer, query }) => { - const signed = signer.getSignedS3URL(handle) - const compression = getCompression(handle.key) +const fetchPreview = async ({ endpoint, handle, sign, type, compression, query }) => { + const url = sign(handle) const r = await fetch( - previewUrl(endpoint, { url: signed, input: type, compression, ...query }), + `${endpoint}/preview${mkSearch({ url, input: type, compression, ...query })}`, ) const json = await r.json() - if (json.error) throw new PreviewFetchError(json.error) + if (json.error) { + if (json.error === 'Not Found') { + throw PreviewError.DoesNotExist({ handle }) + } + if (json.error === 'Forbidden') { + if (json.text && json.text.match(GLACIER_ERROR_RE)) { + throw PreviewError.Archived({ handle }) + } + throw PreviewError.Forbidden({ handle }) + } + console.log('Error from preview endpoint', json) + throw new Error(json.error) + } return json } -const withGatewayEndpoint = (callback) => ( - - {AsyncResult.case({ - Err: R.pipe(AsyncResult.Err, callback), - _: callback, - Ok: R.pipe(R.prop('apiGatewayEndpoint'), AsyncResult.Ok, callback), - })} - -) - -export const previewFetcher = (type, process) => { - const fetch = (x) => fetchPreview(x).then((res) => process(res, x)) - return (handle, callback, extra) => - withSigner((signer) => - withGatewayEndpoint( - AsyncResult.case({ - Err: (e, ...args) => { - const pe = PreviewError.Unexpected({ handle, originalError: e }) - return callback(AsyncResult.Err(pe), ...args) - }, - Ok: (endpoint) => - withData( - { fetch, params: { endpoint, type, handle, signer, ...extra } }, - AsyncResult.case({ - _: callback, - Err: (e, ...args) => { - let pe = PreviewError.Unexpected({ handle, originalError: e }) - if (e instanceof PreviewFetchError && e.message === 'Not Found') { - pe = PreviewError.DoesNotExist({ handle }) - } - return callback(AsyncResult.Err(pe), ...args) - }, - }), - ), - _: callback, +export function usePreview({ type, handle, query }) { + const { apiGatewayEndpoint: endpoint } = Config.use() + const sign = AWS.Signer.useS3Signer() + const compression = getCompression(handle.key) + return Data.use(fetchPreview, { endpoint, handle, sign, type, compression, query }) +} + +export function useProcessing(asyncResult, process, deps = []) { + return useMemoEq([asyncResult, process, deps], () => + AsyncResult.case( + { + Ok: R.tryCatch(R.pipe(process, AsyncResult.Ok), AsyncResult.Err), + _: R.identity, + }, + asyncResult, + ), + ) +} + +export function useErrorHandling(result, { handle, retry } = {}) { + return useMemoEq([result, handle, retry], () => + pipeThru(result)( + AsyncResult.mapCase({ + Err: R.unless(PreviewError.is, (e) => { + console.log('error while loading preview') + console.error(e) + return PreviewError.Unexpected({ handle, retry, originalError: e }) }), - ), - ) + }), + ), + ) } diff --git a/catalog/app/components/Preview/render.js b/catalog/app/components/Preview/render.js new file mode 100644 index 00000000000..f542f37c0a1 --- /dev/null +++ b/catalog/app/components/Preview/render.js @@ -0,0 +1,4 @@ +import * as renderers from './renderers' +import { PreviewData } from './types' + +export default PreviewData.case(renderers) diff --git a/catalog/app/components/Preview/types.js b/catalog/app/components/Preview/types.js index d349b8f3f4b..27ebbbc0cf9 100644 --- a/catalog/app/components/Preview/types.js +++ b/catalog/app/components/Preview/types.js @@ -38,9 +38,14 @@ export const PreviewData = tagged([ ]) export const PreviewError = tagged([ + 'Deleted', // { handle } + 'Archived', // { handle } + 'InvalidVersion', // { handle } + 'Forbidden', // { handle } + 'Gated', // { handle, load } 'TooLarge', // { handle } 'Unsupported', // { handle } 'DoesNotExist', // { handle } - 'Unexpected', // { handle, originalError: any } - 'MalformedJson', // { handle, originalError: SyntaxError } + 'MalformedJson', // { handle, message } + 'Unexpected', // { handle, retry, originalError: any } ]) diff --git a/catalog/app/components/SearchResults/SearchResults.js b/catalog/app/components/SearchResults/SearchResults.js index bfbd5172770..0e94d4f049d 100644 --- a/catalog/app/components/SearchResults/SearchResults.js +++ b/catalog/app/components/SearchResults/SearchResults.js @@ -12,6 +12,7 @@ import AsyncResult from 'utils/AsyncResult' import * as AWS from 'utils/AWS' import { useBucketExistence } from 'utils/BucketCache' import * as Config from 'utils/Config' +import * as Data from 'utils/Data' import Delay from 'utils/Delay' import * as NamedRoutes from 'utils/NamedRoutes' import StyledLink, { linkStyle } from 'utils/StyledLink' @@ -19,6 +20,8 @@ import { getBreadCrumbs } from 'utils/s3paths' import { readableBytes } from 'utils/string' import usePrevious from 'utils/usePrevious' +import * as requests from 'containers/Bucket/requests' + const PER_PAGE = 10 const ES_V = '6.7' const ES_REF = `https://www.elastic.co/guide/en/elasticsearch/reference/${ES_V}/query-dsl-query-string-query.html#query-string-syntax` @@ -79,31 +82,26 @@ function HeaderIcon(props) { ) } -function ObjectHeader({ handle, showBucket, bucketExistenceData }) { - const cfg = Config.use() +function ObjectHeader({ handle, showBucket, downloadable }) { return ( - {!cfg.noDownload && - bucketExistenceData.case({ - _: () => null, - Ok: () => - AWS.Signer.withDownloadUrl(handle, (url) => ( - - - arrow_downward - - - )), - })} + {downloadable && + AWS.Signer.withDownloadUrl(handle, (url) => ( + + + arrow_downward + + + ))} ) } @@ -187,7 +185,10 @@ function VersionInfo({ bucket, path, version, versions }) { const t = M.useTheme() const xs = M.useMediaQuery(t.breakpoints.down('xs')) - const clip = (str, len) => (xs ? str.substring(0, len) : str) + const clip = (str, len) => { + const s = `${str}` + return xs ? s.substring(0, len) : s + } return ( <> @@ -292,7 +293,7 @@ const usePreviewBoxStyles = M.makeStyles((t) => ({ }, })) -function PreviewBox({ data }) { +function PreviewBox({ contents }) { const classes = usePreviewBoxStyles() const [expanded, setExpanded] = React.useState(false) const expand = React.useCallback(() => { @@ -300,7 +301,7 @@ function PreviewBox({ data }) { }, [setExpanded]) return (
- {Preview.render(data)} + {contents} {!expanded && (
@@ -312,51 +313,41 @@ function PreviewBox({ data }) { ) } -function PreviewDisplay({ handle, bucketExistenceData }) { - if (!handle.version) return null - return ( - - Preview - {bucketExistenceData.case({ - _: () => , - Err: () => ( - - Error loading preview: bucket does not exist - - ), - Ok: () => - Preview.load( - handle, - AsyncResult.case({ - Ok: AsyncResult.case({ - Init: (_, { fetch }) => ( - - Large files are not previewed by default{' '} - - Load preview - - - ), - Pending: () => , - Err: (_, { fetch }) => ( - - Error loading preview{' '} - - Retry - - - ), - Ok: (data) => , - }), - Err: () => ( - Preview not available +const renderContents = (contents) => + +function PreviewDisplay({ handle, bucketExistenceData, versionExistenceData }) { + const withData = (callback) => + bucketExistenceData.case({ + _: callback, + Err: () => callback(AsyncResult.Err(Preview.PreviewError.DoesNotExist({ handle }))), + Ok: () => + versionExistenceData.case({ + _: callback, + Err: (e) => + callback( + AsyncResult.Err( + Preview.PreviewError.Unexpected({ handle, originalError: e }), ), - _: () => , - }), - ), - })} - - ) + ), + Ok: requests.ObjectExistence.case({ + Exists: (h) => { + if (h.deleted) { + return callback(AsyncResult.Err(Preview.PreviewError.Deleted({ handle }))) + } + if (h.archived) { + return callback( + AsyncResult.Err(Preview.PreviewError.Archived({ handle })), + ) + } + return Preview.load(handle, callback) + }, + DoesNotExist: () => + callback(AsyncResult.Err(Preview.PreviewError.InvalidVersion({ handle }))), + }), + }), + }) + + return {withData(Preview.display({ renderContents }))} } function Meta({ meta }) { @@ -436,24 +427,36 @@ function RevisionInfo({ bucket, handle, revision, hash, comment, lastModified }) ) } -const getDefaultVersion = (versions) => versions.find((v) => !!v.id) || versions[0] - function ObjectHit({ showBucket, hit: { path, versions, bucket } }) { - const v = getDefaultVersion(versions) - const data = useBucketExistence(bucket) + const cfg = Config.use() + const s3 = AWS.S3.use() + + const v = versions[0] + const handle = { bucket, key: path, version: v.id } + + const bucketExistenceData = useBucketExistence(bucket) + const versionExistenceData = Data.use(requests.getObjectExistence, { s3, ...handle }) + + const downloadable = + !cfg.noDownload && + bucketExistenceData.case({ + _: () => false, + Ok: () => + versionExistenceData.case({ + _: () => false, + Ok: requests.ObjectExistence.case({ + _: () => false, + Exists: ({ deleted, archived }) => !deleted && !archived, + }), + }), + }) + return (
- + - +
) } @@ -566,7 +569,14 @@ export const handleErr = (retry) =>
Error details:
- {e.details} + + {e.details} + )} diff --git a/catalog/app/constants/embed-routes.js b/catalog/app/constants/embed-routes.js index 4b72f00043c..4377f66df57 100644 --- a/catalog/app/constants/embed-routes.js +++ b/catalog/app/constants/embed-routes.js @@ -6,7 +6,7 @@ export const bucketRoot = { url: (bucket) => `/b/${bucket}`, } export const bucketFile = { - path: '/b/:bucket/tree/:path+', + path: '/b/:bucket/tree/:path(.*[^/])', url: (bucket, path, version) => `/b/${bucket}/tree/${encode(path)}${mkSearch({ version })}`, } diff --git a/catalog/app/constants/routes.js b/catalog/app/constants/routes.js index ecc12a72199..9a4c373fd59 100644 --- a/catalog/app/constants/routes.js +++ b/catalog/app/constants/routes.js @@ -91,7 +91,7 @@ export const bucketSearch = { url: (bucket, q, p, mode) => `/b/${bucket}/search${mkSearch({ q, p, mode })}`, } export const bucketFile = { - path: '/b/:bucket/tree/:path+', + path: '/b/:bucket/tree/:path(.*[^/])', url: (bucket, path, version) => `/b/${bucket}/tree/${encode(path)}${mkSearch({ version })}`, } diff --git a/catalog/app/containers/Bucket/File.js b/catalog/app/containers/Bucket/File.js index 514140d144d..8380196d04f 100644 --- a/catalog/app/containers/Bucket/File.js +++ b/catalog/app/containers/Bucket/File.js @@ -10,22 +10,25 @@ import * as M from '@material-ui/core' import { Crumb, copyWithoutSpaces, render as renderCrumbs } from 'components/BreadCrumbs' import Message from 'components/Message' +import * as Preview from 'components/Preview' import Sparkline from 'components/Sparkline' import * as Notifications from 'containers/Notifications' import * as AWS from 'utils/AWS' +import AsyncResult from 'utils/AsyncResult' import * as Config from 'utils/Config' import { useData } from 'utils/Data' import * as NamedRoutes from 'utils/NamedRoutes' import * as SVG from 'utils/SVG' -import StyledLink, { linkStyle } from 'utils/StyledLink' +import { linkStyle } from 'utils/StyledLink' import copyToClipboard from 'utils/clipboard' import parseSearch from 'utils/parseSearch' import { getBreadCrumbs, up, decode, handleToHttpsUri } from 'utils/s3paths' import { readableBytes, readableQuantity } from 'utils/string' import Code from './Code' -import FilePreview from './FilePreview' +import * as FileView from './FileView' import Section from './Section' +import renderPreview from './renderPreview' import * as requests from './requests' const getCrumbs = ({ bucket, path, urls }) => @@ -133,6 +136,7 @@ function VersionInfo({ bucket, path, version }) { {!cfg.noDownload && ( {!v.deleteMarker && + !v.archived && AWS.Signer.withDownloadUrl( { bucket, key: path, version: v.id }, (url) => ( @@ -200,31 +204,10 @@ function VersionInfo({ bucket, path, version }) { ) } -const AnnotationsBox = M.styled('div')(({ theme: t }) => ({ - background: M.colors.lightBlue[50], - border: [[1, 'solid', M.colors.lightBlue[400]]], - borderRadius: t.shape.borderRadius, - fontFamily: t.typography.monospace.fontFamily, - fontSize: t.typography.body2.fontSize, - overflow: 'auto', - padding: t.spacing(1), - whiteSpace: 'pre', - width: '100%', -})) - -function Annotations({ bucket, path, version }) { +function Meta({ bucket, path, version }) { const s3 = AWS.S3.use() const data = useData(requests.objectMeta, { s3, bucket, path, version }) - return data.case({ - Ok: (meta) => - !!meta && - !R.isEmpty(meta) && ( -
- {JSON.stringify(meta, null, 2)} -
- ), - _: () => null, - }) + return } function Analytics({ analyticsBucket, bucket, path }) { @@ -346,8 +329,6 @@ export default function File({ const classes = useStyles() const { urls } = NamedRoutes.use() const { analyticsBucket, noDownload } = Config.use() - const t = M.useTheme() - const xs = M.useMediaQuery(t.breakpoints.down('xs')) const s3 = AWS.S3.use() const path = decode(encodedPath) @@ -396,12 +377,29 @@ export default function File({ _: () => false, Ok: requests.ObjectExistence.case({ _: () => false, - Exists: ({ deleted }) => !deleted, + Exists: ({ deleted, archived }) => !deleted && !archived, }), }) + const handle = { bucket, key: path, version } + + const withPreview = (callback) => + requests.ObjectExistence.case({ + Exists: (h) => { + if (h.deleted) { + return callback(AsyncResult.Err(Preview.PreviewError.Deleted({ handle }))) + } + if (h.archived) { + return callback(AsyncResult.Err(Preview.PreviewError.Archived({ handle }))) + } + return Preview.load(handle, callback) + }, + DoesNotExist: () => + callback(AsyncResult.Err(Preview.PreviewError.InvalidVersion({ handle }))), + }) + return ( - +
{renderCrumbs(getCrumbs({ bucket, path, urls }))}
@@ -420,31 +418,7 @@ export default function File({ )}
- {downloadable && - AWS.Signer.withDownloadUrl({ bucket, key: path, version }, (url) => - xs ? ( - - arrow_downward - - ) : ( - arrow_downward} - download - > - Download file - - ), - )} + {downloadable && }
{objExistsData.case({ _: () => , @@ -456,6 +430,7 @@ export default function File({ ) } + // TODO: handle this more gracefully throw e }, Ok: requests.ObjectExistence.case({ @@ -469,45 +444,15 @@ export default function File({ Err: (e) => { throw e }, - Ok: requests.ObjectExistence.case({ - Exists: (h) => - !h.deleted ? ( - - ) : ( - - - DELETE MARKER - - - Selected version of the object is a{' '} - - delete marker - - - - ), - DoesNotExist: () => ( - - - INVALID VERSION - - - Invalid version id specified - - - ), - }), + Ok: withPreview(renderPreview), })} - + ), _: () => , }), })} - + ) } diff --git a/catalog/app/containers/Bucket/FilePreview.js b/catalog/app/containers/Bucket/FilePreview.js deleted file mode 100644 index 28117edadd7..00000000000 --- a/catalog/app/containers/Bucket/FilePreview.js +++ /dev/null @@ -1,107 +0,0 @@ -import * as React from 'react' -import Button from '@material-ui/core/Button' -import CircularProgress from '@material-ui/core/CircularProgress' -import Typography from '@material-ui/core/Typography' -import { styled } from '@material-ui/styles' - -import * as Preview from 'components/Preview' -import * as AWS from 'utils/AWS' -import AsyncResult from 'utils/AsyncResult' -import * as Config from 'utils/Config' - -const Message = styled('div')({ - textAlign: 'center', - width: '100%', -}) - -export default function FilePreview({ handle }) { - const cfg = Config.use() - return Preview.load( - handle, - AsyncResult.case({ - Ok: AsyncResult.case({ - Init: (_, { fetch }) => ( - - - Large files are not previewed automatically - - - - ), - Pending: () => ( - - - - ), - Err: (_, { fetch }) => ( - - - Error loading preview - - - - ), - Ok: (data) => Preview.render(data), - }), - Err: Preview.PreviewError.case({ - TooLarge: () => ( - - - Object is too large to preview - - {!cfg.noDownload && - AWS.Signer.withDownloadUrl(handle, (url) => ( - - ))} - - ), - Unsupported: () => ( - - - Preview not available - - {!cfg.noDownload && - AWS.Signer.withDownloadUrl(handle, (url) => ( - - ))} - - ), - DoesNotExist: () => ( - - Object does not exist - - ), - MalformedJson: ({ originalError: { message } }) => ( - - - Malformed JSON: {message} - - - ), - Unexpected: (_, { fetch }) => ( - - - Error loading preview - - - - ), - }), - _: () => ( - - - - ), - }), - ) -} diff --git a/catalog/app/containers/Bucket/FileView.js b/catalog/app/containers/Bucket/FileView.js new file mode 100644 index 00000000000..a05dd8d6fac --- /dev/null +++ b/catalog/app/containers/Bucket/FileView.js @@ -0,0 +1,134 @@ +import * as R from 'ramda' +import * as React from 'react' +import * as M from '@material-ui/core' + +// import Message from 'components/Message' +import * as AWS from 'utils/AWS' +import AsyncResult from 'utils/AsyncResult' +import pipeThru from 'utils/pipeThru' + +// import Code from './Code' +import Section from './Section' + +// TODO: move here everything that's reused btw Bucket/File, Bucket/PackageTree and Embed/File + +const useMetaStyles = M.makeStyles((t) => ({ + box: { + background: M.colors.lightBlue[50], + border: [[1, 'solid', M.colors.lightBlue[400]]], + borderRadius: t.shape.borderRadius, + fontFamily: t.typography.monospace.fontFamily, + fontSize: t.typography.body2.fontSize, + overflow: 'auto', + padding: t.spacing(1), + whiteSpace: 'pre', + width: '100%', + }, +})) + +export function Meta({ data }) { + const classes = useMetaStyles() + return pipeThru(data)( + AsyncResult.case({ + Ok: (meta) => + !!meta && + !R.isEmpty(meta) && ( +
+
{JSON.stringify(meta, null, 2)}
+
+ ), + _: () => null, + }), + ) +} + +const useDownloadButtonStyles = M.makeStyles(() => ({ + button: { + flexShrink: 0, + marginBottom: -3, + marginTop: -3, + }, +})) + +export function DownloadButton({ handle }) { + const classes = useDownloadButtonStyles() + const t = M.useTheme() + const xs = M.useMediaQuery(t.breakpoints.down('xs')) + + return AWS.Signer.withDownloadUrl(handle, (url) => + xs ? ( + + arrow_downward + + ) : ( + arrow_downward} + download + > + Download file + + ), + ) +} + +export function Root(props) { + return +} + +/* +export function Header() { +} + +export function TopBar() { +} + +export function GlobalProgress() { +} + +export function GlobalError(props) { + return +} + +const renderDownload = (handle) => !!handle && + +function FileView({ + header, + subheader, +}) { + return ( + +
{header}
+ + {subheader} + 'spacer' + {withDownloadData(renderDownload)} + + {withFileData(AsyncResult.case({ + //_: () => , + _: renderFileProgress, + // TODO: use proper err msgs + //Err: (e) => , + Err: renderFileErr, + Ok: () => ( + <> + {withCodeData(renderCode)} + {withAnalyticsData(renderAnalytics)} + {withPreviewData(renderPreview)} + {withMetaData(renderMeta)} + + ), + }))} + + ) +} +*/ diff --git a/catalog/app/containers/Bucket/Listing.js b/catalog/app/containers/Bucket/Listing.js index a450c4cf5be..aaf6868ed45 100644 --- a/catalog/app/containers/Bucket/Listing.js +++ b/catalog/app/containers/Bucket/Listing.js @@ -10,6 +10,8 @@ import tagged from 'utils/tagged' import useDebouncedInput from 'utils/useDebouncedInput' import usePrevious from 'utils/usePrevious' +const EMPTY = {''} + function WrappedAutosizeInput({ className, ...props }) { return } @@ -342,7 +344,7 @@ export default function Listing({ items, truncated = false, locked = false, load {pagination.paginated.map( ListingItem.case({ Dir: ({ name, to }) => ( - + ), File: ({ name, to, size, modified }) => ( diff --git a/catalog/app/containers/Bucket/Overview.js b/catalog/app/containers/Bucket/Overview.js index 3f89c2a6dbc..657ee4e38bc 100644 --- a/catalog/app/containers/Bucket/Overview.js +++ b/catalog/app/containers/Bucket/Overview.js @@ -933,7 +933,7 @@ const usePreviewBoxStyles = M.makeStyles((t) => ({ }, })) -function PreviewBox({ data, expanded: defaultExpanded = false }) { +function PreviewBox({ contents, expanded: defaultExpanded = false }) { const classes = usePreviewBoxStyles() const [expanded, setExpanded] = React.useState(defaultExpanded) const expand = React.useCallback(() => { @@ -941,7 +941,7 @@ function PreviewBox({ data, expanded: defaultExpanded = false }) { }, [setExpanded]) return (
- {Preview.render(data)} + {contents} {!expanded && (
@@ -953,7 +953,7 @@ function PreviewBox({ data, expanded: defaultExpanded = false }) { ) } -function FilePreview({ handle, headingOverride, fallback, expanded }) { +function FilePreview({ handle, headingOverride, expanded }) { const { urls } = NamedRoutes.use() const crumbs = React.useMemo(() => { @@ -969,74 +969,32 @@ function FilePreview({ handle, headingOverride, fallback, expanded }) { return { dirs, file } }, [handle, urls]) - const renderCrumbs = () => ( - - {crumbs.dirs.map((c) => ( - - -  /{' '} - - ))} - - - ) - - const heading = headingOverride != null ? headingOverride : renderCrumbs() + const heading = + headingOverride != null ? ( + headingOverride + ) : ( + + {crumbs.dirs.map((c) => ( + + +  /{' '} + + ))} + + + ) - return Preview.load( - handle, - AsyncResult.case({ - Ok: AsyncResult.case({ - Init: (_, { fetch }) => ( -
- - Large files are not previewed automatically - - - Load preview - -
- ), - Pending: () => ( -
- -
- ), - Err: (_, { fetch }) => ( -
- - Error loading preview - - - Retry - -
- ), - Ok: (data) => ( -
- -
- ), - }), - Err: Preview.PreviewError.case({ - DoesNotExist: (...args) => (fallback ? fallback(...args) : null), - _: (_, { fetch }) => ( -
- - Error loading preview - - - Retry - -
- ), - }), - _: () => ( -
- -
- ), - }), + // TODO: check for glacier and hide items + return ( +
+ {Preview.load( + handle, + Preview.display({ + renderContents: (contents) => , + renderProgress: () => , + }), + )} +
) } diff --git a/catalog/app/containers/Bucket/PackageTree.js b/catalog/app/containers/Bucket/PackageTree.js index 8dc5ef19e71..d552a1b229a 100644 --- a/catalog/app/containers/Bucket/PackageTree.js +++ b/catalog/app/containers/Bucket/PackageTree.js @@ -7,6 +7,7 @@ import { Link as RRLink } from 'react-router-dom' import * as M from '@material-ui/core' import { Crumb, copyWithoutSpaces, render as renderCrumbs } from 'components/BreadCrumbs' +import * as Preview from 'components/Preview' import Skeleton from 'components/Skeleton' import AsyncResult from 'utils/AsyncResult' import * as AWS from 'utils/AWS' @@ -19,10 +20,11 @@ import Link, { linkStyle } from 'utils/StyledLink' import * as s3paths from 'utils/s3paths' import Code from './Code' -import FilePreview from './FilePreview' +import * as FileView from './FileView' import Listing, { ListingItem } from './Listing' import Section from './Section' import Summary from './Summary' +import renderPreview from './renderPreview' import * as requests from './requests' const MAX_REVISIONS = 5 @@ -288,7 +290,39 @@ function PkgCode({ data, bucket, name, revision, path }) { return code && {code} } -function DirDisplay({ bucket, name, revision, path }) { +const useTopBarStyles = M.makeStyles((t) => ({ + topBar: { + alignItems: 'flex-end', + display: 'flex', + marginBottom: t.spacing(2), + }, + crumbs: { + ...t.typography.body1, + maxWidth: 'calc(100% - 160px)', + overflowWrap: 'break-word', + [t.breakpoints.down('xs')]: { + maxWidth: 'calc(100% - 40px)', + }, + }, + spacer: { + flexGrow: 1, + }, +})) + +function TopBar({ crumbs, children }) { + const classes = useTopBarStyles() + return ( +
+
+ {renderCrumbs(crumbs)} +
+
+ {children} +
+ ) +} + +function DirDisplay({ bucket, name, revision, path, crumbs }) { const s3 = AWS.S3.use() const { apiGatewayEndpoint: endpoint } = Config.use() const credentials = AWS.Credentials.use() @@ -352,6 +386,7 @@ function DirDisplay({ bucket, name, revision, path }) { const lazyHandles = objects.map((basename) => ({ logicalKey: path + basename })) return ( <> + @@ -367,29 +402,35 @@ function DirDisplay({ bucket, name, revision, path }) { Err: (e) => { console.error(e) return ( - - - Error loading directory - - - Seems like there's no such directory in this package - - + <> + + + + Error loading directory + + + Seems like there's no such directory in this package + + + ) }, _: () => ( // TODO: skeleton placeholder - - - + <> + + + + + ), }) } -function FileDisplay({ bucket, name, revision, path }) { +function FileDisplay({ bucket, name, revision, path, crumbs }) { const s3 = AWS.S3.use() - const { apiGatewayEndpoint: endpoint } = Config.use() const credentials = AWS.Credentials.use() + const { apiGatewayEndpoint: endpoint, noDownload } = Config.use() const data = useData(requests.packageFileDetail, { s3, @@ -403,75 +444,88 @@ function FileDisplay({ bucket, name, revision, path }) { const hashData = useData(requests.loadRevisionHash, { s3, bucket, name, id: revision }) + const renderProgress = () => ( + // TODO: skeleton placeholder + <> + + + + + + ) + + const renderError = (headline, detail) => ( + <> + + + + {headline} + + {!!detail && ( + + {detail} + + )} + + + ) + + const withPreview = ({ archived, deleted, handle }, callback) => { + if (deleted) { + return callback(AsyncResult.Err(Preview.PreviewError.Deleted({ handle }))) + } + if (archived) { + return callback(AsyncResult.Err(Preview.PreviewError.Archived({ handle }))) + } + return Preview.load(handle, callback) + } + return data.case({ Ok: (handle) => ( - <> - -
- -
- + + {AsyncResult.case({ + _: renderProgress, + Err: (e) => { + if (e.code === 'Forbidden') { + return renderError('Access Denied', "You don't have access to this object") + } + console.error(e) + return renderError('Error loading file', 'Something went wrong') + }, + Ok: requests.ObjectExistence.case({ + Exists: ({ archived, deleted }) => ( + <> + + {!noDownload && !deleted && !archived && ( + + )} + + +
+ {withPreview({ archived, deleted, handle }, renderPreview)} +
+ + ), + _: () => renderError('No Such Object'), + }), + })} + ), Err: (e) => { console.error(e) - return ( - - - Error loading file - - - Seems like there's no such file in this package - - + return renderError( + 'Error loading file', + "Seems like there's no such file in this package", ) }, - _: () => ( - // TODO: skeleton placeholder - - - - ), + _: renderProgress, }) } -const useStyles = M.makeStyles((t) => ({ - topBar: { - alignItems: 'flex-end', - display: 'flex', - marginBottom: t.spacing(2), - }, - crumbs: { - ...t.typography.body1, - maxWidth: 'calc(100% - 160px)', - overflowWrap: 'break-word', - [t.breakpoints.down('xs')]: { - maxWidth: 'calc(100% - 40px)', - }, - }, +const useStyles = M.makeStyles(() => ({ name: { wordBreak: 'break-all', }, - spacer: { - flexGrow: 1, - }, - button: { - flexShrink: 0, - marginBottom: -3, - marginTop: -3, - }, - warning: { - background: t.palette.warning.light, - borderRadius: t.shape.borderRadius, - display: 'flex', - padding: t.spacing(1.5), - ...t.typography.body2, - }, - warningIcon: { - height: 20, - lineHeight: '20px', - marginRight: t.spacing(1), - opacity: 0.6, - }, })) export default function PackageTree({ @@ -480,12 +534,7 @@ export default function PackageTree({ }, }) { const classes = useStyles() - const s3 = AWS.S3.use() const { urls } = NamedRoutes.use() - const { apiGatewayEndpoint: endpoint, noDownload } = Config.use() - const t = M.useTheme() - const xs = M.useMediaQuery(t.breakpoints.down('xs')) - const credentials = AWS.Credentials.use() const bucketCfg = BucketConfig.useCurrentBucketConfig() const path = s3paths.decode(encodedPath) @@ -509,7 +558,7 @@ export default function PackageTree({ }, [bucket, name, revision, path, urls]) return ( - + {!!bucketCfg && } @@ -518,53 +567,12 @@ export default function PackageTree({ {' @ '} -
-
- {renderCrumbs(crumbs)} -
-
- {!noDownload && !isDir && ( - - {AsyncResult.case({ - Ok: (handle) => - AWS.Signer.withDownloadUrl(handle, (url) => - xs ? ( - - arrow_downward - - ) : ( - arrow_downward} - download - > - Download file - - ), - ), - _: () => null, - })} - - )} -
{isDir ? ( - + ) : ( - + )} - + ) } diff --git a/catalog/app/containers/Bucket/Summary.js b/catalog/app/containers/Bucket/Summary.js index fe7ecc39686..fab94449e2d 100644 --- a/catalog/app/containers/Bucket/Summary.js +++ b/catalog/app/containers/Bucket/Summary.js @@ -51,16 +51,18 @@ function HandleResolver({ resolve, handle, children }) { return children(AsyncResult.Ok(handle)) } +const renderContents = (contents) => {contents} + function SummaryItemFile({ handle, name, mkUrl, resolveLogicalKey }) { - const renderErr = (_, { fetch }) => ( - <> - - Error loading preview - - - Retry - - + const withData = (callback) => ( + + {AsyncResult.case({ + Err: (e, { fetch }) => + Preview.PreviewError.Unexpected({ handle, retry: fetch, originalError: e }), + Ok: (resolved) => Preview.load(resolved, callback), + _: callback, + })} + ) return ( @@ -70,71 +72,7 @@ function SummaryItemFile({ handle, name, mkUrl, resolveLogicalKey }) { {name || basename(handle.logicalKey || handle.key)} - - - {AsyncResult.case({ - Err: renderErr, - _: () => , - Ok: (resolved) => - Preview.load( - resolved, - AsyncResult.case({ - Ok: AsyncResult.case({ - Init: (_, { fetch }) => ( - <> - - Large files are not previewed automatically - - - Load preview - - - ), - Pending: () => , - Err: renderErr, - Ok: (data) => {Preview.render(data)}, - }), - Err: Preview.PreviewError.case({ - TooLarge: () => ( - <> - - Object is too large to preview in browser - - {AWS.Signer.withDownloadUrl(resolved, (url) => ( - - View in Browser - - ))} - - ), - Unsupported: () => ( - <> - - Preview not available - - {AWS.Signer.withDownloadUrl(resolved, (url) => ( - - View in Browser - - ))} - - ), - DoesNotExist: () => ( - Object does not exist - ), - MalformedJson: ({ originalError: { message } }) => ( - - Malformed JSON: {message} - - ), - Unexpected: renderErr, - }), - _: () => , - }), - ), - })} - - + {withData(Preview.display({ renderContents }))} ) } diff --git a/catalog/app/containers/Bucket/renderPreview.js b/catalog/app/containers/Bucket/renderPreview.js new file mode 100644 index 00000000000..fe18994fb5d --- /dev/null +++ b/catalog/app/containers/Bucket/renderPreview.js @@ -0,0 +1,33 @@ +import * as React from 'react' +import * as M from '@material-ui/core' + +import * as Preview from 'components/Preview' + +const Message = M.styled('div')({ + textAlign: 'center', + width: '100%', +}) + +const renderMessage = ({ heading, body, action }) => ( + + {!!heading && ( + + {heading} + + )} + {!!body && ( + + {body} + + )} + {!!action && action} + +) + +const renderProgress = () => ( + + + +) + +export default Preview.display({ renderMessage, renderProgress }) diff --git a/catalog/app/containers/Bucket/requests.js b/catalog/app/containers/Bucket/requests.js index 8eded9a965b..4f6e64f7901 100644 --- a/catalog/app/containers/Bucket/requests.js +++ b/catalog/app/containers/Bucket/requests.js @@ -239,6 +239,7 @@ export async function getObjectExistence({ s3, bucket, key, version }) { key, version: h.VersionId, deleted: !!h.DeleteMarker, + archived: h.StorageClass === 'GLACIER' || h.StorageClass === 'DEEP_ARCHIVE', }) } catch (e) { if (e.code === 405 && version != null) { @@ -270,7 +271,7 @@ export async function getObjectExistence({ s3, bucket, key, version }) { const ensureObjectIsPresent = (...args) => getObjectExistence(...args).then( ObjectExistence.case({ - Exists: ({ deleted, ...h }) => (deleted ? null : h), + Exists: ({ deleted, archived, ...h }) => (deleted || archived ? null : h), _: () => null, }), ) @@ -481,6 +482,7 @@ export const objectVersions = ({ s3, bucket, path }) => size: v.Size, id: v.VersionId, deleteMarker: v.Size == null, + archived: v.StorageClass === 'GLACIER' || v.StorageClass === 'DEEP_ARCHIVE', })), R.sort(R.descend(R.prop('lastModified'))), ), diff --git a/catalog/app/embed/AppBar.js b/catalog/app/embed/AppBar.js index ffd271e88d2..71e24781514 100644 --- a/catalog/app/embed/AppBar.js +++ b/catalog/app/embed/AppBar.js @@ -1,5 +1,7 @@ +import { basename } from 'path' + import * as React from 'react' -import { useHistory, Link } from 'react-router-dom' +import { useHistory, useRouteMatch, Link } from 'react-router-dom' import * as M from '@material-ui/core' import { fade } from '@material-ui/core/styles/colorManipulator' @@ -7,6 +9,8 @@ import * as NamedRoutes from 'utils/NamedRoutes' import parse from 'utils/parseSearch' import { useRoute } from 'utils/router' +import * as EmbedConfig from './EmbedConfig' + const useSearchBoxStyles = M.makeStyles((t) => ({ root: { background: fade(t.palette.common.white, 0.1), @@ -111,15 +115,27 @@ const useStyles = M.makeStyles((t) => ({ appBar: { zIndex: t.zIndex.appBar + 1, }, - link: { - ...t.typography.body1, + btn: { + color: t.palette.common.white, + backgroundColor: fade(t.palette.common.white, 0.1), + '&:hover': { + backgroundColor: fade(t.palette.common.white, 0.2), + // Reset on touch devices, it doesn't add specificity + '@media (hover: none)': { + backgroundColor: fade(t.palette.common.white, 0.1), + }, + }, }, })) export default function AppBar({ bucket }) { + const cfg = EmbedConfig.use() const trigger = M.useScrollTrigger() const classes = useStyles() - const { urls } = NamedRoutes.use() + const { urls, paths } = NamedRoutes.use() + const isSearch = !!useRouteMatch(paths.bucketSearch) + const rootUrl = urls.bucketDir(bucket, cfg.scope) + const showRootLink = !cfg.hideRootLink || isSearch return ( <> @@ -127,15 +143,33 @@ export default function AppBar({ bucket }) { - - s3://{bucket} - + {showRootLink && ( + + {cfg.hideRootLink && isSearch ? ( // eslint-disable-line no-nested-ternary + <> + + arrow_back_ios + {' '} + Back + + ) : cfg.scope ? ( + basename(cfg.scope) + ) : ( + `s3://${bucket}` + )} + + )} diff --git a/catalog/app/embed/Dir.js b/catalog/app/embed/Dir.js index e4b1beb82e9..a8371615b71 100644 --- a/catalog/app/embed/Dir.js +++ b/catalog/app/embed/Dir.js @@ -5,15 +5,13 @@ import * as R from 'ramda' import * as React from 'react' import * as M from '@material-ui/core' -import { Crumb, copyWithoutSpaces, render as renderCrumbs } from 'components/BreadCrumbs' +import { copyWithoutSpaces, render as renderCrumbs } from 'components/BreadCrumbs' import Message from 'components/Message' -import { docs } from 'constants/urls' import AsyncResult from 'utils/AsyncResult' import * as AWS from 'utils/AWS' import { useData } from 'utils/Data' import * as NamedRoutes from 'utils/NamedRoutes' -import Link from 'utils/StyledLink' -import { getBreadCrumbs, ensureNoSlash, withoutPrefix, up, decode } from 'utils/s3paths' +import * as s3paths from 'utils/s3paths' import usePrevious from 'utils/usePrevious' import Code from 'containers/Bucket/Code' @@ -23,25 +21,12 @@ import { displayError } from 'containers/Bucket/errors' import * as requests from 'containers/Bucket/requests' import * as EmbedConfig from './EmbedConfig' +import getCrumbs from './getCrumbs' -const HELP_LINK = `${docs}/walkthrough/working-with-a-bucket` - -const getCrumbs = R.compose( - R.intersperse(Crumb.Sep(<> / )), - ({ bucket, path, urls }) => - [{ label: 'ROOT', path: '' }, ...getBreadCrumbs(path)].map( - ({ label, path: segPath }) => - Crumb.Segment({ - label, - to: segPath === path ? undefined : urls.bucketDir(bucket, segPath), - }), - ), -) - -const formatListing = ({ urls }, r) => { +const formatListing = ({ urls, scope }, r) => { const dirs = r.dirs.map((name) => ListingItem.Dir({ - name: ensureNoSlash(withoutPrefix(r.path, name)), + name: s3paths.ensureNoSlash(s3paths.withoutPrefix(r.path, name)), to: urls.bucketDir(r.bucket, name), }), ) @@ -53,18 +38,15 @@ const formatListing = ({ urls }, r) => { modified, }), ) - const items = [ - ...(r.path !== '' - ? [ - ListingItem.Dir({ - name: '..', - to: urls.bucketDir(r.bucket, up(r.path)), - }), - ] - : []), - ...dirs, - ...files, - ] + const items = [...dirs, ...files] + if (r.path !== '' && r.path !== scope) { + items.unshift( + ListingItem.Dir({ + name: '..', + to: urls.bucketDir(r.bucket, s3paths.up(r.path)), + }), + ) + } // filter-out files with same name as one of dirs return R.uniqBy(ListingItem.case({ Dir: R.prop('name'), File: R.prop('name') }), items) } @@ -86,7 +68,7 @@ export default function Dir({ const classes = useStyles() const { urls } = NamedRoutes.use() const s3 = AWS.S3.use() - const path = decode(encodedPath) + const path = s3paths.decode(encodedPath) const dest = path ? basename(path) : bucket const code = React.useMemo( @@ -144,7 +126,7 @@ export default function Dir({
- {renderCrumbs(getCrumbs({ bucket, path, urls }))} + {renderCrumbs(getCrumbs({ bucket, path, urls, scope: cfg.scope }))}
@@ -169,15 +151,11 @@ export default function Dir({ if (!res) return - const items = formatListing({ urls }, res) + const items = formatListing({ urls, scope: cfg.scope }, res) - if (!items.length) - return ( - // TODO: remove, just show "no files" - - Learn how to upload files. - - ) + if (!items.length) { + return + } const locked = !AsyncResult.Ok.is(x) diff --git a/catalog/app/embed/Embed.js b/catalog/app/embed/Embed.js index 9f924829473..27fb5b5da3f 100644 --- a/catalog/app/embed/Embed.js +++ b/catalog/app/embed/Embed.js @@ -29,6 +29,7 @@ import defer from 'utils/defer' import { ErrorDisplay } from 'utils/error' import * as RT from 'utils/reactTools' import RouterProvider from 'utils/router' +import * as s3paths from 'utils/s3paths' import useConstant from 'utils/useConstant' import useMemoEq from 'utils/useMemoEq' import usePrevious from 'utils/usePrevious' @@ -143,6 +144,10 @@ function useInit() { const { type, ...init } = data try { if (!init.bucket) throw new Error('missing .bucket') + if (init.scope) { + if (typeof init.scope !== 'string') throw new Error('.scope must be a string') + init.scope = s3paths.ensureSlash(init.scope) + } setState(init) } catch (e) { console.error(`Configuration error: ${e.message}`) diff --git a/catalog/app/embed/File.js b/catalog/app/embed/File.js index 4926df12605..029ea043e38 100644 --- a/catalog/app/embed/File.js +++ b/catalog/app/embed/File.js @@ -8,36 +8,31 @@ import { FormattedRelative } from 'react-intl' import { Link } from 'react-router-dom' import * as M from '@material-ui/core' -import { Crumb, copyWithoutSpaces, render as renderCrumbs } from 'components/BreadCrumbs' +import { copyWithoutSpaces, render as renderCrumbs } from 'components/BreadCrumbs' import Message from 'components/Message' +import * as Preview from 'components/Preview' import Sparkline from 'components/Sparkline' import * as Notifications from 'containers/Notifications' import * as AWS from 'utils/AWS' +import AsyncResult from 'utils/AsyncResult' import * as Config from 'utils/Config' import { useData } from 'utils/Data' import * as NamedRoutes from 'utils/NamedRoutes' import * as SVG from 'utils/SVG' -import StyledLink, { linkStyle } from 'utils/StyledLink' +import { linkStyle } from 'utils/StyledLink' import copyToClipboard from 'utils/clipboard' import parseSearch from 'utils/parseSearch' -import { getBreadCrumbs, up, decode, handleToHttpsUri } from 'utils/s3paths' +import * as s3paths from 'utils/s3paths' import { readableBytes, readableQuantity } from 'utils/string' import Code from 'containers/Bucket/Code' -import FilePreview from 'containers/Bucket/FilePreview' +import * as FileView from 'containers/Bucket/FileView' import Section from 'containers/Bucket/Section' +import renderPreview from 'containers/Bucket/renderPreview' import * as requests from 'containers/Bucket/requests' import * as EmbedConfig from './EmbedConfig' - -const getCrumbs = ({ bucket, path, urls }) => - R.chain( - ({ label, path: segPath }) => [ - Crumb.Segment({ label, to: urls.bucketDir(bucket, segPath) }), - Crumb.Sep(<> / ), - ], - [{ label: 'ROOT', path: '' }, ...getBreadCrumbs(up(path))], - ) +import getCrumbs from './getCrumbs' const useVersionInfoStyles = M.makeStyles(({ typography }) => ({ version: { @@ -68,7 +63,8 @@ function VersionInfo({ bucket, path, version }) { const classes = useVersionInfoStyles() - const getHttpsUri = (v) => handleToHttpsUri({ bucket, key: path, version: v.id }) + const getHttpsUri = (v) => + s3paths.handleToHttpsUri({ bucket, key: path, version: v.id }) const getCliArgs = (v) => `--bucket ${bucket} --key "${path}" --version-id ${v.id}` const copyHttpsUri = (v) => (e) => { @@ -135,6 +131,7 @@ function VersionInfo({ bucket, path, version }) { {!cfg.noDownload && ( {!v.deleteMarker && + !v.archived && AWS.Signer.withDownloadUrl( { bucket, key: path, version: v.id }, (url) => ( @@ -202,31 +199,10 @@ function VersionInfo({ bucket, path, version }) { ) } -const AnnotationsBox = M.styled('div')(({ theme: t }) => ({ - background: M.colors.lightBlue[50], - border: [[1, 'solid', M.colors.lightBlue[400]]], - borderRadius: t.shape.borderRadius, - fontFamily: t.typography.monospace.fontFamily, - fontSize: t.typography.body2.fontSize, - overflow: 'auto', - padding: t.spacing(1), - whiteSpace: 'pre', - width: '100%', -})) - -function Annotations({ bucket, path, version }) { +function Meta({ bucket, path, version }) { const s3 = AWS.S3.use() const data = useData(requests.objectMeta, { s3, bucket, path, version }) - return data.case({ - Ok: (meta) => - !!meta && - !R.isEmpty(meta) && ( -
- {JSON.stringify(meta, null, 2)} -
- ), - _: () => null, - }) + return } function Analytics({ analyticsBucket, bucket, path }) { @@ -349,11 +325,9 @@ export default function File({ const classes = useStyles() const { urls } = NamedRoutes.use() const { analyticsBucket, noDownload } = Config.use() - const t = M.useTheme() - const xs = M.useMediaQuery(t.breakpoints.down('xs')) const s3 = AWS.S3.use() - const path = decode(encodedPath) + const path = s3paths.decode(encodedPath) const code = React.useMemo( () => [ @@ -399,14 +373,33 @@ export default function File({ _: () => false, Ok: requests.ObjectExistence.case({ _: () => false, - Exists: ({ deleted }) => !deleted, + Exists: ({ deleted, archived }) => !deleted && !archived, }), }) + const handle = { bucket, key: path, version } + + const withPreview = (callback) => + requests.ObjectExistence.case({ + Exists: (h) => { + if (h.deleted) { + return callback(AsyncResult.Err(Preview.PreviewError.Deleted({ handle }))) + } + if (h.archived) { + return callback(AsyncResult.Err(Preview.PreviewError.Archived({ handle }))) + } + return Preview.load(handle, callback) + }, + DoesNotExist: () => + callback(AsyncResult.Err(Preview.PreviewError.InvalidVersion({ handle }))), + }) + return ( - +
- {renderCrumbs(getCrumbs({ bucket, path, urls }))} + {renderCrumbs( + getCrumbs({ bucket, path, urls, scope: cfg.scope, excludeBase: true }), + )}
@@ -423,31 +416,7 @@ export default function File({ )}
- {downloadable && - AWS.Signer.withDownloadUrl({ bucket, key: path, version }, (url) => - xs ? ( - - arrow_downward - - ) : ( - arrow_downward} - download - > - Download file - - ), - )} + {downloadable && }
{objExistsData.case({ _: () => , @@ -459,6 +428,7 @@ export default function File({ ) } + // TODO: handle this more gracefully throw e }, Ok: requests.ObjectExistence.case({ @@ -474,45 +444,15 @@ export default function File({ Err: (e) => { throw e }, - Ok: requests.ObjectExistence.case({ - Exists: (h) => - !h.deleted ? ( - - ) : ( - - - DELETE MARKER - - - Selected version of the object is a{' '} - - delete marker - - - - ), - DoesNotExist: () => ( - - - INVALID VERSION - - - Invalid version id specified - - - ), - }), + Ok: withPreview(renderPreview), })} - + ), _: () => , }), })} - + ) } diff --git a/catalog/app/embed/customization-example.json b/catalog/app/embed/customization-example.json index b5af32c1589..c968745ce4b 100644 --- a/catalog/app/embed/customization-example.json +++ b/catalog/app/embed/customization-example.json @@ -9,12 +9,19 @@ } }, "typography": { - "fontFamily": "'Open Sans', monospace" + "fontFamily": "'Open Sans', sans-serif" + }, + "overrides": { + "MuiContainer": { + "root": { + "paddingLeft": "8px !important", + "paddingRight": "8px !important" + } + } } }, - "css": [ - "https://fonts.googleapis.com/css2?family=Open+Sans" - ], + "css": ["https://fonts.googleapis.com/css2?family=Open+Sans"], "hideCode": true, - "hideAnalytics": true + "hideAnalytics": true, + "hideRootLink": true } diff --git a/catalog/app/embed/debug-harness.js b/catalog/app/embed/debug-harness.js index 4e9977003be..c5c645e9d10 100644 --- a/catalog/app/embed/debug-harness.js +++ b/catalog/app/embed/debug-harness.js @@ -47,6 +47,7 @@ function Embedder() { credentials: useField('{}'), bucket: useField(''), path: useField(''), + scope: useField(''), rest: useField('{}'), } @@ -55,8 +56,9 @@ function Embedder() { return { bucket: fields.bucket.value, path: fields.path.value, + scope: fields.scope.value, credentials: JSON.parse(fields.credentials.value), - ...JSON.parse(fields.rest.value), + ...JSON.parse(fields.rest.value || '{}'), } } catch (e) { return e @@ -65,6 +67,7 @@ function Embedder() { fields.credentials.value, fields.bucket.value, fields.path.value, + fields.scope.value, fields.rest.value, ]) @@ -124,6 +127,9 @@ function Embedder() { + + + { + const scoped = scope && path.startsWith(scope) + const scopedPath = scoped ? path.substring(scope.length) : path + const root = { label: scoped ? basename(scope) : 'ROOT', path: '' } + const start = excludeBase ? s3paths.up(scopedPath) : scopedPath + const items = [root, ...s3paths.getBreadCrumbs(start)].map(({ label, path: segPath }) => + Crumb.Segment({ + label, + to: + segPath === scopedPath + ? undefined + : urls.bucketDir(bucket, `${scoped ? scope : ''}${segPath}`), + }), + ) + const interspersed = R.intersperse(Crumb.Sep(<> / ), items) + return excludeBase ? [...interspersed, Crumb.Sep(<> /)] : interspersed +} diff --git a/catalog/app/utils/Data.js b/catalog/app/utils/Data.js index 779d1e0b689..fa5f2d55194 100644 --- a/catalog/app/utils/Data.js +++ b/catalog/app/utils/Data.js @@ -30,7 +30,7 @@ export function useData(request, params, { noAutoFetch = false } = {}) { // TODO: accept custom key extraction fn (params => key for comparison) const [state, dispatch] = React.useReducer(reducer, initial) - const fetch = React.useCallback(() => { + const fetch = useMemoEq([request, params], () => () => { dispatch(Action.Request({ request, params })) return request(params) .then(AsyncResult.Ok) @@ -39,9 +39,7 @@ export function useData(request, params, { noAutoFetch = false } = {}) { dispatch(Action.Response({ request, params, result })) return result }) - }, [request, params]) - // FIXME: probably memoization doesnt work here bc params is an object and it - // gets constructed anew every time on the caller side + }) usePrevious({ params, noAutoFetch }, (prev) => { if (R.equals({ params, noAutoFetch }, prev)) return @@ -51,12 +49,11 @@ export function useData(request, params, { noAutoFetch = false } = {}) { const result = useMemoEq(state, mapResult) - const doCase = React.useMemo( - () => (cases, ...args) => AsyncResult.case(cases, result, ...args), - [result], + const doCase = useMemoEq([result], () => (cases, ...args) => + AsyncResult.case(cases, result, ...args), ) - return { result, fetch, case: doCase } + return useMemoEq({ result, fetch, case: doCase }, R.identity) } export const use = useData diff --git a/catalog/app/utils/pipeThru.js b/catalog/app/utils/pipeThru.js new file mode 100644 index 00000000000..a8f3b1e8065 --- /dev/null +++ b/catalog/app/utils/pipeThru.js @@ -0,0 +1,3 @@ +import * as R from 'ramda' + +export default (...args) => (...fns) => R.pipe(...fns)(...args) diff --git a/catalog/app/utils/s3paths.js b/catalog/app/utils/s3paths.js index f8698871749..83cbec441b9 100644 --- a/catalog/app/utils/s3paths.js +++ b/catalog/app/utils/s3paths.js @@ -45,6 +45,10 @@ export const ensureSlash = (str) => `${ensureNoSlash(str)}/` * @returns {string} */ export const up = (prefix) => { + // handle double slashes + if (prefix.endsWith('//')) return prefix.substring(0, prefix.length - 1) + const m = prefix.match(/^(.*\/\/)[^/]+$/) + if (m) return m[1] const d = dirname(prefix) return d === '.' || d === '/' ? '' : ensureSlash(d) } @@ -183,7 +187,12 @@ export const handleToHttpsUri = ({ bucket, key, version }) => * @returns {[{ label: string, path: string }]} */ export const getBreadCrumbs = (path) => - path ? [...getBreadCrumbs(up(path)), { label: basename(path), path }] : [] + path + ? [ + ...getBreadCrumbs(up(path)), + { label: path.endsWith('//') ? '' : basename(path), path }, + ] + : [] export const encode = R.pipe(R.split('/'), R.map(encodeURIComponent), R.join('/')) diff --git a/catalog/app/utils/search.js b/catalog/app/utils/search.js index ac168bfc2c1..99f8300cef4 100644 --- a/catalog/app/utils/search.js +++ b/catalog/app/utils/search.js @@ -123,6 +123,8 @@ const mergeAllHits = R.pipe( R.sortBy((h) => -h.score), ) +const unescape = (s) => s.replace(/\\n/g, '\n') + export default async function search({ req, query, @@ -156,10 +158,10 @@ export default async function search({ return { total, hits } } catch (e) { const match = e.message.match( - /^API Gateway Error: RequestError\(400, 'search_phase_execution_exception', 'token_mgr_error: (.+)'\)$/, + /^API Gateway Error: RequestError\(400, 'search_phase_execution_exception', '(.+)'\)$/, ) if (match) { - throw new BaseError('SearchSyntaxError', { details: match[1] }) + throw new BaseError('SearchSyntaxError', { details: unescape(match[1]) }) } console.log('Search error:', e.message) console.error(e)