@@ -465,6 +469,9 @@ function Toolbar({
const usePanelStyles = M.makeStyles((t) => ({
root: {
zIndex: 1,
+ '& select, & input': {
+ boxSizing: 'content-box',
+ },
},
paper: {
backgroundColor: t.palette.background.paper,
@@ -504,6 +511,7 @@ function Panel({ children, open }: DG.GridPanelProps) {
anchorEl={anchorEl}
modifiers={getPopperModifiers()}
className={classes.root}
+ disablePortal
>
@@ -717,6 +725,16 @@ function FilteredOverlay() {
)
}
+export type CellProps = React.PropsWithChildren<{
+ item: Item
+ title?: string
+ className?: string
+}>
+
+function Cell({ item, ...props }: CellProps) {
+ return
+}
+
function compareBy(a: T, b: T, getValue: (arg: T) => V) {
const va = getValue(a)
const vb = getValue(b)
@@ -760,7 +778,7 @@ const COL_MODIFIED_W = 176
const useStyles = M.makeStyles((t) => ({
'@global': {
'.MuiDataGridMenu-root': {
- zIndex: 1,
+ zIndex: t.zIndex.modal + 1, // show it over modals
},
},
root: {
@@ -774,8 +792,15 @@ const useStyles = M.makeStyles((t) => ({
background: fade(t.palette.background.paper, 0.5),
zIndex: 1,
},
+ '& .MuiDataGrid-checkboxInput': {
+ padding: 7,
+ '& svg': {
+ fontSize: 18,
+ },
+ },
'& .MuiDataGrid-cell': {
border: 'none',
+ outline: 'none !important',
padding: 0,
},
'& .MuiDataGrid-colCell': {
@@ -801,11 +826,11 @@ const useStyles = M.makeStyles((t) => ({
pointerEvents: 'none',
},
// "Size" column
- '&:nth-child(2)': {
+ '&:nth-last-child(2)': {
justifyContent: 'flex-end',
},
// "Last modified" column
- '&:nth-child(3)': {
+ '&:last-child': {
justifyContent: 'flex-end',
'& .MuiDataGrid-colCellTitleContainer': {
order: 1,
@@ -842,6 +867,10 @@ const useStyles = M.makeStyles((t) => ({
alignItems: 'center',
display: 'flex',
},
+ ellipsis: {
+ overflow: 'hidden',
+ textOverflow: 'ellipsis',
+ },
icon: {
fontSize: t.typography.body1.fontSize,
marginRight: t.spacing(0.5),
@@ -858,6 +887,12 @@ interface ListingProps {
prefixFilter?: string
toolbarContents?: React.ReactNode
loadMore?: () => void
+ selection?: DG.GridRowId[]
+ onSelectionChange?: (newSelection: DG.GridRowId[]) => void
+ CellComponent?: React.ComponentType
+ RootComponent?: React.ElementType<{ className: string }>
+ className?: string
+ dataGridProps?: Partial
}
export function Listing({
@@ -867,6 +902,12 @@ export function Listing({
toolbarContents,
prefixFilter,
loadMore,
+ selection,
+ onSelectionChange,
+ CellComponent = Cell,
+ RootComponent = M.Paper,
+ className,
+ dataGridProps,
}: ListingProps) {
const classes = useStyles()
@@ -879,6 +920,32 @@ export function Listing({
[setFilteredToZero],
)
+ const [page, setPage] = React.useState(0)
+ const [pageSize, setPageSize] = React.useState(25)
+
+ const handlePageChange = React.useCallback(
+ ({ page: newPage }: DG.GridPageChangeParams) => {
+ setPage(newPage)
+ },
+ [],
+ )
+
+ const handlePageSizeChange = React.useCallback(
+ ({ pageSize: newPageSize }: DG.GridPageChangeParams) => {
+ setPageSize(newPageSize)
+ },
+ [],
+ )
+
+ usePrevious(items, (prevItems?: Item[]) => {
+ if (!prevItems) return
+ const itemsOnPrevPages = page * pageSize
+ // reset page if items on previous pages change
+ if (!R.equals(R.take(itemsOnPrevPages, items), R.take(itemsOnPrevPages, prevItems))) {
+ setPage(0)
+ }
+ })
+
const columns: DG.GridColumns = React.useMemo(
() => [
{
@@ -904,20 +971,20 @@ export function Listing({
renderCell: (params: DG.GridCellParams) => {
const i = (params.row as unknown) as Item
return (
-
{i.type === 'file' ? 'insert_drive_file' : 'folder_open'}
- {i.name || EMPTY}
-
+ {i.name || EMPTY}
+
)
},
},
@@ -938,13 +1005,13 @@ export function Listing({
renderCell: (params: DG.GridCellParams) => {
const i = (params.row as unknown) as Item
return (
-
{i.size == null ? <> : readableBytes(i.size)}
-
+
)
},
},
@@ -957,18 +1024,18 @@ export function Listing({
renderCell: (params: DG.GridCellParams) => {
const i = (params.row as unknown) as Item
return (
-
{i.modified == null ? <> : i.modified.toLocaleString()}
-
+
)
},
},
],
- [classes],
+ [classes, CellComponent],
)
const noRowsLabel = `No files / directories${
@@ -978,9 +1045,16 @@ export function Listing({
// abuse loading overlay to show warning when all the items are filtered-out
const LoadingOverlay = !locked && filteredToZero ? FilteredOverlay : undefined
+ const handleSelectionModelChange = React.useCallback(
+ (newSelection: DG.GridSelectionModelChangeParams) => {
+ if (onSelectionChange) onSelectionChange(newSelection.selectionModel)
+ },
+ [onSelectionChange],
+ )
+
// TODO: control page, pageSize, filtering and sorting via props
return (
-
+
row.name}
pagination
- pageSize={25}
- // page={1}
- // onPageChange={({ page }) => set page}
- // onPageSizeChange={({ pageSize }) => set page size}
+ pageSize={pageSize}
+ onPageSizeChange={handlePageSizeChange}
+ page={page}
+ onPageChange={handlePageChange}
loading={locked || filteredToZero}
headerHeight={36}
rowHeight={36}
@@ -1008,8 +1082,13 @@ export function Listing({
disableMultipleSelection
disableMultipleColumnsSorting
localeText={{ noRowsLabel, ...localeText }}
+ // selection-related props
+ checkboxSelection={!!onSelectionChange}
+ selectionModel={selection}
+ onSelectionModelChange={handleSelectionModelChange}
+ {...dataGridProps}
/>
-
+
)
}
diff --git a/catalog/app/containers/Bucket/PackageCreateDialog.js b/catalog/app/containers/Bucket/PackageCreateDialog.js
index a7a062e39df..084d256ce87 100644
--- a/catalog/app/containers/Bucket/PackageCreateDialog.js
+++ b/catalog/app/containers/Bucket/PackageCreateDialog.js
@@ -184,13 +184,15 @@ function FilesInput({
value,
])
- const warn = totalSize > PD.MAX_SIZE
+ const warn = totalSize > PD.MAX_UPLOAD_SIZE
// eslint-disable-next-line no-nested-ternary
const label = error ? (
errors[error] || error
) : warn ? (
- <>Total file size exceeds recommended maximum of {readableBytes(PD.MAX_SIZE)}
+ <>
+ Total file size exceeds recommended maximum of {readableBytes(PD.MAX_UPLOAD_SIZE)}
+
) : (
'Drop files here or click to browse'
)
diff --git a/catalog/app/containers/Bucket/PackageDialog/FilesInput.tsx b/catalog/app/containers/Bucket/PackageDialog/FilesInput.tsx
index a1df5404f26..f270cf63ed7 100644
--- a/catalog/app/containers/Bucket/PackageDialog/FilesInput.tsx
+++ b/catalog/app/containers/Bucket/PackageDialog/FilesInput.tsx
@@ -6,11 +6,13 @@ import * as M from '@material-ui/core'
import { fade } from '@material-ui/core/styles'
import useDragging from 'utils/dragging'
+import { withoutPrefix } from 'utils/s3paths'
import { readableBytes } from 'utils/string'
import * as tagged from 'utils/taggedV2'
import useMemoEq from 'utils/useMemoEq'
import * as PD from './PackageDialog'
+import * as S3FilePicker from './S3FilePicker'
const COLORS = {
default: M.colors.grey[900],
@@ -50,6 +52,10 @@ function computeHash(f: F) {
return fh
}
+function ensureExhaustive(x: never): never {
+ throw new Error(`Non-exhaustive match: '${x}'`)
+}
+
export const FilesAction = tagged.create(
'app/containers/Bucket/PackageDialog/FilesInput:FilesAction' as const,
{
@@ -57,6 +63,11 @@ export const FilesAction = tagged.create(
...v,
files: v.files.map(computeHash),
}),
+ AddFromS3: (v: {
+ files: S3FilePicker.S3File[]
+ basePrefix: string
+ prefix?: string
+ }) => v,
Delete: (path: string) => path,
DeleteDir: (prefix: string) => prefix,
Revert: (path: string) => path,
@@ -76,8 +87,10 @@ export interface ExistingFile {
size: number
}
+export type LocalFile = FileWithPath & FileWithHash
+
export interface FilesState {
- added: Record
+ added: Record
deleted: Record
existing: Record
// XXX: workaround used to re-trigger validation and dependent computations
@@ -112,6 +125,17 @@ const handleFilesAction = FilesAction.match<
acc,
) as FilesState
}, state),
+ AddFromS3: ({ files, basePrefix, prefix }) => (state) =>
+ files.reduce((acc, file) => {
+ const path = (prefix || '') + withoutPrefix(basePrefix, file.key)
+ return R.evolve(
+ {
+ added: R.assoc(path, file),
+ deleted: R.dissoc(path),
+ },
+ acc,
+ ) as FilesState
+ }, state),
Delete: (path) =>
R.evolve({
added: R.dissoc(path),
@@ -147,6 +171,8 @@ interface DispatchFilesAction {
type FilesEntryState = 'deleted' | 'modified' | 'unchanged' | 'hashing' | 'added'
+type FilesEntryType = 's3' | 'local'
+
const FilesEntryTag = 'app/containers/Bucket/PackageDialog/FilesInput:FilesEntry' as const
const FilesEntry = tagged.create(FilesEntryTag, {
@@ -155,7 +181,12 @@ const FilesEntry = tagged.create(FilesEntryTag, {
state: FilesEntryState
childEntries: tagged.Instance[]
}) => v,
- File: (v: { name: string; state: FilesEntryState; size: number }) => v,
+ File: (v: {
+ name: string
+ state: FilesEntryState
+ type: FilesEntryType
+ size: number
+ }) => v,
})
// eslint-disable-next-line @typescript-eslint/no-redeclare
@@ -203,30 +234,42 @@ const insertIntoTree = (path: string[] = [], file: FilesEntry, entries: FilesEnt
interface IntermediateEntry {
state: FilesEntryState
+ type: FilesEntryType
path: string
size: number
}
const computeEntries = ({ added, deleted, existing }: FilesState) => {
- const existingEntries = Object.entries(existing).map(([path, { size, hash }]) => {
- if (path in deleted) {
- return { state: 'deleted' as const, path, size }
- }
- if (path in added) {
- const a = added[path]
- // eslint-disable-next-line no-nested-ternary
- const state = !a.hash.ready
- ? ('hashing' as const)
- : a.hash.value === hash
- ? ('unchanged' as const)
- : ('modified' as const)
- return { state, path, size: a.size }
- }
- return { state: 'unchanged' as const, path, size }
- })
- const addedEntries = Object.entries(added).reduce((acc, [path, { size }]) => {
+ const existingEntries: IntermediateEntry[] = Object.entries(existing).map(
+ ([path, { size, hash }]) => {
+ if (path in deleted) {
+ return { state: 'deleted' as const, type: 'local' as const, path, size }
+ }
+ if (path in added) {
+ const a = added[path]
+ let state: FilesEntryState
+ let type: FilesEntryType
+ if (S3FilePicker.isS3File(a)) {
+ type = 's3' as const
+ state = 'modified' as const
+ } else {
+ type = 'local' as const
+ // eslint-disable-next-line no-nested-ternary
+ state = !a.hash.ready
+ ? ('hashing' as const)
+ : a.hash.value === hash
+ ? ('unchanged' as const)
+ : ('modified' as const)
+ }
+ return { state, type, path, size: a.size }
+ }
+ return { state: 'unchanged' as const, type: 'local' as const, path, size }
+ },
+ )
+ const addedEntries = Object.entries(added).reduce((acc, [path, f]) => {
if (path in existing) return acc
- return acc.concat({ state: 'added', path, size })
+ const type = S3FilePicker.isS3File(f) ? ('s3' as const) : ('local' as const)
+ return acc.concat({ state: 'added', type, path, size: f.size })
}, [] as IntermediateEntry[])
const entries: IntermediateEntry[] = [...existingEntries, ...addedEntries]
return entries.reduce((children, { path, ...rest }) => {
@@ -242,7 +285,9 @@ export const HASHING = 'hashing'
export const HASHING_ERROR = 'hashingError'
export const validateHashingComplete = (state: FilesState) => {
- const files = Object.values(state.added)
+ const files = Object.values(state.added).filter(
+ (f) => !S3FilePicker.isS3File(f),
+ ) as FileWithHash[]
if (files.some((f) => f.hash.ready && !f.hash.value)) return HASHING_ERROR
if (files.some((f) => !f.hash.ready)) return HASHING
return undefined
@@ -265,6 +310,19 @@ const useEntryIconStyles = M.makeStyles((t) => ({
fontSize: 18,
padding: 3,
},
+ overlay: {
+ alignItems: 'center',
+ bottom: 0,
+ color: t.palette.background.paper,
+ display: 'flex',
+ fontFamily: t.typography.fontFamily,
+ fontSize: 8,
+ justifyContent: 'center',
+ left: 0,
+ position: 'absolute',
+ right: 0,
+ top: 0,
+ },
stateContainer: {
alignItems: 'center',
background: 'currentColor',
@@ -289,9 +347,12 @@ const useEntryIconStyles = M.makeStyles((t) => ({
},
}))
-type EntryIconProps = React.PropsWithChildren<{ state: FilesEntryState }>
+type EntryIconProps = React.PropsWithChildren<{
+ state: FilesEntryState
+ overlay?: React.ReactNode
+}>
-function EntryIcon({ state, children }: EntryIconProps) {
+function EntryIcon({ state, overlay, children }: EntryIconProps) {
const classes = useEntryIconStyles()
const stateContents =
state &&
@@ -305,6 +366,7 @@ function EntryIcon({ state, children }: EntryIconProps) {
return (
{children}
+ {!!overlay &&
{overlay}
}
{!!stateContents && (
{stateContents === 'hashing' ? (
@@ -378,6 +440,7 @@ const useFileStyles = M.makeStyles((t) => ({
interface FileProps extends React.HTMLAttributes
{
name: string
state?: FilesEntryState
+ type?: FilesEntryType
size?: number
action?: React.ReactNode
interactive?: boolean
@@ -387,6 +450,7 @@ interface FileProps extends React.HTMLAttributes {
function File({
name,
state = 'unchanged',
+ type = 'local',
size,
action,
interactive = false,
@@ -407,7 +471,9 @@ function File({
{...props}
>
-
insert_drive_file
+
+ insert_drive_file
+
{name}
@@ -563,7 +629,7 @@ const Dir = React.forwardRef
(function Dir(
)}
}
+ {warn.s3 && (
+ <>
+ Total size of files from S3 exceeds recommended maximum of{' '}
+ {readableBytes(PD.MAX_S3_SIZE)}.
+
+ )}
)
return 'Drop files here or click to browse'
- }, [error, warn])
+ }, [error, warn.upload, warn.s3])
return (
{label}
@@ -837,6 +914,7 @@ const useContentsStyles = M.makeStyles((t) => ({
minHeight: 80,
outline: 'none',
overflow: 'hidden',
+ position: 'relative',
},
interactive: {
cursor: 'pointer',
@@ -885,41 +963,51 @@ type FileUploadProps = tagged.ValueOf
& {
dispatch: DispatchFilesAction
}
-function FileUpload({ name, state, size, prefix, dispatch }: FileUploadProps) {
+function FileUpload({ name, state, type, size, prefix, dispatch }: FileUploadProps) {
const path = (prefix || '') + name
- const handle = React.useCallback(
- (cons: tagged.ConstructorOf) => (e: React.MouseEvent) => {
+ // eslint-disable-next-line consistent-return
+ const action = React.useMemo(() => {
+ const handle = (a: FilesAction) => (e: React.MouseEvent) => {
// stop click from propagating to the root element and triggering its handler
e.stopPropagation()
- dispatch(cons(path))
- },
- [dispatch, path],
- )
-
- const action = React.useMemo(
- () =>
- ({
- added: { hint: 'Remove', icon: 'clear', handler: handle(FilesAction.Revert) },
- modified: {
+ dispatch(a)
+ }
+ switch (state) {
+ case 'added':
+ return {
+ hint: 'Remove',
+ icon: 'clear',
+ handler: handle(FilesAction.Revert(path)),
+ }
+ case 'modified':
+ return {
hint: 'Revert',
icon: 'undo',
- handler: handle(FilesAction.Revert),
- },
- hashing: {
+ handler: handle(FilesAction.Revert(path)),
+ }
+ case 'hashing':
+ return {
hint: 'Revert',
icon: 'undo',
- handler: handle(FilesAction.Revert),
- },
- deleted: {
+ handler: handle(FilesAction.Revert(path)),
+ }
+ case 'deleted':
+ return {
hint: 'Restore',
icon: 'undo',
- handler: handle(FilesAction.Revert),
- },
- unchanged: { hint: 'Delete', icon: 'clear', handler: handle(FilesAction.Delete) },
- }[state]),
- [state, handle],
- )
+ handler: handle(FilesAction.Revert(path)),
+ }
+ case 'unchanged':
+ return {
+ hint: 'Delete',
+ icon: 'clear',
+ handler: handle(FilesAction.Delete(path)),
+ }
+ default:
+ ensureExhaustive(state)
+ }
+ }, [state, dispatch, path])
const onClick = React.useCallback((e: React.MouseEvent) => {
// stop click from propagating to parent elements and triggering their handlers
@@ -933,6 +1021,7 @@ function FileUpload({ name, state, size, prefix, dispatch }: FileUploadProps) {
role="button"
tabIndex={0}
state={state}
+ type={type}
name={name}
size={size}
action={
@@ -950,7 +1039,7 @@ type DirUploadProps = tagged.ValueOf & {
}
function DirUpload({ name, state, childEntries, prefix, dispatch }: DirUploadProps) {
- const [expanded, setExpanded] = React.useState(true)
+ const [expanded, setExpanded] = React.useState(false)
const toggleExpanded = React.useCallback(
(e) => {
@@ -981,42 +1070,48 @@ function DirUpload({ name, state, childEntries, prefix, dispatch }: DirUploadPro
noClick: true,
})
- const handle = React.useCallback(
- (cons: tagged.ConstructorOf) => (e: React.MouseEvent) => {
+ // eslint-disable-next-line consistent-return
+ const action = React.useMemo(() => {
+ const handle = (a: FilesAction) => (e: React.MouseEvent) => {
// stop click from propagating to the root element and triggering its handler
e.stopPropagation()
- dispatch(cons(path))
- },
- [dispatch, path],
- )
-
- const action = React.useMemo(
- () =>
- ({
- added: { hint: 'Remove', icon: 'clear', handler: handle(FilesAction.RevertDir) },
- modified: {
+ dispatch(a)
+ }
+ switch (state) {
+ case 'added':
+ return {
+ hint: 'Remove',
+ icon: 'clear',
+ handler: handle(FilesAction.RevertDir(path)),
+ }
+ case 'modified':
+ return {
hint: 'Revert',
icon: 'undo',
- handler: handle(FilesAction.RevertDir),
- },
- hashing: {
+ handler: handle(FilesAction.RevertDir(path)),
+ }
+ case 'hashing':
+ return {
hint: 'Revert',
icon: 'undo',
- handler: handle(FilesAction.RevertDir),
- },
- deleted: {
+ handler: handle(FilesAction.RevertDir(path)),
+ }
+ case 'deleted':
+ return {
hint: 'Restore',
icon: 'undo',
- handler: handle(FilesAction.RevertDir),
- },
- unchanged: {
+ handler: handle(FilesAction.RevertDir(path)),
+ }
+ case 'unchanged':
+ return {
hint: 'Delete',
icon: 'clear',
- handler: handle(FilesAction.DeleteDir),
- },
- }[state]),
- [state, handle],
- )
+ handler: handle(FilesAction.DeleteDir(path)),
+ }
+ default:
+ ensureExhaustive(state)
+ }
+ }, [state, dispatch, path])
return (
({
- added: {
- color: t.palette.success.main,
+ hashing: {
marginLeft: t.spacing(1),
},
- deleted: {
- color: t.palette.error.main,
- marginLeft: t.spacing(1),
+ actions: {
+ display: 'flex',
+ marginTop: t.spacing(1),
},
- hashing: {
- marginLeft: t.spacing(1),
+ action: {
+ flexGrow: 1,
+ '& + &': {
+ marginLeft: t.spacing(1),
+ },
},
}))
@@ -1089,6 +1186,7 @@ interface FilesInputProps {
loaded: number
percent: number
}
+ bucket: string
}
export function FilesInput({
@@ -1100,6 +1198,7 @@ export function FilesInput({
onFilesAction,
title,
totalProgress,
+ bucket,
}: FilesInputProps) {
const classes = useFilesInputStyles()
@@ -1138,7 +1237,10 @@ export function FilesInput({
// XXX: maybe observe value and trigger this when it changes,
// regardless of the source of change (e.g. new value supplied directly via the prop)
const waitFor = Object.values(newValue.added).reduce(
- (acc, f) => (f.hash.ready ? acc : acc.concat(f.hash.promise.catch(() => {}))),
+ (acc, f) =>
+ S3FilePicker.isS3File(f) || f.hash.ready
+ ? acc
+ : acc.concat(f.hash.promise.catch(() => {})),
[] as Promise[],
)
cur.scheduleUpdate(waitFor)
@@ -1159,30 +1261,64 @@ export function FilesInput({
}, [dispatch])
const isDragging = useDragging()
- const { getRootProps, getInputProps, isDragActive } = useDropzone({ disabled, onDrop })
+ const { getRootProps, getInputProps, isDragActive, open } = useDropzone({
+ disabled,
+ onDrop,
+ })
const computedEntries = useMemoEq(value, computeEntries)
- const stats = useMemoEq(value, ({ added, deleted, existing }) => ({
- added: Object.entries(added).reduce(
+ const stats = useMemoEq(value, ({ added, existing }) => ({
+ upload: Object.entries(added).reduce(
(acc, [path, f]) => {
+ if (S3FilePicker.isS3File(f)) return acc // dont count s3 files
const e = existing[path]
if (e && (!f.hash.ready || f.hash.value === e.hash)) return acc
return R.evolve({ count: R.inc, size: R.add(f.size) }, acc)
},
{ count: 0, size: 0 },
),
- deleted: Object.keys(deleted).reduce(
- (acc, path) => R.evolve({ count: R.inc, size: R.add(existing[path].size) }, acc),
+ s3: Object.entries(added).reduce(
+ (acc, [, f]) =>
+ S3FilePicker.isS3File(f)
+ ? R.evolve({ count: R.inc, size: R.add(f.size) }, acc)
+ : acc,
{ count: 0, size: 0 },
),
- hashing: Object.values(added).reduce((acc, f) => acc || !f.hash.ready, false),
+ hashing: Object.values(added).reduce(
+ (acc, f) => acc || (!S3FilePicker.isS3File(f) && !f.hash.ready),
+ false,
+ ),
}))
- const warn = stats.added.size > PD.MAX_SIZE
+ const warn = {
+ upload: stats.upload.size > PD.MAX_UPLOAD_SIZE,
+ s3: stats.s3.size > PD.MAX_S3_SIZE,
+ }
+
+ const [s3FilePickerOpen, setS3FilePickerOpen] = React.useState(false)
+
+ const closeS3FilePicker = React.useCallback(
+ (reason: S3FilePicker.CloseReason) => {
+ if (!!reason && typeof reason === 'object') {
+ dispatch(FilesAction.AddFromS3({ files: reason.files, basePrefix: reason.path }))
+ }
+ setS3FilePickerOpen(false)
+ },
+ [dispatch, setS3FilePickerOpen],
+ )
+
+ const handleS3Btn = React.useCallback(() => {
+ setS3FilePickerOpen(true)
+ }, [])
return (
+
)
}
diff --git a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx
index 0a37103cb99..659df240f59 100644
--- a/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx
+++ b/catalog/app/containers/Bucket/PackageDialog/PackageDialog.tsx
@@ -27,7 +27,8 @@ import * as requests from '../requests'
import MetaInputErrorHelper from './MetaInputErrorHelper'
import SelectWorkflow from './SelectWorkflow'
-export const MAX_SIZE = 1000 * 1000 * 1000 // 1GB
+export const MAX_UPLOAD_SIZE = 1000 * 1000 * 1000 // 1GB
+export const MAX_S3_SIZE = 10 * 1000 * 1000 * 1000 // 10GB
export const ES_LAG = 3 * 1000
export const MAX_META_FILE_SIZE = 10 * 1000 * 1000 // 10MB
diff --git a/catalog/app/containers/Bucket/PackageDialog/S3FilePicker.tsx b/catalog/app/containers/Bucket/PackageDialog/S3FilePicker.tsx
new file mode 100644
index 00000000000..2149d074d51
--- /dev/null
+++ b/catalog/app/containers/Bucket/PackageDialog/S3FilePicker.tsx
@@ -0,0 +1,345 @@
+import cx from 'classnames'
+import pLimit from 'p-limit'
+import * as R from 'ramda'
+import * as React from 'react'
+import * as M from '@material-ui/core'
+import * as DG from '@material-ui/data-grid'
+
+import { Crumb, render as renderCrumbs } from 'components/BreadCrumbs'
+import AsyncResult from 'utils/AsyncResult'
+import { useData } from 'utils/Data'
+import { getBreadCrumbs, ensureNoSlash, withoutPrefix } from 'utils/s3paths'
+
+import * as Listing from '../Listing'
+import { displayError } from '../errors'
+import * as requests from '../requests'
+
+import SubmitSpinner from './SubmitSpinner'
+
+const limit = pLimit(5)
+
+export interface S3File {
+ bucket: string
+ key: string
+ version?: string
+ size: number
+}
+
+export const isS3File = (f: any): f is S3File =>
+ !!f &&
+ typeof f === 'object' &&
+ typeof f.bucket === 'string' &&
+ typeof f.key === 'string' &&
+ (typeof f.version === 'string' || typeof f.version === 'undefined') &&
+ typeof f.size === 'number'
+
+const getCrumbs = R.compose(
+ R.intersperse(Crumb.Sep(<> / )),
+ ({ bucket, path }: { bucket: string; path: string }) =>
+ [
+ { label: bucket, path: '' },
+ ...getBreadCrumbs(path),
+ ].map(({ label, path: segPath }) =>
+ Crumb.Segment({ label, to: segPath === path ? undefined : segPath }),
+ ),
+)
+
+type MuiCloseReason = 'backdropClick' | 'escapeKeyDown'
+export type CloseReason = MuiCloseReason | 'cancel' | { path: string; files: S3File[] }
+
+const useStyles = M.makeStyles((t) => ({
+ paper: {
+ height: '100vh',
+ },
+ crumbs: {
+ ...t.typography.body1,
+ marginTop: -t.spacing(1),
+ maxWidth: '100%',
+ overflowWrap: 'break-word',
+ paddingLeft: t.spacing(3),
+ paddingRight: t.spacing(3),
+ },
+ lock: {
+ background: 'rgba(255,255,255,0.5)',
+ bottom: 52,
+ left: 0,
+ position: 'absolute',
+ right: 0,
+ top: 56,
+ zIndex: 1,
+ },
+}))
+
+interface DialogProps {
+ bucket: string
+ open: boolean
+ onClose: (reason: CloseReason) => void
+}
+
+export function Dialog({ bucket, open, onClose }: DialogProps) {
+ const classes = useStyles()
+
+ const bucketListing = requests.useBucketListing()
+
+ const [path, setPath] = React.useState('')
+ const [prefix, setPrefix] = React.useState('')
+ const [prev, setPrev] = React.useState(null)
+ const [selection, setSelection] = React.useState([])
+
+ const [locked, setLocked] = React.useState(false)
+
+ const cancel = React.useCallback(() => onClose('cancel'), [onClose])
+
+ const handleClose = React.useCallback(
+ (_e: {}, reason: MuiCloseReason) => {
+ if (!locked) onClose(reason)
+ },
+ [locked, onClose],
+ )
+
+ const crumbs = React.useMemo(() => getCrumbs({ bucket, path }), [bucket, path])
+
+ const getCrumbLinkProps = ({ to }: { to: string }) => ({
+ onClick: () => {
+ setPath(to)
+ },
+ })
+
+ React.useLayoutEffect(() => {
+ // reset accumulated results when bucket, path and / or prefix change
+ setPrev(null)
+ }, [bucket, path, prefix])
+
+ const data = useData(bucketListing, { bucket, path, prefix, prev, drain: true })
+
+ const loadMore = React.useCallback(() => {
+ AsyncResult.case(
+ {
+ Ok: (res: requests.BucketListingResult) => {
+ // this triggers a re-render and fetching of next page of results
+ if (res.continuationToken) setPrev(res)
+ },
+ _: () => {},
+ },
+ data.result,
+ )
+ }, [data.result])
+
+ const add = React.useCallback(() => {
+ data.case({
+ Ok: async (r: requests.BucketListingResult) => {
+ try {
+ setLocked(true)
+ const dirsByBasename = R.fromPairs(
+ r.dirs.map((name) => [ensureNoSlash(withoutPrefix(r.path, name)), name]),
+ )
+ const filesByBasename = R.fromPairs(
+ r.files.map((f) => [withoutPrefix(r.path, f.key), f]),
+ )
+ const { dirs, files } = selection.reduce(
+ (acc, id) => {
+ const dir = dirsByBasename[id]
+ if (dir) return { ...acc, dirs: [...acc.dirs, dir] }
+ const file = filesByBasename[id]
+ if (file) return { ...acc, files: [...acc.files, file] }
+ return acc // shouldnt happen
+ },
+ { files: [] as requests.BucketListingFile[], dirs: [] as string[] },
+ )
+
+ const dirsPromises = dirs.map((dir) =>
+ limit(bucketListing, { bucket, path: dir, delimiter: false, drain: true }),
+ )
+
+ const dirsChildren = await Promise.all(dirsPromises)
+ const allChildren = dirsChildren.reduce(
+ (acc, res) => acc.concat(res.files),
+ [] as requests.BucketListingFile[],
+ )
+
+ onClose({ files: files.concat(allChildren), path })
+ } finally {
+ setLocked(false)
+ }
+ },
+ _: () => {},
+ })
+ }, [onClose, selection, data, bucket, path, bucketListing])
+
+ const handleExited = React.useCallback(() => {
+ setPath('')
+ setPrefix('')
+ setPrev(null)
+ setSelection([])
+ }, [])
+
+ return (
+
+ Add files from S3
+
+ {renderCrumbs(crumbs, { getLinkProps: getCrumbLinkProps })}
+
+ {data.case({
+ // TODO: customized error display?
+ Err: displayError(),
+ Init: () => null,
+ _: (x: $TSFixMe) => {
+ const res: requests.BucketListingResult | null = AsyncResult.getPrevResult(x)
+ return res ? (
+
+ ) : (
+ // TODO: skeleton
+
+
+
+ )
+ },
+ })}
+ {locked && }
+
+ {locked ? (
+ Adding files
+ ) : (
+
+
+ {selection.length} item{selection.length === 1 ? '' : 's'} selected
+
+
+ )}
+ Cancel
+
+ Add files
+
+
+
+ )
+}
+
+function useFormattedListing(r: requests.BucketListingResult) {
+ return React.useMemo(() => {
+ const dirs = r.dirs.map((name) => ({
+ type: 'dir' as const,
+ name: ensureNoSlash(withoutPrefix(r.path, name)),
+ to: name,
+ }))
+ const files = r.files.map(({ key, size, modified, archived }) => ({
+ type: 'file' as const,
+ name: withoutPrefix(r.path, key),
+ to: key,
+ size,
+ modified,
+ archived,
+ }))
+ const items = [...dirs, ...files]
+ // filter-out files with same name as one of dirs
+ return R.uniqBy(R.prop('name'), items)
+ }, [r])
+}
+
+const useDirContentsStyles = M.makeStyles((t) => ({
+ root: {
+ borderBottom: `1px solid ${t.palette.divider}`,
+ borderTop: `1px solid ${t.palette.divider}`,
+ flexGrow: 1,
+ marginLeft: t.spacing(2),
+ marginRight: t.spacing(2),
+ marginTop: t.spacing(1),
+ },
+ interactive: {
+ cursor: 'pointer',
+ },
+}))
+
+interface DirContentsProps {
+ response: requests.BucketListingResult
+ locked: boolean
+ setPath: (path: string) => void
+ setPrefix: (prefix: string) => void
+ loadMore: () => void
+ selection: DG.GridRowId[]
+ onSelectionChange: (newSelection: DG.GridRowId[]) => void
+}
+
+function DirContents({
+ response,
+ locked,
+ setPath,
+ setPrefix,
+ loadMore,
+ selection,
+ onSelectionChange,
+}: DirContentsProps) {
+ const classes = useDirContentsStyles()
+ const items = useFormattedListing(response)
+ const { bucket, path, prefix, truncated } = response
+
+ React.useLayoutEffect(() => {
+ // reset selection when bucket, path and / or prefix change
+ onSelectionChange([])
+ }, [onSelectionChange, bucket, path, prefix])
+
+ const CellComponent = React.useMemo(
+ () =>
+ function Cell({ item, className, ...props }: Listing.CellProps) {
+ const onClick = React.useCallback(() => {
+ if (item.type === 'dir') {
+ setPath(item.to)
+ setPrefix('')
+ }
+ }, [item])
+ return (
+ // eslint-disable-next-line jsx-a11y/interactive-supports-focus, jsx-a11y/click-events-have-key-events
+
+ )
+ },
+ [classes.interactive, setPath, setPrefix],
+ )
+
+ return (
+
+ }
+ />
+ )
+}
diff --git a/catalog/app/containers/Bucket/PackageUpdateDialog.tsx b/catalog/app/containers/Bucket/PackageUpdateDialog.tsx
index ceefbd4cce7..91e75d4c3ed 100644
--- a/catalog/app/containers/Bucket/PackageUpdateDialog.tsx
+++ b/catalog/app/containers/Bucket/PackageUpdateDialog.tsx
@@ -20,6 +20,7 @@ import * as validators from 'utils/validators'
import type * as workflows from 'utils/workflows'
import * as PD from './PackageDialog'
+import { isS3File, S3File } from './PackageDialog/S3FilePicker'
import * as requests from './requests'
interface Manifest {
@@ -59,6 +60,16 @@ interface Uploads {
}
}
+interface LocalEntry {
+ path: string
+ file: PD.LocalFile
+}
+
+interface S3Entry {
+ path: string
+ file: S3File
+}
+
const getTotalProgress = R.pipe(
R.values,
R.reduce(
@@ -147,7 +158,7 @@ function DialogForm({
[setUploads],
)
- const updatePackage = requests.useUpdatePackage()
+ const createPackage = requests.useCreatePackage()
const onSubmit = async ({
name,
@@ -163,11 +174,17 @@ function DialogForm({
workflow: workflows.Workflow
// eslint-disable-next-line consistent-return
}) => {
- const addedEntries = Object.entries(files.added).map(([path, file]) => ({
- path,
- file,
- }))
- const toUpload = addedEntries.filter(({ path, file }) => {
+ const addedS3Entries = [] as S3Entry[]
+ const addedLocalEntries = [] as LocalEntry[]
+ Object.entries(files.added).forEach(([path, file]) => {
+ if (isS3File(file)) {
+ addedS3Entries.push({ path, file })
+ } else {
+ addedLocalEntries.push({ path, file })
+ }
+ })
+
+ const toUpload = addedLocalEntries.filter(({ path, file }) => {
const e = files.existing[path]
return !e || e.hash !== file.hash.value
})
@@ -227,8 +244,9 @@ function DialogForm({
string,
{ physicalKey: string; size: number; hash: string; meta: unknown },
]
- const newEntries = pipeThru(toUpload, uploaded)(
- R.zipWith((f, r) => {
+
+ const uploadedEntries = pipeThru(toUpload, uploaded)(
+ R.zipWith((f, r) => {
invariant(f.file.hash.value, 'File must have a hash')
return [
f.path,
@@ -250,9 +268,18 @@ function DialogForm({
{ physicalKey: string; size: number; hash: string; meta: unknown }
>
+ const s3Entries = pipeThru(addedS3Entries)(
+ R.map(({ path, file }: S3Entry) => [
+ path,
+ { physicalKey: s3paths.handleToS3Url(file) },
+ ]),
+ R.fromPairs,
+ ) as Record
+
const contents = pipeThru(files.existing)(
R.omit(Object.keys(files.deleted)),
- R.mergeLeft(newEntries),
+ R.mergeLeft(uploadedEntries),
+ R.mergeLeft(s3Entries),
R.toPairs,
R.map(([path, data]) => ({
logical_key: path,
@@ -265,7 +292,7 @@ function DialogForm({
)
try {
- const res = await updatePackage(
+ const res = await createPackage(
{
contents,
message: msg,
@@ -455,6 +482,7 @@ function DialogForm({
onFilesAction={onFilesAction}
isEqual={R.equals}
initialValue={initialFiles}
+ bucket={bucket}
/>
diff --git a/catalog/app/containers/Bucket/requests/bucketListing.ts b/catalog/app/containers/Bucket/requests/bucketListing.ts
index 35f618dd943..2f46e87a4f2 100644
--- a/catalog/app/containers/Bucket/requests/bucketListing.ts
+++ b/catalog/app/containers/Bucket/requests/bucketListing.ts
@@ -1,11 +1,56 @@
import type { S3 } from 'aws-sdk'
+import * as React from 'react'
import * as R from 'ramda'
+import * as AWS from 'utils/AWS'
+
import * as errors from '../errors'
import { decodeS3Key } from './utils'
-interface File {
+const DEFAULT_DRAIN_REQUESTS = 10
+
+interface DrainObjectListParams {
+ s3: S3
+ bucket: string
+ prefix: string
+ delimiter?: string
+ continuationToken?: string
+ maxRequests: true | number
+}
+
+const drainObjectList = async ({
+ s3,
+ bucket,
+ prefix,
+ delimiter,
+ continuationToken,
+ maxRequests,
+}: DrainObjectListParams) => {
+ let reqNo = 0
+ let Contents: S3.Object[] = []
+ let CommonPrefixes: S3.CommonPrefixList = []
+ let ContinuationToken: string | undefined
+ while (true) {
+ // eslint-disable-next-line no-await-in-loop
+ const r = await s3
+ .listObjectsV2({
+ Bucket: bucket,
+ Delimiter: delimiter,
+ Prefix: prefix,
+ ContinuationToken: ContinuationToken || continuationToken,
+ EncodingType: 'url',
+ })
+ .promise()
+ Contents = Contents.concat(r.Contents || [])
+ CommonPrefixes = CommonPrefixes.concat(r.CommonPrefixes || [])
+ reqNo += 1
+ if (!r.IsTruncated || reqNo >= maxRequests) return { ...r, Contents, CommonPrefixes }
+ ContinuationToken = r.NextContinuationToken
+ }
+}
+
+export interface BucketListingFile {
bucket: string
key: string
modified: Date
@@ -16,7 +61,7 @@ interface File {
export interface BucketListingResult {
dirs: string[]
- files: File[]
+ files: BucketListingFile[]
truncated: boolean
continuationToken?: string
bucket: string
@@ -24,30 +69,36 @@ export interface BucketListingResult {
prefix?: string
}
-interface BucketListingParams {
+interface BucketListingDependencies {
s3: S3
+}
+
+interface BucketListingParams {
bucket: string
path?: string
prefix?: string
prev?: BucketListingResult
+ delimiter?: string | false
+ drain?: true | number
}
-export const bucketListing = ({
+export const bucketListing = async ({
s3,
bucket,
path = '',
prefix,
prev,
-}: BucketListingParams): Promise =>
- s3
- .listObjectsV2({
- Bucket: bucket,
- Delimiter: '/',
- Prefix: path + (prefix || ''),
- EncodingType: 'url',
- ContinuationToken: prev ? prev.continuationToken : undefined,
- })
- .promise()
+ delimiter = '/',
+ drain = 0,
+}: BucketListingParams & BucketListingDependencies): Promise =>
+ drainObjectList({
+ s3,
+ bucket,
+ prefix: path + (prefix || ''),
+ delimiter: delimiter === false ? undefined : delimiter,
+ continuationToken: prev ? prev.continuationToken : undefined,
+ maxRequests: drain === true ? DEFAULT_DRAIN_REQUESTS : drain,
+ })
.then((res) => {
let dirs = (res.CommonPrefixes || [])
.map((p) => decodeS3Key(p.Prefix!))
@@ -60,7 +111,6 @@ export const bucketListing = ({
// filter-out "directory-files" (files that match prefixes)
.filter(({ Key }: S3.Object) => Key !== path && !Key!.endsWith('/'))
.map((i: S3.Object) => ({
- // TODO: expose VersionId?
bucket,
key: i.Key!,
modified: i.LastModified!,
@@ -81,3 +131,11 @@ export const bucketListing = ({
}
})
.catch(errors.catchErrors())
+
+export function useBucketListing() {
+ const s3: S3 = AWS.S3.use()
+ return React.useCallback(
+ (params: BucketListingParams) => bucketListing({ s3, ...params }),
+ [s3],
+ )
+}
diff --git a/catalog/app/containers/Bucket/requests/package.ts b/catalog/app/containers/Bucket/requests/package.ts
index 106a5bab374..de50fb04a06 100644
--- a/catalog/app/containers/Bucket/requests/package.ts
+++ b/catalog/app/containers/Bucket/requests/package.ts
@@ -1,9 +1,11 @@
+import type { S3 } from 'aws-sdk'
import * as R from 'ramda'
import * as React from 'react'
import { JsonValue } from 'components/JsonEditor/constants'
import * as APIConnector from 'utils/APIConnector'
import * as AWS from 'utils/AWS'
+import * as Config from 'utils/Config'
import { makeSchemaDefaultsSetter, JsonSchema } from 'utils/json-schema'
import mkSearch from 'utils/mkSearch'
import pipeThru from 'utils/pipeThru'
@@ -17,7 +19,6 @@ interface AWSCredentials {
}
// "CREATE package" - creates package from scratch to new target
-// "UPDATE package" - creates package from scratch (using existing manifest) to defined target
// "COPY package" - creates package from source (existing manifest) to new target
// "WRAP package" - creates package (wraps directory) from source (bucket + directory) to new target
@@ -28,14 +29,11 @@ interface FileEntry {
}
interface FileUpload {
- hash: string
logical_key: string
- physical_key: {
- bucket: string
- key: string
- version: string
- }
- size: number
+ physical_key: string
+ hash?: string
+ size?: number
+ meta?: {}
}
interface RequestBodyBase {
@@ -45,10 +43,7 @@ interface RequestBodyBase {
workflow?: string | null
}
-interface RequestBodyCreate extends RequestBodyBase {
- contents: FileUpload[]
- name: string
-}
+type RequestBodyCreate = string
interface RequestBodyCopy extends RequestBodyBase {
name: string
@@ -67,21 +62,13 @@ interface RequestBodyWrap extends RequestBodyBase {
entries: FileEntry[]
}
-type RequestBody = RequestBodyCreate | RequestBodyWrap | RequestBodyCopy
-
const ENDPOINT_CREATE = '/packages'
-const ENDPOINT_UPDATE = '/packages'
-
const ENDPOINT_COPY = '/packages/promote'
const ENDPOINT_WRAP = '/packages/from-folder'
-type Endpoint =
- | typeof ENDPOINT_CREATE
- | typeof ENDPOINT_UPDATE
- | typeof ENDPOINT_COPY
- | typeof ENDPOINT_WRAP
+const CREATE_PACKAGE_PAYLOAD_KEY = 'user-requests/create-package'
// TODO: reuse it from some other place, don't remember where I saw it
interface ManifestHandleTarget {
@@ -107,14 +94,6 @@ interface CreatePackageParams extends BasePackageParams {
}
}
-interface UpdatePackageParams extends BasePackageParams {
- contents: FileUpload[]
- target: {
- bucket: string
- name: string
- }
-}
-
interface CopyPackageParams extends BasePackageParams {
source: ManifestHandleSource
target: ManifestHandleTarget
@@ -132,20 +111,53 @@ interface Response {
// FIXME: this is copypasted from PackageDialog -- next time we need to TSify utils/APIConnector properly
interface ApiRequest {
- (opts: {
+