Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Ability to add S3 folders / files to package #2171

Merged
merged 32 commits into from
Apr 28, 2021
Merged
Show file tree
Hide file tree
Changes from 31 commits
Commits
Show all changes
32 commits
Select commit Hold shift + click to select a range
d12a115
Bucket/Dir: use type from requests
nl0 Apr 12, 2021
41d5c9b
Bucket/Dir: small refactoring
nl0 Apr 12, 2021
1cdffd4
utils/s3paths: handleToS3Url: make 'version' paramter optional (for TS)
nl0 Apr 15, 2021
95e1b77
requests/bucketListing: support disabling delimiter, hook
nl0 Apr 15, 2021
86b81ed
requests/package: refactor, remove duplicate code
nl0 Apr 15, 2021
a10b54d
merge folders into pkg WIP
nl0 Apr 15, 2021
64156a0
BreadCrumbs: add some customization options
nl0 Apr 16, 2021
936ed94
Bucket/Listing: customizable root element
nl0 Apr 16, 2021
1f9e6ea
some styling for s3 file picker & stuff
nl0 Apr 16, 2021
9859a34
show overlay for s3 files
nl0 Apr 19, 2021
df4b08f
lock file browser and show progress while loading children
nl0 Apr 19, 2021
e11816b
styling adjustments mostly
nl0 Apr 20, 2021
dfc1348
further work on merge folder stuff
nl0 Apr 20, 2021
4659700
address CR feedback / cleanup
nl0 Apr 21, 2021
c4a8f8b
add serviceBucket to config schema
nl0 Apr 22, 2021
a421f18
utils/AWS/S3: sign requests to serviceBucket
nl0 Apr 22, 2021
61afa6b
pass package creation data via serviceBucket
nl0 Apr 22, 2021
0007860
pass version id / job id as req body
nl0 Apr 23, 2021
9901904
utils/s3paths: handleToS3Url: encode key
nl0 Apr 23, 2021
55cb899
utils/AWS/S3: rm unused api
nl0 Apr 23, 2021
2719b97
FilesInput: dont render collapsed children for perf
nl0 Apr 23, 2021
38f8320
dont send s3 file size
nl0 Apr 23, 2021
dd1ba6a
Merge branch 'master' into add-s3-files-to-package
nl0 Apr 23, 2021
87649e1
Listing: reset page when items change, fix display issue
nl0 Apr 23, 2021
6efcf59
size warnings, layout
nl0 Apr 23, 2021
5a387ba
full-heigh picker dialog, scroll table body
nl0 Apr 23, 2021
5ab482b
changelog entry
nl0 Apr 23, 2021
3df7fd8
bucketListing: support draining
nl0 Apr 23, 2021
63f1fb1
S3FilePicker: drain bucketListing (load in 10k batches)
nl0 Apr 23, 2021
b39428a
add service bucket to config template
sir-sigurd Apr 23, 2021
fd171c9
use correct const for max s3 files size
nl0 Apr 24, 2021
b3732bf
merge with master
fiskus Apr 28, 2021
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 7 additions & 4 deletions catalog/app/components/BreadCrumbs/BreadCrumbs.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import * as R from 'ramda'
import * as React from 'react'

import Link from 'utils/StyledLink'
Expand All @@ -10,13 +11,15 @@ export const Crumb = tagged([
'Sep', // value
])

export const Segment = ({ label, to }) =>
to ? <Link to={to}>{label || EMPTY}</Link> : label || EMPTY
export const Segment = ({ label, to, getLinkProps = R.identity }) =>
to != null ? <Link {...getLinkProps({ to })}>{label || EMPTY}</Link> : label || EMPTY

export const render = (items) =>
export const render = (items, { getLinkProps = undefined } = {}) =>
items.map(
Crumb.case({
Segment: (s, i) => <Segment key={`${i}:${s.label}`} {...s} />,
Segment: (s, i) => (
<Segment key={`${i}:${s.label}`} getLinkProps={getLinkProps} {...s} />
),
Sep: (s, i) => <React.Fragment key={`__sep${i}`}>{s}</React.Fragment>,
}),
)
Expand Down
85 changes: 35 additions & 50 deletions catalog/app/containers/Bucket/Dir.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -33,24 +33,6 @@ interface RouteMap {

type Urls = NamedRoutes.Urls<RouteMap>

interface ListingFile {
bucket: string
key: string
modified: Date
size: number
etag: string
archived: boolean
}

interface ListingResponse {
dirs: string[]
files: ListingFile[]
truncated: boolean
bucket: string
path: string
prefix: string
}

const getCrumbs = R.compose(
R.intersperse(Crumb.Sep(<>&nbsp;/ </>)),
({ bucket, path, urls }: { bucket: string; path: string; urls: Urls }) =>
Expand All @@ -63,39 +45,42 @@ const getCrumbs = R.compose(
),
)

const formatListing = ({ urls }: { urls: Urls }, r: ListingResponse) => {
const dirs = r.dirs.map((name) => ({
type: 'dir' as const,
name: ensureNoSlash(withoutPrefix(r.path, name)),
to: urls.bucketDir(r.bucket, name),
}))
const files = r.files.map(({ key, size, modified, archived }) => ({
type: 'file' as const,
name: withoutPrefix(r.path, key),
to: urls.bucketFile(r.bucket, key),
size,
modified,
archived,
}))
const items = [
...(r.path !== '' && !r.prefix
? [
{
type: 'dir' as const,
name: '..',
to: urls.bucketDir(r.bucket, up(r.path)),
},
]
: []),
...dirs,
...files,
]
// filter-out files with same name as one of dirs
return R.uniqBy(R.prop('name'), items)
function useFormattedListing(r: requests.BucketListingResult) {
const { urls } = NamedRoutes.use<RouteMap>()
return React.useMemo(() => {
const dirs = r.dirs.map((name) => ({
type: 'dir' as const,
name: ensureNoSlash(withoutPrefix(r.path, name)),
to: urls.bucketDir(r.bucket, name),
}))
const files = r.files.map(({ key, size, modified, archived }) => ({
type: 'file' as const,
name: withoutPrefix(r.path, key),
to: urls.bucketFile(r.bucket, key),
size,
modified,
archived,
}))
const items = [
...(r.path !== '' && !r.prefix
? [
{
type: 'dir' as const,
name: '..',
to: urls.bucketDir(r.bucket, up(r.path)),
},
]
: []),
...dirs,
...files,
]
// filter-out files with same name as one of dirs
return R.uniqBy(R.prop('name'), items)
}, [r, urls])
}

interface DirContentsProps {
response: ListingResponse
response: requests.BucketListingResult
locked: boolean
bucket: string
path: string
Expand Down Expand Up @@ -127,7 +112,7 @@ function DirContents({
[history, urls, bucket, path],
)

const items = React.useMemo(() => formatListing({ urls }, response), [urls, response])
const items = useFormattedListing(response)

// TODO: should prefix filtering affect summary?
return (
Expand Down Expand Up @@ -277,7 +262,7 @@ export default function Dir({
Err: displayError(),
Init: () => null,
_: (x: $TSFixMe) => {
const res: ListingResponse | null = AsyncResult.getPrevResult(x)
const res: requests.BucketListingResult | null = AsyncResult.getPrevResult(x)
return res ? (
<DirContents
response={res}
Expand Down
Loading