Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[flyte-core] Flyte Connection #5126

Draft
wants to merge 43 commits into
base: master
Choose a base branch
from
Draft
Changes from 1 commit
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
dfffe71
Add secret
pingsutw Mar 27, 2024
fa7573c
Create mock class
pingsutw Mar 27, 2024
0e235e1
update mock
pingsutw Mar 27, 2024
60754bc
feat: Add Secrets support to webapi plugin requests
pingsutw Mar 27, 2024
1e82896
wip
pingsutw Apr 3, 2024
18e91a2
merged master
pingsutw Apr 4, 2024
42bbe73
secretPtr
pingsutw Apr 4, 2024
a61fa8c
wip
pingsutw Apr 5, 2024
d895c71
Add connection
pingsutw Apr 17, 2024
61263ad
update proto
pingsutw Apr 17, 2024
0e69505
merged master
pingsutw Apr 17, 2024
e928f35
lint
pingsutw Apr 17, 2024
0b80b41
cleanup
pingsutw Apr 17, 2024
249fbe9
wip
pingsutw Apr 17, 2024
d525cf9
wip
pingsutw Apr 29, 2024
79302e5
go generate
pingsutw Apr 29, 2024
df830ea
fix tests
pingsutw Apr 29, 2024
9574f59
test
pingsutw Apr 29, 2024
7dedb8f
more tests
pingsutw Apr 29, 2024
23357c5
fix test
pingsutw Apr 29, 2024
307c06c
lint
pingsutw Apr 30, 2024
9429ac2
Merge branch 'master' of github.com:flyteorg/flyte into agent-secret
pingsutw Apr 30, 2024
aecfaa5
lint
pingsutw Apr 30, 2024
7afda69
merged master
pingsutw May 1, 2024
0e25e5b
Add interface
pingsutw May 2, 2024
1f01450
wip
pingsutw May 10, 2024
0808ec0
Merge branch 'master' of github.com:flyteorg/flyte into agent-secret
pingsutw May 10, 2024
b4f264c
Add ExternalResourceAttributes
pingsutw May 10, 2024
466c61c
nit
pingsutw May 10, 2024
b19daea
update test
pingsutw May 13, 2024
090a10b
update test
pingsutw May 13, 2024
d96975d
fix test
pingsutw May 13, 2024
22966d0
fix test
pingsutw May 13, 2024
54e5cc8
fix test
pingsutw May 13, 2024
a8b7d06
fix test
pingsutw May 13, 2024
78b7dfc
Add more tests
pingsutw May 13, 2024
bf9b39f
install tabulate
pingsutw May 13, 2024
f7e79b4
Address comment
pingsutw May 15, 2024
d12a889
nit
pingsutw May 15, 2024
ff3ece0
Merge branch 'master' of github.com:flyteorg/flyte into agent-secret
pingsutw May 15, 2024
0c0f723
rename connection_ref
pingsutw May 16, 2024
39093e1
make generate
pingsutw May 31, 2024
ad0a88f
merged master
pingsutw Jun 26, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix test
Signed-off-by: Kevin Su <[email protected]>
pingsutw committed May 13, 2024
commit 22966d01e9af82ee5c23f11667aceff957cdf314

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Original file line number Diff line number Diff line change
@@ -496,7 +496,7 @@ func Test_dynamicNodeHandler_buildContextualDynamicWorkflow_withLaunchPlans(t *t
composedPBStore.OnWriteRawMatch(
mock.MatchedBy(func(ctx context.Context) bool { return true }),
storage.DataReference("s3://my-s3-bucket/foo/bar/futures_compiled.pb"),
int64(1501),
int64(1550),
storage.Options{},
mock.MatchedBy(func(rdr *bytes.Reader) bool { return true })).Return(errors.New("foo"))


Unchanged files with check annotations Beta

func (m *ExecutionManager) getExternalResourceAttributes(ctx context.Context, request *admin.ExecutionCreateRequest) (
*admin.ExternalResourceAttributes, error) {
if request.Spec.ExternalResourceAttributes != nil {
return request.Spec.ExternalResourceAttributes, nil

Check warning on line 427 in flyteadmin/pkg/manager/impl/execution_manager.go

Codecov / codecov/patch

flyteadmin/pkg/manager/impl/execution_manager.go#L427

Added line #L427 was not covered by tests
}
resource, err := m.resourceManager.GetResource(ctx, interfaces.ResourceRequest{
ResourceType: admin.MatchableResource_EXTERNAL_RESOURCE,
})
if err != nil && !errors.IsDoesNotExistError(err) {
logger.Errorf(ctx, "Failed to get external resource with error: %v", err)
return nil, err

Check warning on line 437 in flyteadmin/pkg/manager/impl/execution_manager.go

Codecov / codecov/patch

flyteadmin/pkg/manager/impl/execution_manager.go#L436-L437

Added lines #L436 - L437 were not covered by tests
}
if resource != nil && resource.Attributes.GetExternalResourceAttributes() != nil {
return resource.Attributes.GetExternalResourceAttributes(), nil

Check warning on line 440 in flyteadmin/pkg/manager/impl/execution_manager.go

Codecov / codecov/patch

flyteadmin/pkg/manager/impl/execution_manager.go#L440

Added line #L440 was not covered by tests
}
externalResource := m.config.ExternalResourceConfiguration().GetExternalResource()
connections := make(map[string]*core.Connection)
for key, connection := range externalResource.Connections {
connections[key] = &core.Connection{
Secrets: connection.Secrets,
Configs: connection.Configs,

Check warning on line 448 in flyteadmin/pkg/manager/impl/execution_manager.go

Codecov / codecov/patch

flyteadmin/pkg/manager/impl/execution_manager.go#L446-L448

Added lines #L446 - L448 were not covered by tests
}
}
externalResourceAttribute, err := m.getExternalResourceAttributes(ctx, &request)
if err != nil {
return nil, nil, err

Check warning on line 987 in flyteadmin/pkg/manager/impl/execution_manager.go

Codecov / codecov/patch

flyteadmin/pkg/manager/impl/execution_manager.go#L987

Added line #L987 was not covered by tests
}
var executionClusterLabel *admin.ExecutionClusterLabel
return p.clusterPoolAssignmentConfiguration
}
func (p *ConfigurationProvider) ExternalResourceConfiguration() interfaces.ExternalResourceConfiguration {
return p.externalResourceConfiguration

Check warning on line 63 in flyteadmin/pkg/runtime/configuration_provider.go

Codecov / codecov/patch

flyteadmin/pkg/runtime/configuration_provider.go#L62-L63

Added lines #L62 - L63 were not covered by tests
}
func NewConfigurationProvider() interfaces.Configuration {
// ExternalResourcesConfigurationProvider Implementation of an interfaces.ExternalResourceConfiguration
type ExternalResourcesConfigurationProvider struct{}
func (e ExternalResourcesConfigurationProvider) GetExternalResource() interfaces.ExternalResource {
return externalResourcesConfig.GetConfig().(*interfaces.ExternalResourceConfig).ExternalResource

Check warning on line 16 in flyteadmin/pkg/runtime/external_resource_provider.go

Codecov / codecov/patch

flyteadmin/pkg/runtime/external_resource_provider.go#L15-L16

Added lines #L15 - L16 were not covered by tests
}
func NewExternalResourceConfigurationProvider() interfaces.ExternalResourceConfiguration {
}
if externalResource != nil && externalResource.GetConnections() != nil {
connections := make(map[string]v1alpha1.Connection)
for name, connection := range externalResource.GetConnections() {
connections[name] = v1alpha1.Connection{
Secrets: connection.GetSecrets(),
Configs: connection.GetConfigs(),

Check warning on line 116 in flyteadmin/pkg/workflowengine/impl/prepare_execution.go

Codecov / codecov/patch

flyteadmin/pkg/workflowengine/impl/prepare_execution.go#L112-L116

Added lines #L112 - L116 were not covered by tests
}
}
executionConfig.ExternalResourceAttribute = v1alpha1.ExternalResourceAttributes{
Connections: connections,

Check warning on line 121 in flyteadmin/pkg/workflowengine/impl/prepare_execution.go

Codecov / codecov/patch

flyteadmin/pkg/workflowengine/impl/prepare_execution.go#L120-L121

Added lines #L120 - L121 were not covered by tests
}
}