Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add containerisation and step bundles info to the workflow run plan #992

Merged
merged 9 commits into from
Aug 16, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
70 changes: 70 additions & 0 deletions _tests/integration/docker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@
package integration

import (
"encoding/json"
"strings"
"testing"

"github.com/bitrise-io/bitrise/models"
"github.com/bitrise-io/go-utils/command"
"github.com/ryanuber/go-glob"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -158,3 +161,70 @@ func Test_Docker(t *testing.T) {
})
}
}

func Test_Docker_JSON_Logs(t *testing.T) {
testCases := map[string]struct {
workflowName string
configPath string
inventoryPath string
requiredContainerImage string
requiredServiceImages []string
}{
"With group with step execution and service containers": {
workflowName: "docker-login-multiple-containers",
configPath: "docker_multiple_containers_bitrise.yml",
inventoryPath: "docker_multiple_containers_secrets.yml",
requiredContainerImage: "localhost:5001/healthy-image",
requiredServiceImages: []string{
"localhost:5002/healthy-image",
"localhost:5003/healthy-image",
},
},
}
for testName, testCase := range testCases {
t.Run(testName, func(t *testing.T) {
cmd := command.New(binPath(), "run", testCase.workflowName, "--config", testCase.configPath, "--inventory", testCase.inventoryPath, "--output-format", "json")
out, _ := cmd.RunAndReturnTrimmedCombinedOutput()
//require.NoError(t, err, out)
checkRequiredContainers(t, out, testCase.requiredContainerImage, testCase.requiredServiceImages)
})
}
}

func checkRequiredContainers(t *testing.T, log string, requiredContainerImage string, requiredServiceImages []string) {
lines := strings.Split(log, "\n")
require.True(t, len(lines) > 0)

var bitriseStartedEvent models.WorkflowRunPlan
for _, line := range lines {
var eventLogStruct struct {
EventType string `json:"event_type"`
Content models.WorkflowRunPlan `json:"content"`
}
require.NoError(t, json.Unmarshal([]byte(line), &eventLogStruct))
if eventLogStruct.EventType == "bitrise_started" {
bitriseStartedEvent = eventLogStruct.Content
break
}
}

var usedContainerImages []string
var usedServiceImages []string

for _, workflowPlans := range bitriseStartedEvent.ExecutionPlan {
for _, stepPlans := range workflowPlans.Steps {
if stepPlans.WithGroupUUID != "" {
withGroupPlan := bitriseStartedEvent.WithGroupPlans[stepPlans.WithGroupUUID]

usedContainerImages = append(usedContainerImages, withGroupPlan.Container.Image)
for _, servicePlan := range withGroupPlan.Services {
usedServiceImages = append(usedServiceImages, servicePlan.Image)
}
}
}
}

require.Equal(t, 1, len(usedContainerImages), log)
require.EqualValues(t, requiredContainerImage, usedContainerImages[0], log)
require.EqualValues(t, requiredServiceImages, usedServiceImages, log)
}
11 changes: 8 additions & 3 deletions _tests/integration/modular_config_module.yml
Original file line number Diff line number Diff line change
@@ -1,8 +1,13 @@
step_bundles:
print:
steps:
- script:
inputs:
- content: echo "Hello $NAME!"

workflows:
print_hello_world:
envs:
- NAME: World
steps:
- script:
inputs:
- content: echo "Hello $NAME!"
- bundle::print: { }
47 changes: 46 additions & 1 deletion _tests/integration/modular_config_test.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package integration

import (
"encoding/json"
"os"
"strings"
"testing"

"github.com/bitrise-io/bitrise/models"
"github.com/bitrise-io/go-utils/command"
"github.com/stretchr/testify/require"
)

func Test_ModularConfig_Run(t *testing.T) {
func Test_ModularConfig(t *testing.T) {
configPth := "modular_config_main.yml"
deployDir := os.Getenv("BITRISE_DEPLOY_DIR")

Expand Down Expand Up @@ -41,3 +44,45 @@ func Test_ModularConfig_Run(t *testing.T) {
require.NoError(t, err, out)
require.Contains(t, out, "Hello World!")
}

func Test_ModularConfig_Run_JSON_Logs(t *testing.T) {
configPth := "modular_config_main.yml"

cmd := command.New(binPath(), "run", "print_hello_world", "--config", configPth, "--output-format", "json")
out, err := cmd.RunAndReturnTrimmedCombinedOutput()
require.NoError(t, err, out)
require.Contains(t, out, "Hello World!")
checkRequiredStepBundle(t, out, "print")
}

func checkRequiredStepBundle(t *testing.T, log string, requiredStepBundle string) {
lines := strings.Split(log, "\n")
require.True(t, len(lines) > 0)

var bitriseStartedEvent models.WorkflowRunPlan
for _, line := range lines {
var eventLogStruct struct {
EventType string `json:"event_type"`
Content models.WorkflowRunPlan `json:"content"`
}
require.NoError(t, json.Unmarshal([]byte(line), &eventLogStruct))
if eventLogStruct.EventType == "bitrise_started" {
bitriseStartedEvent = eventLogStruct.Content
break
}
}

var usedStepBundles []string

for _, workflowPlans := range bitriseStartedEvent.ExecutionPlan {
for _, stepPlans := range workflowPlans.Steps {
if stepPlans.StepBundleUUID != "" {
stepBundlePlan := bitriseStartedEvent.StepBundlePlans[stepPlans.StepBundleUUID]
usedStepBundles = append(usedStepBundles, stepBundlePlan.ID)
}
}
}

require.Equal(t, 1, len(usedStepBundles), log)
require.EqualValues(t, requiredStepBundle, usedStepBundles[0], log)
}
35 changes: 33 additions & 2 deletions cli/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -307,7 +307,11 @@ func (r WorkflowRunner) runWorkflows(tracker analytics.Tracker) (models.BuildRun
ProjectType: r.config.Config.ProjectType,
}

plan, err := createWorkflowRunPlan(r.config.Modes, r.config.Workflow, r.config.Config.Workflows, r.config.Config.StepBundles, func() string { return uuid.Must(uuid.NewV4()).String() })
plan, err := createWorkflowRunPlan(
r.config.Modes, r.config.Workflow, r.config.Config.Workflows,
r.config.Config.StepBundles, r.config.Config.Containers, r.config.Config.Services,
func() string { return uuid.Must(uuid.NewV4()).String() },
)
if err != nil {
return models.BuildRunResultsModel{}, fmt.Errorf("failed to create workflow execution plan: %w", err)
}
Expand Down Expand Up @@ -495,8 +499,14 @@ func registerRunModes(modes models.WorkflowRunModes) error {
return nil
}

func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel, stepBundles map[string]models.StepBundleModel, uuidProvider func() string) (models.WorkflowRunPlan, error) {
func createWorkflowRunPlan(
modes models.WorkflowRunModes, targetWorkflow string, workflows map[string]models.WorkflowModel,
stepBundles map[string]models.StepBundleModel, containers map[string]models.Container, services map[string]models.Container,
uuidProvider func() string,
) (models.WorkflowRunPlan, error) {
var executionPlan []models.WorkflowExecutionPlan
withGroupPlans := map[string]models.WithGroupPlan{}
stepBundlePlans := map[string]models.StepBundlePlan{}

workflowList := walkWorkflows(targetWorkflow, workflows, nil)
for _, workflowID := range workflowList {
Expand Down Expand Up @@ -530,6 +540,21 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string,

groupID := uuidProvider()

var containerPlan models.ContainerPlan
if with.ContainerID != "" {
containerPlan.Image = containers[with.ContainerID].Image
}

var servicePlans []models.ContainerPlan
for _, serviceID := range with.ServiceIDs {
servicePlans = append(servicePlans, models.ContainerPlan{Image: services[serviceID].Image})
}

withGroupPlans[groupID] = models.WithGroupPlan{
Services: servicePlans,
Container: containerPlan,
}

for _, stepListStepItem := range with.Steps {
stepID, step, err := stepListStepItem.GetStepIDAndStep()
if err != nil {
Expand Down Expand Up @@ -560,6 +585,10 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string,
bundleEnvs := append(bundleDefinition.Environments, bundleOverride.Environments...)
bundleUUID := uuidProvider()

stepBundlePlans[bundleUUID] = models.StepBundlePlan{
ID: bundleID,
}

for idx, stepListStepItem := range bundleDefinition.Steps {
stepID, step, err := stepListStepItem.GetStepIDAndStep()
if err != nil {
Expand Down Expand Up @@ -611,6 +640,8 @@ func createWorkflowRunPlan(modes models.WorkflowRunModes, targetWorkflow string,
NoOutputTimeoutMode: modes.NoOutputTimeout > 0,
SecretFilteringMode: modes.SecretFilteringMode,
SecretEnvsFilteringMode: modes.SecretEnvsFilteringMode,
WithGroupPlans: withGroupPlans,
StepBundlePlans: stepBundlePlans,
ExecutionPlan: executionPlan,
}, nil
}
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ require (
github.com/bitrise-io/go-utils v1.0.13
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22
github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef
github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1
github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pull stepman's current master branch

github.com/go-git/go-git/v5 v5.12.0
github.com/gofrs/uuid v4.3.1+incompatible
github.com/hashicorp/go-version v1.4.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22 h1:/SD9xE4LlX/Ju9YZ+n/yW/uDs7h
github.com/bitrise-io/go-utils/v2 v2.0.0-alpha.22/go.mod h1:Laih4ji980SQkRgdnMCH0g4u2GZI/5nnbqmYT9UfKFQ=
github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef h1:R5FOa8RHjqZwMN9g1FQ8W7nXxQAG7iwq1Cw+mUk5S9A=
github.com/bitrise-io/goinp v0.0.0-20240103152431-054ed78518ef/go.mod h1:27ldH2bkCdYN5CEJ6x92EK+gkd5EcDBkA7dMrSKQFYU=
github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1 h1:/hVIftPENx0k6JuXDgPbpWXCx1Lm4GbvsUVfgr6618A=
github.com/bitrise-io/stepman v0.0.0-20240628140527-5e941cdb67a1/go.mod h1:netRLDQD95IzWZbzmn7CBolzNqH1tErRKS31BrZKt9s=
github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf h1:Sw+nLHrAqcPE7jbsIgFMvaRsvQOZAA95xFCPUSb0ZKE=
github.com/bitrise-io/stepman v0.0.0-20240731124408-5b7e38abd0bf/go.mod h1:Lq9nEqKerBD35w3eSU8lf83F7uZPkXfmRSZEUDJN40w=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
Expand Down
21 changes: 18 additions & 3 deletions models/workflow_run_plan.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,11 @@ type StepExecutionPlan struct {

Step stepmanModels.StepModel `json:"-"`
// With (container) group
WithGroupUUID string `json:"-"`
WithGroupUUID string `json:"with_group_uuid,omitempty"`
ContainerID string `json:"-"`
ServiceIDs []string `json:"-"`
// Step Bundle group
StepBundleUUID string `json:"-"`
StepBundleUUID string `json:"step_bundle_uuid,omitempty"`
StepBundleEnvs []envmanModels.EnvironmentItemModel `json:"-"`
}

Expand All @@ -41,6 +41,19 @@ type WorkflowExecutionPlan struct {
IsSteplibOfflineMode bool `json:"-"`
}

type ContainerPlan struct {
Image string `json:"image"`
}

type WithGroupPlan struct {
Services []ContainerPlan `json:"services,omitempty"`
Container ContainerPlan `json:"container,omitempty"`
}

type StepBundlePlan struct {
ID string `json:"id"`
}

type WorkflowRunPlan struct {
Version string `json:"version"`
LogFormatVersion string `json:"log_format_version"`
Expand All @@ -53,5 +66,7 @@ type WorkflowRunPlan struct {
SecretFilteringMode bool `json:"secret_filtering_mode"`
SecretEnvsFilteringMode bool `json:"secret_envs_filtering_mode"`

ExecutionPlan []WorkflowExecutionPlan `json:"execution_plan"`
WithGroupPlans map[string]WithGroupPlan `json:"with_groups,omitempty"`
StepBundlePlans map[string]StepBundlePlan `json:"step_bundles,omitempty"`
ExecutionPlan []WorkflowExecutionPlan `json:"execution_plan"`
}
12 changes: 0 additions & 12 deletions tools/tools.go
Original file line number Diff line number Diff line change
Expand Up @@ -134,18 +134,6 @@ func StepmanSetup(collection string) error {
return stepman.Setup(collection, "", log)
}

// StepmanUpdate ...
func StepmanUpdate(collection string) error {
log := log.NewLogger(log.GetGlobalLoggerOpts())
return stepman.UpdateLibrary(collection, log)
}

// StepmanActivate ...
func StepmanActivate(collection, stepID, stepVersion, dir, ymlPth string, isOfflineMode bool) error {
log := log.NewLogger(log.GetGlobalLoggerOpts())
return stepman.Activate(collection, stepID, stepVersion, dir, ymlPth, false, log, isOfflineMode)
}

// StepmanStepInfo ...
func StepmanStepInfo(collection, stepID, stepVersion string) (stepmanModels.StepInfoModel, error) {
log := log.NewLogger(log.GetGlobalLoggerOpts())
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading