Merge pull request #194 from orbs-network/feature/inspect-local-conta…

…iners

Inspect state of local containers to provide more debug information

.version

@@ -1 +1 @@
-v1.8.2
+v1.9.0

README.md

@@ -23,21 +23,17 @@ Staring from version 0.17.0, Boyar only works with Docker version higher than 19
 
 ## Building
 
-```bash
-export GOOS=linux
-./build-binaries.sh
-```
-
-Alternatively, building in Docker:
+Building in Docker:
 
 ```
 ./docker-build.sh
 ```
 
-To publish new version to S3: update `.version` file and run this command:
+Alternative faster build:
 
 ```bash
-./.circleci/export.sh
+export GOOS=linux
+./build-binaries.sh
 ```
 
 ## Tips

strelets/adapter/orchestrator.go

@@ -2,6 +2,7 @@ package adapter
 
 import (
 	"context"
+	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/mount"
 	"io"
 	"time"
@@ -44,6 +45,10 @@ type ServiceConfig struct {
 	LogsMountPointNames map[string]string // simple name -> namespaced name
 }
 
+type ContainerDebugStatus struct {
+	ContainerState *types.ContainerState
+}
+
 type ContainerStatus struct {
 	Name   string
 	NodeID string
@@ -52,6 +57,8 @@ type ContainerStatus struct {
 
 	Logs string
 
+	Debug ContainerDebugStatus
+
 	CreatedAt time.Time
 }
 

strelets/adapter/swarm_get_status.go

@@ -17,14 +17,24 @@ func (d *dockerSwarmOrchestrator) GetStatus(ctx context.Context, since time.Dura
 			name, _ := d.getServiceName(ctx, task.ServiceID)
 			logs, _ := d.getLogs(ctx, task.ServiceID, since)
 
-			results = append(results, &ContainerStatus{
+			status := &ContainerStatus{
 				Name:      name,
 				State:     task.Status.Message,
 				Error:     task.Status.Err,
 				NodeID:    task.NodeID,
 				CreatedAt: task.CreatedAt,
 				Logs:      logs,
-			})
+			}
+
+			if task.Status.ContainerStatus != nil {
+				containerId := task.Status.ContainerStatus.ContainerID
+				containerJSON, err := d.client.ContainerInspect(ctx, containerId)
+				if err == nil { // skipping because it only works on the same machine
+					status.Debug.ContainerState = containerJSON.State
+				}
+			}
+
+			results = append(results, status)
 		}
 	}
 

strelets/adapter/swarm_get_status_test.go

@@ -3,6 +3,7 @@ package adapter
 import (
 	"context"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
 	dockerSwarm "github.com/docker/docker/api/types/swarm"
 	dockerClient "github.com/docker/docker/client"
 	"github.com/orbs-network/boyarin/test/helpers"
@@ -13,13 +14,15 @@ import (
 	"time"
 )
 
+const DEFAULT_DOCKER_IMAGE = "busybox"
+
 func TestDockerSwarm_GetStatusIfUnableToStart(t *testing.T) {
 	helpers.SkipUnlessSwarmIsEnabled(t)
 	helpers.WithContext(func(ctx context.Context) {
 		helpers.InitSwarmEnvironment(t, ctx)
 
 		serviceId := startDefunctContainer(t)
-		defer destroyDefunctBusybox(t, serviceId)
+		defer destroyDefunctContainer(t, serviceId)
 
 		swarm, err := NewDockerSwarm(OrchestratorOptions{}, log.GetLogger())
 		require.NoError(t, err)
@@ -28,7 +31,7 @@ func TestDockerSwarm_GetStatusIfUnableToStart(t *testing.T) {
 			status, err := swarm.GetStatus(context.TODO(), 5*time.Second)
 			require.NoError(t, err)
 			for _, s := range status {
-				if s.Name == defunctName {
+				if s.Name == DEFUNCT_NAME {
 					t.Log("polling reloadingBusybox:", "s.Error=", s.Error, "s.Logs=", s.Logs)
 					return strings.Contains(s.Error, "executable file not found")
 				}
@@ -45,7 +48,7 @@ func TestDockerSwarm_GetStatusIfExitsImmediately(t *testing.T) {
 		helpers.InitSwarmEnvironment(t, ctx)
 
 		serviceId := startReloadingContainer(t)
-		defer destroyDefunctBusybox(t, serviceId)
+		defer destroyDefunctContainer(t, serviceId)
 
 		swarm, err := NewDockerSwarm(OrchestratorOptions{}, log.GetLogger())
 		require.NoError(t, err)
@@ -54,7 +57,7 @@ func TestDockerSwarm_GetStatusIfExitsImmediately(t *testing.T) {
 			status, err := swarm.GetStatus(context.TODO(), 5*time.Second)
 			require.NoError(t, err)
 			for _, s := range status {
-				if s.Name == reloadingName {
+				if s.Name == RELOADING_NAME {
 					t.Log("polling reloadingBusybox:", "s.Error=", s.Error, "s.Logs=", s.Logs)
 					return strings.Contains(s.Error, "non-zero exit") && strings.Contains(s.Logs, "I can not be contained")
 				}
@@ -65,7 +68,36 @@ func TestDockerSwarm_GetStatusIfExitsImmediately(t *testing.T) {
 	})
 }
 
-const defunctName = "DefunctContainer"
+func TestDockerSwarm_GetStatusOfUnhealthyContainer(t *testing.T) {
+	helpers.SkipUnlessSwarmIsEnabled(t)
+	helpers.WithContext(func(ctx context.Context) {
+		helpers.InitSwarmEnvironment(t, ctx)
+
+		serviceId := startUnhealthyContainer(t)
+		defer destroyDefunctContainer(t, serviceId)
+
+		swarm, err := NewDockerSwarm(OrchestratorOptions{}, log.GetLogger())
+		require.NoError(t, err)
+
+		require.True(t, helpers.Eventually(30*time.Second, func() bool {
+			status, err := swarm.GetStatus(context.TODO(), 5*time.Second)
+			require.NoError(t, err)
+			for _, s := range status {
+				if s.Name == UNHEALTHY_NAME {
+					t.Log("polling reloadingBusybox:", "s.Error=", s.Error, "s.Logs=", s.Logs)
+					if s.Debug.ContainerState != nil && len(s.Debug.ContainerState.Health.Log) > 0 {
+						return strings.Contains(s.Error, "task: non-zero exit (137): dockerexec: unhealthy container") &&
+							strings.Contains(s.Debug.ContainerState.Health.Log[0].Output, "HEALTHCHECK FAILED")
+					}
+				}
+			}
+
+			return false
+		}), "should be able to retrieve logs from constantly reloading container")
+	})
+}
+
+const DEFUNCT_NAME = "DefunctContainer"
 
 func startDefunctContainer(t *testing.T) (serviceId string) {
 	client, err := dockerClient.NewClientWithOpts(dockerClient.WithVersion(DOCKER_API_VERSION))
@@ -74,20 +106,20 @@ func startDefunctContainer(t *testing.T) (serviceId string) {
 	spec := dockerSwarm.ServiceSpec{
 		TaskTemplate: dockerSwarm.TaskSpec{
 			ContainerSpec: &dockerSwarm.ContainerSpec{
-				Image:   "alpine",
+				Image:   DEFAULT_DOCKER_IMAGE,
 				Command: []string{"this-program-does-not-exist"},
 			},
 		},
 	}
-	spec.Name = defunctName
+	spec.Name = DEFUNCT_NAME
 
 	resp, err := client.ServiceCreate(context.Background(), spec, types.ServiceCreateOptions{})
 	require.NoError(t, err)
 
 	return resp.ID
 }
 
-const reloadingName = "ReloadingContainer"
+const RELOADING_NAME = "ReloadingContainer"
 
 func startReloadingContainer(t *testing.T) (serviceId string) {
 	client, err := dockerClient.NewClientWithOpts(dockerClient.WithVersion(DOCKER_API_VERSION))
@@ -96,20 +128,48 @@ func startReloadingContainer(t *testing.T) (serviceId string) {
 	spec := dockerSwarm.ServiceSpec{
 		TaskTemplate: dockerSwarm.TaskSpec{
 			ContainerSpec: &dockerSwarm.ContainerSpec{
-				Image:   "alpine",
+				Image:   DEFAULT_DOCKER_IMAGE,
 				Command: []string{"sh", "-c", "echo I can not be contained && exit 999"},
 			},
 		},
 	}
-	spec.Name = reloadingName
+	spec.Name = RELOADING_NAME
+
+	resp, err := client.ServiceCreate(context.Background(), spec, types.ServiceCreateOptions{})
+	require.NoError(t, err)
+
+	return resp.ID
+}
+
+const UNHEALTHY_NAME = "UnhealthyContainer"
+
+func startUnhealthyContainer(t *testing.T) (serviceId string) {
+	client, err := dockerClient.NewClientWithOpts(dockerClient.WithVersion(DOCKER_API_VERSION))
+	require.NoError(t, err)
+
+	spec := dockerSwarm.ServiceSpec{
+		TaskTemplate: dockerSwarm.TaskSpec{
+			ContainerSpec: &dockerSwarm.ContainerSpec{
+				Image:   DEFAULT_DOCKER_IMAGE,
+				Command: []string{"sh", "-c", "sleep 10000"},
+				Healthcheck: &container.HealthConfig{
+					Interval: 100 * time.Millisecond,
+					Retries:  1,
+					Test: []string{"CMD-SHELL",
+						"sh -c \"echo HEALTHCHECK FAILED; exit 1\""},
+				},
+			},
+		},
+	}
+	spec.Name = UNHEALTHY_NAME
 
 	resp, err := client.ServiceCreate(context.Background(), spec, types.ServiceCreateOptions{})
 	require.NoError(t, err)
 
 	return resp.ID
 }
 
-func destroyDefunctBusybox(t *testing.T, serviceId string) {
+func destroyDefunctContainer(t *testing.T, serviceId string) {
 	client, err := dockerClient.NewClientWithOpts(dockerClient.WithVersion(DOCKER_API_VERSION))
 	require.NoError(t, err)