Skip to content

Commit 73ff107

Browse files
skudasovskudasov
authored
Docker-tc instead of Pumba (#2434)
Docker-tc instead of Pumba
1 parent 3626fdf commit 73ff107

6 files changed

Lines changed: 205 additions & 1 deletion

File tree

‎framework/.changeset/v0.14.8.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- Replace Pumba with Docker-TC to avoid Docker API compatibility issues

‎framework/chaos/chaos.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,17 +3,137 @@ package chaos
33
import (
44
"context"
55
"fmt"
6+
"slices"
67
"strings"
78
"time"
89

910
"github.com/docker/docker/api/types/container"
1011
"github.com/docker/docker/api/types/mount"
1112
"github.com/google/uuid"
1213
"github.com/testcontainers/testcontainers-go"
14+
"github.com/testcontainers/testcontainers-go/wait"
1315

1416
"github.com/smartcontractkit/chainlink-testing-framework/framework"
1517
)
1618

19+
/*
20+
* A simple wrapper for "docker-tc" chaos actions.
21+
* One small caveat is that in order for chaos to work your containers should be on a network so
22+
* interfaces like 'vetha57f116' are created inside them.
23+
* Works for any network, by default we are testing containers on "ctf" network.
24+
*/
25+
26+
const (
27+
// Docker and docker-tc commands
28+
CmdPause = "pause"
29+
CmdDelay = "delay"
30+
CmdLoss = "loss"
31+
CmdDuplicate = "duplicate"
32+
CmdCorrupt = "corrupt"
33+
)
34+
35+
const (
36+
// dockerTCContainerName default "docker-tc" container name
37+
dockerTCContainerName = "dtc"
38+
// dockerTCInternalSvc docker-tc internal service name
39+
dockerTCInternalSvc = "localhost:4080"
40+
)
41+
42+
var (
43+
defaultCURLCMD = fmt.Sprintf("docker exec %s curl", dockerTCContainerName)
44+
tcCommands = []string{CmdDelay, CmdLoss, CmdCorrupt, CmdDuplicate}
45+
)
46+
47+
// DockerChaos is a chaos generator for Docker
48+
type DockerChaos struct {
49+
Experiments map[string]string
50+
}
51+
52+
// NewDockerChaos creates a new "docker-tc" instance or reuses existing one
53+
func NewDockerChaos(ctx context.Context) (*DockerChaos, error) {
54+
framework.L.Info().
55+
Str("Container", dockerTCContainerName).
56+
Msg("Starting new docker-tc container")
57+
req := testcontainers.ContainerRequest{
58+
Image: "lukaszlach/docker-tc",
59+
Name: dockerTCContainerName,
60+
CapAdd: []string{"NET_ADMIN"},
61+
WaitingFor: wait.ForLog("Starting Docker Traffic Control"),
62+
HostConfigModifier: func(h *container.HostConfig) {
63+
h.Privileged = true
64+
h.NetworkMode = "host"
65+
h.Mounts = []mount.Mount{
66+
{
67+
Type: "bind",
68+
Source: "/var/run/docker.sock",
69+
Target: "/var/run/docker.sock",
70+
ReadOnly: true,
71+
},
72+
}
73+
},
74+
}
75+
_, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
76+
ContainerRequest: req,
77+
Started: true,
78+
Reuse: true,
79+
})
80+
if err != nil {
81+
return nil, fmt.Errorf("failed to start docker-tc container: %w", err)
82+
}
83+
return &DockerChaos{
84+
Experiments: make(map[string]string, 0),
85+
}, nil
86+
}
87+
88+
// RemoveAll removes all the experiments
89+
func (m *DockerChaos) RemoveAll() error {
90+
for containerName, experimentCmd := range m.Experiments {
91+
framework.L.Info().
92+
Str("Container", containerName).
93+
Str("Cmd", experimentCmd).
94+
Msg("Removing chaos for container")
95+
if _, err := framework.ExecCmd(experimentCmd); err != nil {
96+
return fmt.Errorf("failed to remove chaos experiment: name: %s, command:%s, err: %w", containerName, experimentCmd, err)
97+
}
98+
}
99+
m.Experiments = make(map[string]string)
100+
return nil
101+
}
102+
103+
// Chaos executes either Docker or "docker-tc" commands
104+
func (m *DockerChaos) Chaos(containerName string, cmd, val string) error {
105+
if _, ok := m.Experiments[containerName]; ok {
106+
return fmt.Errorf("chaos is already applied, only a single chaos can be applied to a container, call RemoveAll first")
107+
}
108+
// tc commands
109+
if slices.Contains(tcCommands, cmd) {
110+
m.Experiments[containerName] = fmt.Sprintf("%s -X DELETE %s/%s", defaultCURLCMD, dockerTCInternalSvc, containerName)
111+
out, err := framework.ExecCmd(fmt.Sprintf("%s -d %s=%s %s/%s", defaultCURLCMD, cmd, val, dockerTCInternalSvc, containerName))
112+
if err != nil {
113+
return err
114+
}
115+
return verifyTCOutput(string(out))
116+
}
117+
// docker commands
118+
m.Experiments[containerName] = fmt.Sprintf("docker unpause %s", containerName)
119+
_, err := framework.ExecCmd(fmt.Sprintf("docker pause %s", containerName))
120+
if err != nil {
121+
return err
122+
}
123+
return nil
124+
}
125+
126+
func verifyTCOutput(out string) error {
127+
if !strings.Contains(out, "Controlling traffic") {
128+
return fmt.Errorf("experiment failed to apply, set debug logs, export CTF_LOG_LEVEL=debug. Your container also must be on a network, 'ctf' or any other, won't work with default 'bridge'")
129+
}
130+
return nil
131+
}
132+
133+
// DEPRECATED: Since Pumba has outdated Docker dependencies it may not work without additional
134+
// setting to allow using Docker client which is out of client<>server compatibility range.
135+
// Use NewDockerChaos for pause and network experiments!
136+
//
17137
// ExecPumba executes Pumba (https://github.com/alexei-led/pumba) command
18138
// since handling various docker race conditions is hard and there is no easy API for that
19139
// for now you can provide time to wait until chaos is applied

‎framework/chaos/chaos_test.go‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
package chaos_test
2+
3+
import (
4+
"testing"
5+
"time"
6+
7+
"github.com/stretchr/testify/require"
8+
9+
"github.com/smartcontractkit/chainlink-testing-framework/framework/chaos"
10+
"github.com/smartcontractkit/chainlink-testing-framework/framework/rpc"
11+
)
12+
13+
func TestSmokeChaos(t *testing.T) {
14+
c, err := rpc.StartAnvil([]string{"--balance", "1", "--block-time", "5"})
15+
require.NoError(t, err)
16+
17+
i, err := c.Inspect(t.Context())
18+
require.NoError(t, err)
19+
containerName := i.Name[1:]
20+
21+
dtc, err := chaos.NewDockerChaos(t.Context())
22+
require.NoError(t, err)
23+
24+
tests := []struct {
25+
name string
26+
containerName string
27+
cmd string
28+
value string
29+
wantErr bool
30+
}{
31+
{
32+
name: "pause container",
33+
containerName: containerName,
34+
cmd: chaos.CmdPause,
35+
},
36+
{
37+
name: "delay container",
38+
containerName: containerName,
39+
cmd: chaos.CmdDelay,
40+
value: "8000ms",
41+
},
42+
{
43+
name: "loss container",
44+
containerName: containerName,
45+
cmd: chaos.CmdLoss,
46+
value: "50%",
47+
},
48+
{
49+
name: "corrupt traffic in container",
50+
containerName: containerName,
51+
cmd: chaos.CmdCorrupt,
52+
value: "10%",
53+
},
54+
{
55+
name: "duplicate traffic in container",
56+
containerName: containerName,
57+
cmd: chaos.CmdDuplicate,
58+
value: "50%",
59+
},
60+
}
61+
62+
for _, tc := range tests {
63+
t.Run(tc.name, func(t *testing.T) {
64+
err = dtc.Chaos(tc.containerName, tc.cmd, tc.value)
65+
if tc.wantErr {
66+
require.Error(t, err)
67+
} else {
68+
require.NoError(t, err)
69+
}
70+
time.Sleep(5 * time.Second)
71+
72+
err = dtc.RemoveAll()
73+
require.NoError(t, err)
74+
})
75+
}
76+
}

‎framework/components/fake/fake.go‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,6 @@ const (
1414
DefaultFakeServicePort = 9111
1515
)
1616

17-
1817
var (
1918
Service *gin.Engine
2019
validMethod = regexp.MustCompile("GET|POST|PATCH|PUT|DELETE")

‎framework/osutil.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import (
66
"io"
77
"os/exec"
88
"strings"
9+
"sync"
910
)
1011

1112
// ExecCmd executes a command and logs the output interactively
@@ -48,20 +49,25 @@ func ExecCmdWithOpts(ctx context.Context, command string, stdoutFunc func(string
4849
// create a buffer, listen to both pipe outputs, wait them to finish and merge output
4950
// both log it and return merged output
5051
var combinedBuf strings.Builder
52+
combinedBufMu := &sync.Mutex{}
5153
stdoutDone := make(chan struct{})
5254
stderrDone := make(chan struct{})
5355

5456
go func() {
5557
readStdPipe(stdout, func(m string) {
5658
stdoutFunc(m)
59+
combinedBufMu.Lock()
5760
combinedBuf.WriteString(m + "\n")
61+
combinedBufMu.Unlock()
5862
})
5963
close(stdoutDone)
6064
}()
6165
go func() {
6266
readStdPipe(stderr, func(m string) {
6367
stderrFunc(m)
68+
combinedBufMu.Lock()
6469
combinedBuf.WriteString(m + "\n")
70+
combinedBufMu.Unlock()
6571
})
6672
close(stderrDone)
6773
}()

‎framework/rpc/rpc.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414

1515
"github.com/ethereum/go-ethereum/core/types"
1616

17+
"github.com/smartcontractkit/chainlink-testing-framework/framework"
1718
f "github.com/smartcontractkit/chainlink-testing-framework/framework"
1819

1920
"github.com/testcontainers/testcontainers-go"
@@ -330,6 +331,7 @@ func StartAnvil(params []string) (*AnvilContainer, error) {
330331
entryPoint = append(entryPoint, params...)
331332
req := testcontainers.ContainerRequest{
332333
Image: "ghcr.io/foundry-rs/foundry:stable",
334+
Networks: []string{framework.DefaultNetworkName},
333335
ExposedPorts: []string{"8545/tcp"},
334336
WaitingFor: wait.ForListeningPort("8545").WithStartupTimeout(10 * time.Second),
335337
Entrypoint: entryPoint,

0 commit comments

Comments
 (0)