From d27ce19f6c17bd84146fb9e14c57995899479373 Mon Sep 17 00:00:00 2001 From: Jesse Duffield Date: Wed, 11 May 2022 10:42:00 +1000 Subject: [PATCH] simplify CPU monitoring logic --- go.mod | 1 - go.sum | 2 - pkg/commands/container.go | 47 +++-------- pkg/commands/container_stats.go | 51 +++++++++--- pkg/commands/container_stats_test.go | 17 ++++ pkg/commands/docker.go | 78 ++++--------------- pkg/gui/gui.go | 15 ++-- vendor/github.com/acarl005/stripansi/LICENSE | 21 ----- .../github.com/acarl005/stripansi/README.md | 30 ------- .../acarl005/stripansi/stripansi.go | 13 ---- vendor/modules.txt | 1 - 11 files changed, 91 insertions(+), 185 deletions(-) create mode 100644 pkg/commands/container_stats_test.go delete mode 100644 vendor/github.com/acarl005/stripansi/LICENSE delete mode 100644 vendor/github.com/acarl005/stripansi/README.md delete mode 100644 vendor/github.com/acarl005/stripansi/stripansi.go diff --git a/go.mod b/go.mod index 2a821d84..284cbc4c 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.18 require ( github.com/OpenPeeDeeP/xdg v0.2.1-0.20190312153938-4ba9e1eb294c - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1 github.com/cloudfoundry/jibber_jabber v0.0.0-20151120183258-bcc4c8345a21 github.com/docker/docker v0.7.3-0.20190307005417-54dddadc7d5d diff --git a/go.sum b/go.sum index 0e320053..1901c7e0 100644 --- a/go.sum +++ b/go.sum @@ -6,8 +6,6 @@ github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+q github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/OpenPeeDeeP/xdg v0.2.1-0.20190312153938-4ba9e1eb294c h1:YDsGA6tou+tAxVe0Dre29iSbQ8TrWdWfwOisKArJT5E= github.com/OpenPeeDeeP/xdg v0.2.1-0.20190312153938-4ba9e1eb294c/go.mod h1:tMoSueLQlMf0TCldjrJLNIjAc5qAOIcHt5REi88/Ygo= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1 h1:1fx+RA5lk1ZkzPAUP7DEgZnVHYxEcHO77vQO/V8z/2Q= github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1/go.mod h1:z0nyIb42Zs97wyX1V+8MbEFhHeTw1OgFQfR6q57ZuHc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= diff --git a/pkg/commands/container.go b/pkg/commands/container.go index 6374581d..b043c658 100644 --- a/pkg/commands/container.go +++ b/pkg/commands/container.go @@ -6,6 +6,7 @@ import ( "os/exec" "strconv" "strings" + "sync" "time" "github.com/docker/docker/api/types/container" @@ -38,12 +39,13 @@ type Container struct { OSCommand *OSCommand Config *config.AppConfig Log *logrus.Entry - CLIStats ContainerCliStat // for realtime we use the CLI, for long-term we use the client - StatHistory []RecordedStats + StatHistory []*RecordedStats Details Details MonitoringStats bool DockerCommand LimitedDockerCommand Tr *i18n.TranslationSet + + StatsMutex sync.Mutex } // Details is a struct containing what we get back from `docker inspect` on a container @@ -230,19 +232,6 @@ type Details struct { } `json:"NetworkSettings"` } -// ContainerCliStat is a stat object returned by the CLI docker stat command -type ContainerCliStat struct { - BlockIO string `json:"BlockIO"` - CPUPerc string `json:"CPUPerc"` - Container string `json:"Container"` - ID string `json:"ID"` - MemPerc string `json:"MemPerc"` - MemUsage string `json:"MemUsage"` - Name string `json:"Name"` - NetIO string `json:"NetIO"` - PIDs string `json:"PIDs"` -} - // GetDisplayStrings returns the dispaly string of Container func (c *Container) GetDisplayStrings(isFocused bool) []string { image := strings.TrimPrefix(c.Container.Image, "sha256:") @@ -285,17 +274,13 @@ func (c *Container) getHealthStatus() string { // GetDisplayCPUPerc colors the cpu percentage based on how extreme it is func (c *Container) GetDisplayCPUPerc() string { - stats := c.CLIStats - - if stats.CPUPerc == "" { + stats, ok := c.getLastStats() + if !ok { return "" } - percentage, err := strconv.ParseFloat(strings.TrimSuffix(stats.CPUPerc, "%"), 32) - if err != nil { - // probably complaining about not being able to convert '--' - return "" - } + percentage := stats.DerivedStats.CPUPercentage + formattedPercentage := fmt.Sprintf("%.2f%%", stats.DerivedStats.CPUPercentage) var clr color.Attribute if percentage > 90 { @@ -306,7 +291,7 @@ func (c *Container) GetDisplayCPUPerc() string { clr = color.FgWhite } - return utils.ColoredString(stats.CPUPerc, clr) + return utils.ColoredString(formattedPercentage, clr) } // ProducingLogs tells us whether we should bother checking a container's logs @@ -400,20 +385,6 @@ func (c *Container) Top() (container.ContainerTopOKBody, error) { return c.Client.ContainerTop(context.Background(), c.ID, []string{}) } -// EraseOldHistory removes any history before the user-specified max duration -func (c *Container) EraseOldHistory() { - if c.Config.UserConfig.Stats.MaxDuration == 0 { - return - } - - for i, stat := range c.StatHistory { - if time.Since(stat.RecordedAt) < c.Config.UserConfig.Stats.MaxDuration { - c.StatHistory = c.StatHistory[i:] - return - } - } -} - // ViewLogs attaches to a subprocess viewing the container's logs func (c *Container) ViewLogs() (*exec.Cmd, error) { templateString := c.OSCommand.Config.UserConfig.CommandTemplates.ViewContainerLogs diff --git a/pkg/commands/container_stats.go b/pkg/commands/container_stats.go index 34817a4d..5caab8d9 100644 --- a/pkg/commands/container_stats.go +++ b/pkg/commands/container_stats.go @@ -149,9 +149,8 @@ type ContainerStats struct { func (s *ContainerStats) CalculateContainerCPUPercentage() float64 { cpuUsageDelta := s.CPUStats.CPUUsage.TotalUsage - s.PrecpuStats.CPUUsage.TotalUsage cpuTotalUsageDelta := s.CPUStats.SystemCPUUsage - s.PrecpuStats.SystemCPUUsage - numberOfCores := len(s.CPUStats.CPUUsage.PercpuUsage) - value := float64(cpuUsageDelta*100) * float64(numberOfCores) / float64(cpuTotalUsageDelta) + value := float64(cpuUsageDelta*100) / float64(cpuTotalUsageDelta) if math.IsNaN(value) { return 0 } @@ -169,11 +168,10 @@ func (s *ContainerStats) CalculateContainerMemoryUsage() float64 { // RenderStats returns a string containing the rendered stats of the container func (c *Container) RenderStats(viewWidth int) (string, error) { - history := c.StatHistory - if len(history) == 0 { + stats, ok := c.getLastStats() + if !ok { return "", nil } - currentStats := history[len(history)-1] graphSpecs := c.Config.UserConfig.Stats.Graphs graphs := make([]string, len(graphSpecs)) @@ -185,11 +183,11 @@ func (c *Container) RenderStats(viewWidth int) (string, error) { graphs[i] = utils.ColoredString(graph, utils.GetColorAttribute(spec.Color)) } - pidsCount := fmt.Sprintf("PIDs: %d", currentStats.ClientStats.PidsStats.Current) - dataReceived := fmt.Sprintf("Traffic received: %s", utils.FormatDecimalBytes(currentStats.ClientStats.Networks.Eth0.RxBytes)) - dataSent := fmt.Sprintf("Traffic sent: %s", utils.FormatDecimalBytes(currentStats.ClientStats.Networks.Eth0.TxBytes)) + pidsCount := fmt.Sprintf("PIDs: %d", stats.ClientStats.PidsStats.Current) + dataReceived := fmt.Sprintf("Traffic received: %s", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.RxBytes)) + dataSent := fmt.Sprintf("Traffic sent: %s", utils.FormatDecimalBytes(stats.ClientStats.Networks.Eth0.TxBytes)) - originalJSON, err := json.MarshalIndent(currentStats, "", " ") + originalJSON, err := json.MarshalIndent(stats, "", " ") if err != nil { return "", err } @@ -205,8 +203,43 @@ func (c *Container) RenderStats(viewWidth int) (string, error) { return contents, nil } +func (c *Container) appendStats(stats *RecordedStats) { + c.StatsMutex.Lock() + defer c.StatsMutex.Unlock() + + c.StatHistory = append(c.StatHistory, stats) + c.eraseOldHistory() +} + +// eraseOldHistory removes any history before the user-specified max duration +func (c *Container) eraseOldHistory() { + if c.Config.UserConfig.Stats.MaxDuration == 0 { + return + } + + for i, stat := range c.StatHistory { + if time.Since(stat.RecordedAt) < c.Config.UserConfig.Stats.MaxDuration { + c.StatHistory = c.StatHistory[i:] + return + } + } +} + +func (c *Container) getLastStats() (*RecordedStats, bool) { + c.StatsMutex.Lock() + defer c.StatsMutex.Unlock() + history := c.StatHistory + if len(history) == 0 { + return nil, false + } + return history[len(history)-1], true +} + // PlotGraph returns the plotted graph based on the graph spec and the stat history func (c *Container) PlotGraph(spec config.GraphConfig, width int) (string, error) { + c.StatsMutex.Lock() + defer c.StatsMutex.Unlock() + data := make([]float64, len(c.StatHistory)) max := spec.Max diff --git a/pkg/commands/container_stats_test.go b/pkg/commands/container_stats_test.go new file mode 100644 index 00000000..5aaabd47 --- /dev/null +++ b/pkg/commands/container_stats_test.go @@ -0,0 +1,17 @@ +package commands + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCalculateContainerCPUPercentage(t *testing.T) { + container := &ContainerStats{} + container.CPUStats.CPUUsage.TotalUsage = 10 + container.CPUStats.SystemCPUUsage = 10 + container.PrecpuStats.CPUUsage.TotalUsage = 5 + container.PrecpuStats.SystemCPUUsage = 2 + + assert.EqualValues(t, 62.5, container.CalculateContainerCPUPercentage()) +} diff --git a/pkg/commands/docker.go b/pkg/commands/docker.go index d1383ca0..b6848395 100644 --- a/pkg/commands/docker.go +++ b/pkg/commands/docker.go @@ -13,7 +13,6 @@ import ( "sync" "time" - "github.com/acarl005/stripansi" "github.com/docker/docker/api/types" "github.com/docker/docker/client" "github.com/imdario/mergo" @@ -121,65 +120,20 @@ func (c *DockerCommand) Close() error { return utils.CloseMany(c.Closers) } -// MonitorContainerStats is a function -func (c *DockerCommand) MonitorContainerStats() { - // TODO: pass in a stop channel to these so we don't restart every time we come back from a subprocess - go c.MonitorCLIContainerStats() - go c.MonitorClientContainerStats() -} - -// MonitorCLIContainerStats monitors a stream of container stats and updates the containers as each new stats object is received -func (c *DockerCommand) MonitorCLIContainerStats() { - onError := func(err error) { - c.ErrorChan <- err - time.Sleep(2 * time.Second) - } - - for { - command := `docker stats --all --no-trunc --format '{{json .}}'` - cmd := c.OSCommand.RunCustomCommand(command) - - r, err := cmd.StdoutPipe() - if err != nil { - onError(err) - continue - } - - _ = cmd.Start() - - scanner := bufio.NewScanner(r) - scanner.Split(bufio.ScanLines) - for scanner.Scan() { - var stats ContainerCliStat - // need to strip ANSI codes because uses escape sequences to clear the screen with each refresh - cleanString := stripansi.Strip(scanner.Text()) - if err := json.Unmarshal([]byte(cleanString), &stats); err != nil { - onError(err) - continue - } - c.ContainerMutex.Lock() - for _, container := range c.Containers { - if container.ID == stats.ID { - container.CLIStats = stats - } - } - c.ContainerMutex.Unlock() - } - - _ = cmd.Wait() - } -} - -// MonitorClientContainerStats is a function -func (c *DockerCommand) MonitorClientContainerStats() { +func (c *DockerCommand) MonitorContainerStats(ctx context.Context) { // periodically loop through running containers and see if we need to create a monitor goroutine for any // every second we check if we need to spawn a new goroutine ticker := time.NewTicker(time.Second) defer ticker.Stop() - for range ticker.C { - for _, container := range c.Containers { - if !container.MonitoringStats { - go c.createClientStatMonitor(container) + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + for _, container := range c.Containers { + if !container.MonitoringStats { + go c.createClientStatMonitor(container) + } } } } @@ -189,7 +143,10 @@ func (c *DockerCommand) createClientStatMonitor(container *Container) { container.MonitoringStats = true stream, err := c.Client.ContainerStats(context.Background(), container.ID, true) if err != nil { - c.ErrorChan <- err + // not creating error panel because if we've disconnected from docker we'll + // have already created an error panel + c.Log.Error(err) + container.MonitoringStats = false return } @@ -201,7 +158,7 @@ func (c *DockerCommand) createClientStatMonitor(container *Container) { var stats ContainerStats _ = json.Unmarshal(data, &stats) - recordedStats := RecordedStats{ + recordedStats := &RecordedStats{ ClientStats: stats, DerivedStats: DerivedStats{ CPUPercentage: stats.CalculateContainerCPUPercentage(), @@ -210,10 +167,7 @@ func (c *DockerCommand) createClientStatMonitor(container *Container) { RecordedAt: time.Now(), } - c.ContainerMutex.Lock() - container.StatHistory = append(container.StatHistory, recordedStats) - container.EraseOldHistory() - c.ContainerMutex.Unlock() + container.appendStats(recordedStats) } container.MonitoringStats = false diff --git a/pkg/gui/gui.go b/pkg/gui/gui.go index 0d6fe6a9..63784ce4 100644 --- a/pkg/gui/gui.go +++ b/pkg/gui/gui.go @@ -235,10 +235,11 @@ func (gui *Gui) Run() error { throttledRefresh := throttle.ThrottleFunc(time.Millisecond*50, true, gui.refresh) defer throttledRefresh.Stop() - finish := make(chan struct{}) - defer func() { close(finish) }() + ctx, finish := context.WithCancel(context.Background()) + defer finish() - go gui.listenForEvents(finish, throttledRefresh.Trigger) + go gui.listenForEvents(ctx, throttledRefresh.Trigger) + go gui.DockerCommand.MonitorContainerStats(ctx) go func() { gui.waitForIntro.Wait() @@ -247,11 +248,9 @@ func (gui *Gui) Run() error { gui.goEvery(time.Millisecond*30, gui.reRenderMain) gui.goEvery(time.Millisecond*1000, gui.DockerCommand.UpdateContainerDetails) gui.goEvery(time.Millisecond*1000, gui.checkForContextChange) - gui.goEvery(time.Millisecond*2000, gui.rerenderContainersAndServices) + gui.goEvery(time.Millisecond*1000, gui.rerenderContainersAndServices) }() - gui.DockerCommand.MonitorContainerStats() - go func() { for err := range gui.ErrorChan { if err == nil { @@ -301,7 +300,7 @@ func (gui *Gui) refresh() { }() } -func (gui *Gui) listenForEvents(finish chan struct{}, refresh func()) { +func (gui *Gui) listenForEvents(ctx context.Context, refresh func()) { errorCount := 0 onError := func(err error) { @@ -336,7 +335,7 @@ outer: for { select { - case <-finish: + case <-ctx.Done(): return case message := <-messageChan: // We could be more granular about what events should trigger which refreshes. diff --git a/vendor/github.com/acarl005/stripansi/LICENSE b/vendor/github.com/acarl005/stripansi/LICENSE deleted file mode 100644 index 00abe0db..00000000 --- a/vendor/github.com/acarl005/stripansi/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2018 Andrew Carlson - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/vendor/github.com/acarl005/stripansi/README.md b/vendor/github.com/acarl005/stripansi/README.md deleted file mode 100644 index 8bdb1f50..00000000 --- a/vendor/github.com/acarl005/stripansi/README.md +++ /dev/null @@ -1,30 +0,0 @@ -Strip ANSI -========== - -This Go package removes ANSI escape codes from strings. - -Ideally, we would prevent these from appearing in any text we want to process. -However, sometimes this can't be helped, and we need to be able to deal with that noise. -This will use a regexp to remove those unwanted escape codes. - - -## Install - -```sh -$ go get -u github.com/acarl005/stripansi -``` - -## Usage - -```go -import ( - "fmt" - "github.com/acarl005/stripansi" -) - -func main() { - msg := "\x1b[38;5;140m foo\x1b[0m bar" - cleanMsg := stripansi.Strip(msg) - fmt.Println(cleanMsg) // " foo bar" -} -``` diff --git a/vendor/github.com/acarl005/stripansi/stripansi.go b/vendor/github.com/acarl005/stripansi/stripansi.go deleted file mode 100644 index 235732a7..00000000 --- a/vendor/github.com/acarl005/stripansi/stripansi.go +++ /dev/null @@ -1,13 +0,0 @@ -package stripansi - -import ( - "regexp" -) - -const ansi = "[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))" - -var re = regexp.MustCompile(ansi) - -func Strip(str string) string { - return re.ReplaceAllString(str, "") -} diff --git a/vendor/modules.txt b/vendor/modules.txt index d8d11e20..ee991184 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -9,7 +9,6 @@ github.com/Microsoft/go-winio/pkg/guid github.com/OpenPeeDeeP/xdg # github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d ## explicit -github.com/acarl005/stripansi # github.com/boz/go-throttle v0.0.0-20160922054636-fdc4eab740c1 ## explicit github.com/boz/go-throttle