571 lines
18 KiB
Go
571 lines
18 KiB
Go
package main
|
||
|
||
import (
|
||
"encoding/json"
|
||
"fmt"
|
||
"log"
|
||
"os"
|
||
"os/exec"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// ── Типы ─────────────────────────────────────────────────────────────────────
|
||
|
||
type dcContainerInfo struct {
|
||
ID string
|
||
Name string
|
||
Status string // running, exited, dead, restarting, paused, created
|
||
Health string // healthy, unhealthy, starting, none
|
||
Image string
|
||
}
|
||
|
||
type dcStateFile struct {
|
||
Timestamp string `json:"timestamp"`
|
||
Containers map[string]dcContainerState `json:"containers"` // name → state
|
||
}
|
||
|
||
type dcContainerState struct {
|
||
Status string `json:"status"`
|
||
Health string `json:"health"`
|
||
}
|
||
|
||
type dcChangeKind string
|
||
|
||
const (
|
||
dcKindStatusChanged dcChangeKind = "status_changed"
|
||
dcKindHealthChanged dcChangeKind = "health_changed"
|
||
dcKindDisappeared dcChangeKind = "disappeared"
|
||
dcKindAppeared dcChangeKind = "appeared"
|
||
)
|
||
|
||
type dcStateChange struct {
|
||
Name string
|
||
Kind dcChangeKind
|
||
PrevState dcContainerState
|
||
CurrState dcContainerState
|
||
Image string
|
||
}
|
||
|
||
func (ch dcStateChange) isProblem() bool {
|
||
switch ch.Kind {
|
||
case dcKindDisappeared:
|
||
return true
|
||
case dcKindStatusChanged:
|
||
return ch.CurrState.Status != "running"
|
||
case dcKindHealthChanged:
|
||
return ch.CurrState.Health == "unhealthy"
|
||
}
|
||
return false
|
||
}
|
||
|
||
func (ch dcStateChange) isNew() bool {
|
||
return ch.Kind == dcKindAppeared
|
||
}
|
||
|
||
func (ch dcStateChange) isRecovery() bool {
|
||
switch ch.Kind {
|
||
case dcKindStatusChanged:
|
||
return ch.CurrState.Status == "running" && ch.PrevState.Status != "running"
|
||
case dcKindHealthChanged:
|
||
return ch.CurrState.Health == "healthy" && ch.PrevState.Health == "unhealthy"
|
||
}
|
||
return false
|
||
}
|
||
|
||
// ── Точка входа ───────────────────────────────────────────────────────────────
|
||
|
||
func runDockerCheck() error {
|
||
hostname, err := os.Hostname()
|
||
if err != nil {
|
||
hostname = "unknown"
|
||
log.Printf("⚠️ Не удалось получить hostname: %v", err)
|
||
}
|
||
|
||
log.Printf("🔍 DockerCheck запущен на хосте: %s", hostname)
|
||
|
||
// Проверяем доступность docker
|
||
if err := dcCheckDockerAvailable(); err != nil {
|
||
return fmt.Errorf("docker недоступен: %v", err)
|
||
}
|
||
|
||
// Получаем текущий список контейнеров
|
||
containers, err := dcGetContainers()
|
||
if err != nil {
|
||
return fmt.Errorf("ошибка получения списка контейнеров: %v", err)
|
||
}
|
||
|
||
if len(containers) == 0 {
|
||
log.Println("ℹ️ Запущенных контейнеров не найдено")
|
||
}
|
||
|
||
log.Printf("📦 Найдено контейнеров: %d", len(containers))
|
||
|
||
// Выводим текущее состояние
|
||
for _, c := range containers {
|
||
emoji := dcStatusEmoji(c.Status, c.Health)
|
||
if c.Health != "" && c.Health != "none" {
|
||
log.Printf("%s %-30s | %-12s | health: %s | %s", emoji, c.Name, c.Status, c.Health, c.Image)
|
||
} else {
|
||
log.Printf("%s %-30s | %-12s | %s", emoji, c.Name, c.Status, c.Image)
|
||
}
|
||
}
|
||
|
||
// Загружаем предыдущее состояние
|
||
prevState := dcLoadState()
|
||
isFirstRun := len(prevState.Containers) == 0
|
||
|
||
// Строим map текущего состояния
|
||
current := make(map[string]dcContainerInfo, len(containers))
|
||
for _, c := range containers {
|
||
current[c.Name] = c
|
||
}
|
||
|
||
// Сравниваем
|
||
changes := dcCompareStates(prevState.Containers, current, isFirstRun)
|
||
|
||
// Статистика
|
||
runningCount := 0
|
||
problemCount := 0
|
||
for _, c := range containers {
|
||
if c.Status == "running" && c.Health != "unhealthy" {
|
||
runningCount++
|
||
} else {
|
||
problemCount++
|
||
}
|
||
}
|
||
|
||
log.Println("\n📊 Итоги проверки контейнеров:")
|
||
log.Printf(" ✅ Работают нормально: %d", runningCount)
|
||
log.Printf(" ❌ Проблемных: %d", problemCount)
|
||
log.Printf(" 🔄 Изменений статуса: %d", len(changes))
|
||
|
||
if len(changes) > 0 {
|
||
log.Println("\n🔄 Изменения с последней проверки:")
|
||
log.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
||
for _, ch := range changes {
|
||
arrow := dcChangeArrow(ch)
|
||
switch ch.Kind {
|
||
case dcKindAppeared:
|
||
log.Printf(" %s %-30s | появился (%s) | %s", arrow, ch.Name, ch.CurrState.Status, ch.Image)
|
||
case dcKindDisappeared:
|
||
log.Printf(" %s %-30s | исчез (был: %s)", arrow, ch.Name, ch.PrevState.Status)
|
||
case dcKindStatusChanged:
|
||
log.Printf(" %s %-30s | %s → %s", arrow, ch.Name, ch.PrevState.Status, ch.CurrState.Status)
|
||
case dcKindHealthChanged:
|
||
log.Printf(" %s %-30s | health: %s → %s", arrow, ch.Name, ch.PrevState.Health, ch.CurrState.Health)
|
||
}
|
||
}
|
||
log.Println("━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━")
|
||
|
||
if isTelegramConfigured() {
|
||
dcSendAlert(hostname, changes)
|
||
} else {
|
||
log.Println("\nℹ️ Telegram уведомления отключены (не настроены данные)")
|
||
}
|
||
} else {
|
||
if len(prevState.Containers) == 0 {
|
||
log.Println("\nℹ️ Первый запуск — состояние сохранено, уведомления при изменениях")
|
||
} else {
|
||
log.Println("\n✅ Изменений нет — уведомления не отправляются")
|
||
}
|
||
}
|
||
|
||
// Сохраняем текущее состояние
|
||
dcSaveState(current)
|
||
|
||
log.Println("=== Проверка контейнеров завершена ===")
|
||
return nil
|
||
}
|
||
|
||
// ── Получение контейнеров ────────────────────────────────────────────────────
|
||
|
||
func dcCheckDockerAvailable() error {
|
||
cmd := exec.Command("docker", "info")
|
||
out, err := cmd.CombinedOutput()
|
||
if err != nil {
|
||
return fmt.Errorf("docker info завершился с ошибкой: %v\n%s", err, string(out))
|
||
}
|
||
return nil
|
||
}
|
||
|
||
func dcGetContainers() ([]dcContainerInfo, error) {
|
||
// Шаг 1: получаем ID, Name, Status, Image без .Health — оно доступно
|
||
// только через docker inspect и отсутствует у контейнеров без healthcheck.
|
||
cmd := exec.Command(
|
||
"docker", "ps", "-a",
|
||
"--format", "{{.ID}}|{{.Names}}|{{.Status}}|{{.Image}}",
|
||
)
|
||
out, err := cmd.CombinedOutput()
|
||
if err != nil {
|
||
return nil, fmt.Errorf("docker ps завершился с ошибкой: %v\n%s", err, string(out))
|
||
}
|
||
|
||
var containers []dcContainerInfo
|
||
for _, line := range strings.Split(strings.TrimSpace(string(out)), "\n") {
|
||
if line == "" {
|
||
continue
|
||
}
|
||
parts := strings.SplitN(line, "|", 4)
|
||
if len(parts) != 4 {
|
||
continue
|
||
}
|
||
|
||
c := dcContainerInfo{
|
||
ID: strings.TrimSpace(parts[0]),
|
||
Name: strings.TrimSpace(parts[1]),
|
||
Image: strings.TrimSpace(parts[3]),
|
||
}
|
||
c.Status = dcNormalizeStatus(parts[2])
|
||
containers = append(containers, c)
|
||
}
|
||
|
||
// Шаг 2: получаем health отдельно через docker inspect для каждого контейнера.
|
||
// Контейнеры без healthcheck вернут пустую строку — это нормально.
|
||
for i := range containers {
|
||
containers[i].Health = dcGetHealth(containers[i].ID)
|
||
}
|
||
|
||
return containers, nil
|
||
}
|
||
|
||
// dcGetHealth возвращает health статус контейнера через docker inspect.
|
||
// Если healthcheck не настроен или inspect завершился с ошибкой — возвращает "none".
|
||
func dcGetHealth(containerID string) string {
|
||
out, err := exec.Command(
|
||
"docker", "inspect",
|
||
"--format", "{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}",
|
||
containerID,
|
||
).Output()
|
||
if err != nil {
|
||
return "none"
|
||
}
|
||
health := strings.ToLower(strings.TrimSpace(string(out)))
|
||
if health == "" {
|
||
return "none"
|
||
}
|
||
return health
|
||
}
|
||
|
||
// dcNormalizeStatus приводит вывод docker ps к короткому статусу
|
||
func dcNormalizeStatus(raw string) string {
|
||
raw = strings.TrimSpace(raw)
|
||
lower := strings.ToLower(raw)
|
||
switch {
|
||
case strings.HasPrefix(lower, "up"):
|
||
return "running"
|
||
case strings.HasPrefix(lower, "exited"):
|
||
return "exited"
|
||
case strings.HasPrefix(lower, "restarting"):
|
||
return "restarting"
|
||
case strings.HasPrefix(lower, "dead"):
|
||
return "dead"
|
||
case strings.HasPrefix(lower, "paused"):
|
||
return "paused"
|
||
case strings.HasPrefix(lower, "created"):
|
||
return "created"
|
||
case strings.HasPrefix(lower, "removal"):
|
||
return "removing"
|
||
default:
|
||
return lower
|
||
}
|
||
}
|
||
|
||
// ── Сравнение состояний ──────────────────────────────────────────────────────
|
||
|
||
func dcCompareStates(prev map[string]dcContainerState, curr map[string]dcContainerInfo, isFirstRun bool) []dcStateChange {
|
||
var changes []dcStateChange
|
||
|
||
// Проверяем текущие контейнеры
|
||
for name, currInfo := range curr {
|
||
prevInfo, existed := prev[name]
|
||
|
||
if !existed {
|
||
if !isFirstRun {
|
||
// Новый контейнер появился после первого запуска — алертим
|
||
changes = append(changes, dcStateChange{
|
||
Name: name,
|
||
Kind: dcKindAppeared,
|
||
PrevState: dcContainerState{},
|
||
CurrState: dcContainerState{Status: currInfo.Status, Health: currInfo.Health},
|
||
Image: currInfo.Image,
|
||
})
|
||
}
|
||
continue
|
||
}
|
||
|
||
currState := dcContainerState{Status: currInfo.Status, Health: currInfo.Health}
|
||
|
||
// Изменение статуса
|
||
if prevInfo.Status != currState.Status {
|
||
changes = append(changes, dcStateChange{
|
||
Name: name,
|
||
Kind: dcKindStatusChanged,
|
||
PrevState: prevInfo,
|
||
CurrState: currState,
|
||
Image: currInfo.Image,
|
||
})
|
||
}
|
||
|
||
// Изменение health (независимо от статуса)
|
||
if prevInfo.Health != currState.Health &&
|
||
currState.Health != "" && currState.Health != "none" &&
|
||
prevInfo.Health != "" && prevInfo.Health != "none" {
|
||
changes = append(changes, dcStateChange{
|
||
Name: name,
|
||
Kind: dcKindHealthChanged,
|
||
PrevState: prevInfo,
|
||
CurrState: currState,
|
||
Image: currInfo.Image,
|
||
})
|
||
}
|
||
}
|
||
|
||
// Проверяем исчезнувшие контейнеры (были в state, нет сейчас)
|
||
for name, prevInfo := range prev {
|
||
if _, exists := curr[name]; !exists {
|
||
changes = append(changes, dcStateChange{
|
||
Name: name,
|
||
Kind: dcKindDisappeared,
|
||
PrevState: prevInfo,
|
||
CurrState: dcContainerState{},
|
||
Image: "",
|
||
})
|
||
}
|
||
}
|
||
|
||
return changes
|
||
}
|
||
|
||
// ── State-файл ────────────────────────────────────────────────────────────────
|
||
|
||
func dcLoadState() dcStateFile {
|
||
data, err := os.ReadFile(cfgDockerStateFilePath)
|
||
if err != nil {
|
||
return dcStateFile{Containers: make(map[string]dcContainerState)}
|
||
}
|
||
|
||
var state dcStateFile
|
||
if err := json.Unmarshal(data, &state); err != nil {
|
||
log.Printf("⚠️ Не удалось прочитать state-файл docker: %v", err)
|
||
return dcStateFile{Containers: make(map[string]dcContainerState)}
|
||
}
|
||
|
||
return state
|
||
}
|
||
|
||
func dcSaveState(current map[string]dcContainerInfo) {
|
||
containers := make(map[string]dcContainerState, len(current))
|
||
for name, info := range current {
|
||
containers[name] = dcContainerState{
|
||
Status: info.Status,
|
||
Health: info.Health,
|
||
}
|
||
}
|
||
|
||
state := dcStateFile{
|
||
Timestamp: time.Now().Format("2006-01-02 15:04:05"),
|
||
Containers: containers,
|
||
}
|
||
|
||
data, err := json.MarshalIndent(state, "", " ")
|
||
if err != nil {
|
||
log.Printf("⚠️ Ошибка сериализации state-файла docker: %v", err)
|
||
return
|
||
}
|
||
|
||
if err := os.WriteFile(cfgDockerStateFilePath, data, 0644); err != nil {
|
||
log.Printf("⚠️ Ошибка записи state-файла %s: %v", cfgDockerStateFilePath, err)
|
||
}
|
||
}
|
||
|
||
// ── Отправка алерта ──────────────────────────────────────────────────────────
|
||
|
||
func dcSendAlert(hostname string, changes []dcStateChange) {
|
||
var problems, recoveries, appeared []dcStateChange
|
||
for _, ch := range changes {
|
||
switch {
|
||
case ch.isNew():
|
||
appeared = append(appeared, ch)
|
||
case ch.isRecovery():
|
||
recoveries = append(recoveries, ch)
|
||
case ch.isProblem():
|
||
problems = append(problems, ch)
|
||
}
|
||
}
|
||
|
||
// Если нет никаких изменений — молчим
|
||
if len(problems) == 0 && len(recoveries) == 0 && len(appeared) == 0 {
|
||
return
|
||
}
|
||
|
||
// Заголовок по наиболее критичному событию
|
||
var header string
|
||
switch {
|
||
case len(problems) > 0:
|
||
header = "🚨 <b>АЛЕРТ: Проблемы с контейнерами!</b>"
|
||
case len(recoveries) > 0 && len(appeared) > 0:
|
||
header = "⚠️ <b>Изменения в состоянии контейнеров</b>"
|
||
case len(recoveries) > 0:
|
||
header = "✅ <b>Контейнеры восстановлены</b>"
|
||
default:
|
||
header = "ℹ️ <b>Новые контейнеры</b>"
|
||
}
|
||
|
||
message := fmt.Sprintf(
|
||
"%s\n\n"+
|
||
"🖥 <b>Хост:</b> <code>%s</code>\n"+
|
||
"⏰ <b>Время:</b> %s\n\n",
|
||
header,
|
||
hostname,
|
||
time.Now().Format("2006-01-02 15:04:05"),
|
||
)
|
||
|
||
// Проблемные контейнеры
|
||
if len(problems) > 0 {
|
||
message += "━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
|
||
message += fmt.Sprintf("❌ <b>Проблемы (%d):</b>\n\n", len(problems))
|
||
for _, ch := range problems {
|
||
emoji := dcChangeEmoji(ch)
|
||
switch ch.Kind {
|
||
case dcKindDisappeared:
|
||
message += fmt.Sprintf(
|
||
"%s <b>%s</b>\n"+
|
||
" 📌 Контейнер исчез (был: <code>%s</code>)\n\n",
|
||
emoji, ch.Name, ch.PrevState.Status,
|
||
)
|
||
case dcKindStatusChanged:
|
||
message += fmt.Sprintf(
|
||
"%s <b>%s</b>\n"+
|
||
" 📌 <code>%s</code> → <code>%s</code>\n"+
|
||
" 🐳 <code>%s</code>\n"+
|
||
" 🔧 <code>docker logs %s --tail 50</code>\n\n",
|
||
emoji, ch.Name,
|
||
ch.PrevState.Status, ch.CurrState.Status,
|
||
ch.Image, ch.Name,
|
||
)
|
||
case dcKindHealthChanged:
|
||
message += fmt.Sprintf(
|
||
"%s <b>%s</b>\n"+
|
||
" 📌 health: <code>%s</code> → <code>%s</code>\n"+
|
||
" 🐳 <code>%s</code>\n"+
|
||
" 🔧 <code>docker inspect %s</code>\n\n",
|
||
emoji, ch.Name,
|
||
ch.PrevState.Health, ch.CurrState.Health,
|
||
ch.Image, ch.Name,
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Восстановленные контейнеры
|
||
if len(recoveries) > 0 {
|
||
message += "━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
|
||
message += fmt.Sprintf("✅ <b>Восстановились (%d):</b>\n\n", len(recoveries))
|
||
for _, ch := range recoveries {
|
||
switch ch.Kind {
|
||
case dcKindStatusChanged:
|
||
message += fmt.Sprintf(
|
||
"✅ <b>%s</b>\n"+
|
||
" 📌 <code>%s</code> → <code>%s</code>\n\n",
|
||
ch.Name, ch.PrevState.Status, ch.CurrState.Status,
|
||
)
|
||
case dcKindHealthChanged:
|
||
message += fmt.Sprintf(
|
||
"✅ <b>%s</b>\n"+
|
||
" 📌 health: <code>%s</code> → <code>%s</code>\n\n",
|
||
ch.Name, ch.PrevState.Health, ch.CurrState.Health,
|
||
)
|
||
}
|
||
}
|
||
}
|
||
|
||
// Новые контейнеры
|
||
if len(appeared) > 0 {
|
||
message += "━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
|
||
message += fmt.Sprintf("🆕 <b>Появились (%d):</b>\n\n", len(appeared))
|
||
for _, ch := range appeared {
|
||
message += fmt.Sprintf(
|
||
"🆕 <b>%s</b>\n"+
|
||
" 📌 Статус: <code>%s</code>\n"+
|
||
" 🐳 <code>%s</code>\n\n",
|
||
ch.Name, ch.CurrState.Status, ch.Image,
|
||
)
|
||
}
|
||
}
|
||
|
||
message += "━━━━━━━━━━━━━━━━━━━━━━━━━━━\n"
|
||
message += "💡 <i>docker ps -a | docker logs <имя> --tail 50</i>"
|
||
|
||
if err := telegramSendHTML(message); err != nil {
|
||
log.Printf("❌ Ошибка отправки в Telegram: %v", err)
|
||
} else {
|
||
log.Println("✅ Алерт успешно отправлен в Telegram")
|
||
}
|
||
if err := emailSendHTML("Auspex: Проблемы с контейнерами", message); err != nil {
|
||
log.Printf("❌ Ошибка отправки Email: %v", err)
|
||
} else {
|
||
log.Println("✅ Алерт успешно отправлен на Email")
|
||
}
|
||
if err := mattermostSend(message); err != nil {
|
||
log.Printf("❌ Ошибка отправки в Mattermost: %v", err)
|
||
} else {
|
||
log.Println("✅ Алерт успешно отправлен в Mattermost")
|
||
}
|
||
}
|
||
|
||
// ── Вспомогательные функции ──────────────────────────────────────────────────
|
||
|
||
func dcStatusEmoji(status, health string) string {
|
||
if health == "unhealthy" {
|
||
return "🤒"
|
||
}
|
||
switch status {
|
||
case "running":
|
||
return "✅"
|
||
case "restarting":
|
||
return "🔄"
|
||
case "exited":
|
||
return "🔴"
|
||
case "dead":
|
||
return "💀"
|
||
case "paused":
|
||
return "⏸"
|
||
default:
|
||
return "❓"
|
||
}
|
||
}
|
||
|
||
func dcChangeEmoji(ch dcStateChange) string {
|
||
switch ch.Kind {
|
||
case dcKindAppeared:
|
||
return "🆕"
|
||
case dcKindDisappeared:
|
||
return "👻"
|
||
case dcKindHealthChanged:
|
||
return "🤒"
|
||
case dcKindStatusChanged:
|
||
switch ch.CurrState.Status {
|
||
case "restarting":
|
||
return "🔄"
|
||
case "exited":
|
||
return "🔴"
|
||
case "dead":
|
||
return "💀"
|
||
}
|
||
}
|
||
return "❓"
|
||
}
|
||
|
||
func dcChangeArrow(ch dcStateChange) string {
|
||
if ch.isNew() {
|
||
return "🆕"
|
||
}
|
||
if ch.isRecovery() {
|
||
return "✅"
|
||
}
|
||
if ch.isProblem() {
|
||
return "❌"
|
||
}
|
||
return "🔄"
|
||
}
|