Add disk space reporting to runner capabilities
Some checks failed
CI / build-and-test (push) Failing after 4s
Some checks failed
CI / build-and-test (push) Failing after 4s
- Add DiskInfo struct with total, free, used bytes and usage percentage - Detect disk space using unix.Statfs on startup - Add periodic capabilities updates every 5 minutes - Add disk space warnings at 85% (warning) and 95% (critical) thresholds - Send updated capabilities to Gitea server periodically This helps monitor runners that are running low on disk space. Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
3e22214bbd
commit
ff8375c6e1
@ -30,6 +30,15 @@ import (
|
||||
"gitea.com/gitea/act_runner/internal/pkg/ver"
|
||||
)
|
||||
|
||||
const (
|
||||
// DiskSpaceWarningThreshold is the percentage at which to warn about low disk space
|
||||
DiskSpaceWarningThreshold = 85.0
|
||||
// DiskSpaceCriticalThreshold is the percentage at which to log critical warnings
|
||||
DiskSpaceCriticalThreshold = 95.0
|
||||
// CapabilitiesUpdateInterval is how often to update capabilities (including disk space)
|
||||
CapabilitiesUpdateInterval = 5 * time.Minute
|
||||
)
|
||||
|
||||
func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) func(cmd *cobra.Command, args []string) error {
|
||||
return func(cmd *cobra.Command, args []string) error {
|
||||
cfg, err := config.LoadDefault(*configFile)
|
||||
@ -147,6 +156,9 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
capabilitiesJson := capabilities.ToJSON()
|
||||
log.Infof("detected capabilities: %s", capabilitiesJson)
|
||||
|
||||
// Check disk space and warn if low
|
||||
checkDiskSpaceWarnings(capabilities)
|
||||
|
||||
// declare the labels of the runner before fetching tasks
|
||||
resp, err := runner.Declare(ctx, ls.Names(), capabilitiesJson)
|
||||
if err != nil && connect.CodeOf(err) == connect.CodeUnimplemented {
|
||||
@ -160,6 +172,9 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
resp.Msg.Runner.Name, resp.Msg.Runner.Version, resp.Msg.Runner.Labels)
|
||||
}
|
||||
|
||||
// Start periodic capabilities update goroutine
|
||||
go periodicCapabilitiesUpdate(ctx, runner, ls.Names(), dockerHost)
|
||||
|
||||
poller := poll.New(cfg, cli, runner)
|
||||
|
||||
if daemArgs.Once || reg.Ephemeral {
|
||||
@ -194,6 +209,53 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
|
||||
}
|
||||
}
|
||||
|
||||
// checkDiskSpaceWarnings logs warnings if disk space is low
|
||||
func checkDiskSpaceWarnings(capabilities *envcheck.RunnerCapabilities) {
|
||||
if capabilities.Disk == nil {
|
||||
return
|
||||
}
|
||||
|
||||
usedPercent := capabilities.Disk.UsedPercent
|
||||
freeGB := float64(capabilities.Disk.Free) / (1024 * 1024 * 1024)
|
||||
|
||||
if usedPercent >= DiskSpaceCriticalThreshold {
|
||||
log.Errorf("CRITICAL: Disk space critically low! %.1f%% used, only %.2f GB free. Runner may fail to execute jobs!", usedPercent, freeGB)
|
||||
} else if usedPercent >= DiskSpaceWarningThreshold {
|
||||
log.Warnf("WARNING: Disk space running low. %.1f%% used, %.2f GB free. Consider cleaning up disk space.", usedPercent, freeGB)
|
||||
}
|
||||
}
|
||||
|
||||
// periodicCapabilitiesUpdate periodically updates capabilities including disk space
|
||||
func periodicCapabilitiesUpdate(ctx context.Context, runner *run.Runner, labelNames []string, dockerHost string) {
|
||||
ticker := time.NewTicker(CapabilitiesUpdateInterval)
|
||||
defer ticker.Stop()
|
||||
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
log.Debug("stopping periodic capabilities update")
|
||||
return
|
||||
case <-ticker.C:
|
||||
// Detect updated capabilities (disk space changes over time)
|
||||
capabilities := envcheck.DetectCapabilities(ctx, dockerHost)
|
||||
capabilitiesJson := capabilities.ToJSON()
|
||||
|
||||
// Check for disk space warnings
|
||||
checkDiskSpaceWarnings(capabilities)
|
||||
|
||||
// Send updated capabilities to server
|
||||
_, err := runner.Declare(ctx, labelNames, capabilitiesJson)
|
||||
if err != nil {
|
||||
log.WithError(err).Debug("failed to update capabilities")
|
||||
} else {
|
||||
log.Debugf("capabilities updated: disk %.1f%% used, %.2f GB free",
|
||||
capabilities.Disk.UsedPercent,
|
||||
float64(capabilities.Disk.Free)/(1024*1024*1024))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type daemonArgs struct {
|
||||
Once bool
|
||||
}
|
||||
|
||||
@ -12,8 +12,17 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/docker/docker/client"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
// DiskInfo holds disk space information
|
||||
type DiskInfo struct {
|
||||
Total uint64 `json:"total_bytes"`
|
||||
Free uint64 `json:"free_bytes"`
|
||||
Used uint64 `json:"used_bytes"`
|
||||
UsedPercent float64 `json:"used_percent"`
|
||||
}
|
||||
|
||||
// RunnerCapabilities represents the capabilities of a runner for AI consumption
|
||||
type RunnerCapabilities struct {
|
||||
OS string `json:"os"`
|
||||
@ -25,6 +34,7 @@ type RunnerCapabilities struct {
|
||||
Tools map[string][]string `json:"tools,omitempty"`
|
||||
Features *CapabilityFeatures `json:"features,omitempty"`
|
||||
Limitations []string `json:"limitations,omitempty"`
|
||||
Disk *DiskInfo `json:"disk,omitempty"`
|
||||
}
|
||||
|
||||
// CapabilityFeatures represents feature support flags
|
||||
@ -64,9 +74,40 @@ func DetectCapabilities(ctx context.Context, dockerHost string) *RunnerCapabilit
|
||||
// Detect common tools
|
||||
detectTools(ctx, cap)
|
||||
|
||||
// Detect disk space
|
||||
cap.Disk = detectDiskSpace()
|
||||
|
||||
return cap
|
||||
}
|
||||
|
||||
// detectDiskSpace detects disk space on the root filesystem
|
||||
func detectDiskSpace() *DiskInfo {
|
||||
var stat unix.Statfs_t
|
||||
|
||||
// Get stats for root filesystem (or current working directory)
|
||||
path := "/"
|
||||
if runtime.GOOS == "windows" {
|
||||
path = "C:\\"
|
||||
}
|
||||
|
||||
err := unix.Statfs(path, &stat)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
total := stat.Blocks * uint64(stat.Bsize)
|
||||
free := stat.Bavail * uint64(stat.Bsize)
|
||||
used := total - free
|
||||
usedPercent := float64(used) / float64(total) * 100
|
||||
|
||||
return &DiskInfo{
|
||||
Total: total,
|
||||
Free: free,
|
||||
Used: used,
|
||||
UsedPercent: usedPercent,
|
||||
}
|
||||
}
|
||||
|
||||
// ToJSON converts capabilities to JSON string for transmission
|
||||
func (c *RunnerCapabilities) ToJSON() string {
|
||||
data, err := json.Marshal(c)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user