Add disk space reporting to runner capabilities
Some checks failed
CI / build-and-test (push) Failing after 4s

- Add DiskInfo struct with total, free, used bytes and usage percentage
- Detect disk space using unix.Statfs on startup
- Add periodic capabilities updates every 5 minutes
- Add disk space warnings at 85% (warning) and 95% (critical) thresholds
- Send updated capabilities to Gitea server periodically

This helps monitor runners that are running low on disk space.

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
GitCaddy 2026-01-11 05:16:49 +00:00
parent 3e22214bbd
commit ff8375c6e1
2 changed files with 103 additions and 0 deletions

View File

@ -30,6 +30,15 @@ import (
"gitea.com/gitea/act_runner/internal/pkg/ver"
)
const (
// DiskSpaceWarningThreshold is the percentage at which to warn about low disk space
DiskSpaceWarningThreshold = 85.0
// DiskSpaceCriticalThreshold is the percentage at which to log critical warnings
DiskSpaceCriticalThreshold = 95.0
// CapabilitiesUpdateInterval is how often to update capabilities (including disk space)
CapabilitiesUpdateInterval = 5 * time.Minute
)
func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) func(cmd *cobra.Command, args []string) error {
return func(cmd *cobra.Command, args []string) error {
cfg, err := config.LoadDefault(*configFile)
@ -147,6 +156,9 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
capabilitiesJson := capabilities.ToJSON()
log.Infof("detected capabilities: %s", capabilitiesJson)
// Check disk space and warn if low
checkDiskSpaceWarnings(capabilities)
// declare the labels of the runner before fetching tasks
resp, err := runner.Declare(ctx, ls.Names(), capabilitiesJson)
if err != nil && connect.CodeOf(err) == connect.CodeUnimplemented {
@ -160,6 +172,9 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
resp.Msg.Runner.Name, resp.Msg.Runner.Version, resp.Msg.Runner.Labels)
}
// Start periodic capabilities update goroutine
go periodicCapabilitiesUpdate(ctx, runner, ls.Names(), dockerHost)
poller := poll.New(cfg, cli, runner)
if daemArgs.Once || reg.Ephemeral {
@ -194,6 +209,53 @@ func runDaemon(ctx context.Context, daemArgs *daemonArgs, configFile *string) fu
}
}
// checkDiskSpaceWarnings logs warnings if disk space is low
func checkDiskSpaceWarnings(capabilities *envcheck.RunnerCapabilities) {
if capabilities.Disk == nil {
return
}
usedPercent := capabilities.Disk.UsedPercent
freeGB := float64(capabilities.Disk.Free) / (1024 * 1024 * 1024)
if usedPercent >= DiskSpaceCriticalThreshold {
log.Errorf("CRITICAL: Disk space critically low! %.1f%% used, only %.2f GB free. Runner may fail to execute jobs!", usedPercent, freeGB)
} else if usedPercent >= DiskSpaceWarningThreshold {
log.Warnf("WARNING: Disk space running low. %.1f%% used, %.2f GB free. Consider cleaning up disk space.", usedPercent, freeGB)
}
}
// periodicCapabilitiesUpdate periodically updates capabilities including disk space
func periodicCapabilitiesUpdate(ctx context.Context, runner *run.Runner, labelNames []string, dockerHost string) {
ticker := time.NewTicker(CapabilitiesUpdateInterval)
defer ticker.Stop()
for {
select {
case <-ctx.Done():
log.Debug("stopping periodic capabilities update")
return
case <-ticker.C:
// Detect updated capabilities (disk space changes over time)
capabilities := envcheck.DetectCapabilities(ctx, dockerHost)
capabilitiesJson := capabilities.ToJSON()
// Check for disk space warnings
checkDiskSpaceWarnings(capabilities)
// Send updated capabilities to server
_, err := runner.Declare(ctx, labelNames, capabilitiesJson)
if err != nil {
log.WithError(err).Debug("failed to update capabilities")
} else {
log.Debugf("capabilities updated: disk %.1f%% used, %.2f GB free",
capabilities.Disk.UsedPercent,
float64(capabilities.Disk.Free)/(1024*1024*1024))
}
}
}
}
type daemonArgs struct {
Once bool
}

View File

@ -12,8 +12,17 @@ import (
"time"
"github.com/docker/docker/client"
"golang.org/x/sys/unix"
)
// DiskInfo holds disk space information
type DiskInfo struct {
Total uint64 `json:"total_bytes"`
Free uint64 `json:"free_bytes"`
Used uint64 `json:"used_bytes"`
UsedPercent float64 `json:"used_percent"`
}
// RunnerCapabilities represents the capabilities of a runner for AI consumption
type RunnerCapabilities struct {
OS string `json:"os"`
@ -25,6 +34,7 @@ type RunnerCapabilities struct {
Tools map[string][]string `json:"tools,omitempty"`
Features *CapabilityFeatures `json:"features,omitempty"`
Limitations []string `json:"limitations,omitempty"`
Disk *DiskInfo `json:"disk,omitempty"`
}
// CapabilityFeatures represents feature support flags
@ -64,9 +74,40 @@ func DetectCapabilities(ctx context.Context, dockerHost string) *RunnerCapabilit
// Detect common tools
detectTools(ctx, cap)
// Detect disk space
cap.Disk = detectDiskSpace()
return cap
}
// detectDiskSpace detects disk space on the root filesystem
func detectDiskSpace() *DiskInfo {
var stat unix.Statfs_t
// Get stats for root filesystem (or current working directory)
path := "/"
if runtime.GOOS == "windows" {
path = "C:\\"
}
err := unix.Statfs(path, &stat)
if err != nil {
return nil
}
total := stat.Blocks * uint64(stat.Bsize)
free := stat.Bavail * uint64(stat.Bsize)
used := total - free
usedPercent := float64(used) / float64(total) * 100
return &DiskInfo{
Total: total,
Free: free,
Used: used,
UsedPercent: usedPercent,
}
}
// ToJSON converts capabilities to JSON string for transmission
func (c *RunnerCapabilities) ToJSON() string {
data, err := json.Marshal(c)