Skip to content

Commit c4c3a4d

Browse files
authored
feat(services): add per-service Docker resource limits (#53)
Mirror CashPilot #95 (2026-07-05 fleet OOM remediation): declare an optional docker.resources block (mem_limit, mem_reservation, oom_score_adj) in the service YAML and apply it on the container HostConfig at creation, so earner memory ceilings and OOM-kill priority are durable across restarts instead of set out-of-band. - catalog: parse docker.resources into DockerConfig.Resources (OomScoreAdj is *int so absent is distinguishable from an explicit 0) - runtime: set HostConfig Memory / MemoryReservation / OomScoreAdj from the block when present; memory-swap left unset. Binary-unit size parsing ("768m" = 768 MiB, "2g" = 2 GiB) matching docker --memory - services: protect storj (2g) / mysterium (768m) / anyone-protocol (1536m) with negative oom_score_adj; cap expendable earners (128-256m) with positive scores so the kernel reaps them first under pressure - tests: size parsing, HostConfig application, and YAML parse
1 parent d3140e6 commit c4c3a4d

20 files changed

Lines changed: 341 additions & 12 deletions

internal/catalog/catalog.go

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -41,18 +41,32 @@ type Referral struct {
4141
}
4242

4343
type DockerConfig struct {
44-
Image string `json:"image" yaml:"image"`
45-
Platforms []string `json:"platforms" yaml:"platforms"`
46-
Env []EnvVar `json:"env" yaml:"env"`
47-
Ports []string `json:"ports" yaml:"ports"`
48-
Volumes []string `json:"volumes" yaml:"volumes"`
49-
Command string `json:"command" yaml:"command"`
50-
NetworkMode string `json:"networkMode" yaml:"network_mode"`
51-
CapAdd []string `json:"capAdd" yaml:"cap_add"`
52-
Privileged bool `json:"privileged" yaml:"privileged"`
53-
StopTimeout int `json:"stopTimeout" yaml:"stop_timeout"`
54-
Setup string `json:"setup" yaml:"setup"`
55-
Notes string `json:"notes" yaml:"notes"`
44+
Image string `json:"image" yaml:"image"`
45+
Platforms []string `json:"platforms" yaml:"platforms"`
46+
Env []EnvVar `json:"env" yaml:"env"`
47+
Ports []string `json:"ports" yaml:"ports"`
48+
Volumes []string `json:"volumes" yaml:"volumes"`
49+
Command string `json:"command" yaml:"command"`
50+
NetworkMode string `json:"networkMode" yaml:"network_mode"`
51+
CapAdd []string `json:"capAdd" yaml:"cap_add"`
52+
Privileged bool `json:"privileged" yaml:"privileged"`
53+
StopTimeout int `json:"stopTimeout" yaml:"stop_timeout"`
54+
Resources ResourceLimits `json:"resources" yaml:"resources"`
55+
Setup string `json:"setup" yaml:"setup"`
56+
Notes string `json:"notes" yaml:"notes"`
57+
}
58+
59+
// ResourceLimits is the optional docker.resources block from a service YAML. Its
60+
// fields map to Docker HostConfig knobs applied at container creation (see
61+
// internal/runtime.applyResourceLimits) so a service's memory ceiling and OOM
62+
// priority survive restarts instead of being set out-of-band. MemLimit and
63+
// MemReservation are Docker-style size strings ("768m", "2g"); an empty string
64+
// leaves that limit unset. OomScoreAdj is a pointer so an absent value is
65+
// distinguishable from an explicit 0 and is applied only when present.
66+
type ResourceLimits struct {
67+
MemLimit string `json:"memLimit" yaml:"mem_limit"`
68+
MemReservation string `json:"memReservation" yaml:"mem_reservation"`
69+
OomScoreAdj *int `json:"oomScoreAdj" yaml:"oom_score_adj"`
5670
}
5771

5872
type EnvVar struct {

internal/catalog/catalog_test.go

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,3 +38,64 @@ docker:
3838
t.Fatal("docker-backed service should not be manual-only")
3939
}
4040
}
41+
42+
// TestLoadEmbeddedParsesDockerResources verifies the optional docker.resources
43+
// block maps into DockerConfig.Resources, and that a service without the block
44+
// leaves the fields at their zero values (empty strings and a nil OomScoreAdj, so
45+
// an absent OOM score is distinguishable from an explicit 0).
46+
func TestLoadEmbeddedParsesDockerResources(t *testing.T) {
47+
fsys := fstest.MapFS{
48+
"services/bandwidth/limited.yml": {
49+
Data: []byte(`
50+
name: Limited
51+
slug: limited
52+
category: bandwidth
53+
status: active
54+
docker:
55+
image: example/limited
56+
resources:
57+
mem_limit: "256m"
58+
mem_reservation: "128m"
59+
oom_score_adj: -100
60+
`),
61+
},
62+
"services/bandwidth/plain.yml": {
63+
Data: []byte(`
64+
name: Plain
65+
slug: plain
66+
category: bandwidth
67+
status: active
68+
docker:
69+
image: example/plain
70+
`),
71+
},
72+
}
73+
74+
cat, err := LoadEmbedded(fsys)
75+
if err != nil {
76+
t.Fatalf("LoadEmbedded returned error: %v", err)
77+
}
78+
79+
limited, ok := cat.Get("limited")
80+
if !ok {
81+
t.Fatal("expected limited service")
82+
}
83+
res := limited.Docker.Resources
84+
if res.MemLimit != "256m" {
85+
t.Fatalf("MemLimit = %q, want %q", res.MemLimit, "256m")
86+
}
87+
if res.MemReservation != "128m" {
88+
t.Fatalf("MemReservation = %q, want %q", res.MemReservation, "128m")
89+
}
90+
if res.OomScoreAdj == nil || *res.OomScoreAdj != -100 {
91+
t.Fatalf("OomScoreAdj = %v, want -100", res.OomScoreAdj)
92+
}
93+
94+
plain, ok := cat.Get("plain")
95+
if !ok {
96+
t.Fatal("expected plain service")
97+
}
98+
if pr := plain.Docker.Resources; pr.MemLimit != "" || pr.MemReservation != "" || pr.OomScoreAdj != nil {
99+
t.Fatalf("expected empty resources for a service without the block, got %+v", pr)
100+
}
101+
}

internal/runtime/runtime.go

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"io"
1010
"os/exec"
1111
goruntime "runtime"
12+
"strconv"
1213
"strings"
1314
"sync"
1415
"time"
@@ -172,6 +173,9 @@ func (p *DockerProvider) Deploy(ctx context.Context, spec DeploySpec, progress f
172173
if svc.Docker.NetworkMode == "" {
173174
hostConfig.NetworkMode = "bridge"
174175
}
176+
if err := applyResourceLimits(hostConfig, svc.Docker.Resources); err != nil {
177+
return ContainerInfo{}, err
178+
}
175179

176180
if progress != nil {
177181
progress("Creating " + name)
@@ -570,6 +574,77 @@ func buildMounts(raw []string, env map[string]string) []mount.Mount {
570574
return mounts
571575
}
572576

577+
// applyResourceLimits sets the optional memory and OOM-priority knobs from a
578+
// service's docker.resources block onto the container HostConfig. Each limit is
579+
// applied only when present in the YAML: an empty MemLimit/MemReservation leaves
580+
// Docker's default (unlimited) in place, and a nil OomScoreAdj leaves the daemon
581+
// default. Memory strings use Docker's binary units ("768m" = 768 MiB, "2g" =
582+
// 2 GiB), matching `docker run --memory` / compose mem_limit. memory-swap is left
583+
// unset (0) on purpose so Docker derives it from Memory rather than pinning swap.
584+
// It returns an error for a malformed size string so a bad service definition
585+
// fails fast at deploy instead of silently running unbounded.
586+
func applyResourceLimits(hostConfig *container.HostConfig, res catalog.ResourceLimits) error {
587+
if res.MemLimit != "" {
588+
bytes, err := parseMemoryBytes(res.MemLimit)
589+
if err != nil {
590+
return fmt.Errorf("invalid mem_limit %q: %w", res.MemLimit, err)
591+
}
592+
hostConfig.Memory = bytes
593+
}
594+
if res.MemReservation != "" {
595+
bytes, err := parseMemoryBytes(res.MemReservation)
596+
if err != nil {
597+
return fmt.Errorf("invalid mem_reservation %q: %w", res.MemReservation, err)
598+
}
599+
hostConfig.MemoryReservation = bytes
600+
}
601+
if res.OomScoreAdj != nil {
602+
hostConfig.OomScoreAdj = *res.OomScoreAdj
603+
}
604+
return nil
605+
}
606+
607+
// parseMemoryBytes converts a Docker-style memory size string into a byte count
608+
// using binary units, matching `docker run --memory` and compose mem_limit: a bare
609+
// number is bytes, and a k/m/g/t suffix (case-insensitive, with an optional
610+
// trailing "b", e.g. "768m" or "2gb") multiplies by 1024, 1024^2, 1024^3 or
611+
// 1024^4. So "768m" is 768*1024*1024 = 805306368 bytes and "2g" is 2147483648. A
612+
// fractional mantissa ("1.5g") is allowed and truncated toward zero. It returns an
613+
// error for an empty string, a non-positive value, or an unparseable number.
614+
func parseMemoryBytes(s string) (int64, error) {
615+
raw := strings.ToLower(strings.TrimSpace(s))
616+
if raw == "" {
617+
return 0, errors.New("empty memory value")
618+
}
619+
// An explicit trailing "b" ("768mb") is treated the same as the bare unit.
620+
raw = strings.TrimSuffix(raw, "b")
621+
if raw == "" {
622+
return 0, fmt.Errorf("%q has no numeric value", s)
623+
}
624+
var multiplier int64 = 1
625+
switch raw[len(raw)-1] {
626+
case 'k':
627+
multiplier = 1 << 10
628+
case 'm':
629+
multiplier = 1 << 20
630+
case 'g':
631+
multiplier = 1 << 30
632+
case 't':
633+
multiplier = 1 << 40
634+
}
635+
if multiplier != 1 {
636+
raw = raw[:len(raw)-1]
637+
}
638+
value, err := strconv.ParseFloat(strings.TrimSpace(raw), 64)
639+
if err != nil {
640+
return 0, fmt.Errorf("%q is not a valid size: %w", s, err)
641+
}
642+
if value <= 0 {
643+
return 0, fmt.Errorf("%q must be a positive size", s)
644+
}
645+
return int64(value * float64(multiplier)), nil
646+
}
647+
573648
func managedContainerVolumes(ctx context.Context, cli *client.Client, name string) ([]string, error) {
574649
inspect, err := cli.ContainerInspect(ctx, name)
575650
if err != nil {

internal/runtime/runtime_test.go

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,3 +658,132 @@ func TestDockerStatsIntegrationReportsLiveCPU(t *testing.T) {
658658
t.Fatalf("expected memory > 0 MB for a running container, got %v", mem)
659659
}
660660
}
661+
662+
// oomPtr returns a pointer to i for building catalog.ResourceLimits.OomScoreAdj
663+
// (a *int, so an absent value is distinguishable from an explicit 0) in tests.
664+
func oomPtr(i int) *int { return &i }
665+
666+
// TestParseMemoryBytes pins the Docker-style binary-unit parsing used for the
667+
// docker.resources mem_limit/mem_reservation strings: k/m/g/t are 1024-based (so
668+
// "768m" is 768 MiB, not 768 MB), an optional trailing "b", surrounding
669+
// whitespace, mixed case and a fractional mantissa are accepted, and
670+
// empty/non-positive/garbage values are rejected so a bad service definition fails
671+
// fast instead of deploying an earner unbounded.
672+
func TestParseMemoryBytes(t *testing.T) {
673+
cases := []struct {
674+
in string
675+
want int64
676+
wantErr bool
677+
}{
678+
{in: "768m", want: 768 * 1024 * 1024}, // mysterium
679+
{in: "2g", want: 2 * 1024 * 1024 * 1024}, // storj
680+
{in: "1536m", want: 1536 * 1024 * 1024}, // anyone-protocol
681+
{in: "256m", want: 256 * 1024 * 1024}, // honeygain/earnapp/proxyrack
682+
{in: "128m", want: 128 * 1024 * 1024}, // expendable earners
683+
{in: "512k", want: 512 * 1024}, // kibibytes
684+
{in: "1024", want: 1024}, // bare byte count, no suffix
685+
{in: "2gb", want: 2 * 1024 * 1024 * 1024}, // explicit trailing "b"
686+
{in: "768M", want: 768 * 1024 * 1024}, // case-insensitive suffix
687+
{in: " 256m ", want: 256 * 1024 * 1024}, // surrounding whitespace
688+
{in: "1.5g", want: 1536 * 1024 * 1024}, // fractional mantissa
689+
{in: "", wantErr: true}, // empty
690+
{in: "b", wantErr: true}, // unit only, no number
691+
{in: "m", wantErr: true}, // unit only, no number
692+
{in: "abc", wantErr: true}, // not a number
693+
{in: "0m", wantErr: true}, // non-positive
694+
{in: "-5m", wantErr: true}, // negative
695+
}
696+
for _, tc := range cases {
697+
got, err := parseMemoryBytes(tc.in)
698+
if tc.wantErr {
699+
if err == nil {
700+
t.Errorf("parseMemoryBytes(%q) = %d, want error", tc.in, got)
701+
}
702+
continue
703+
}
704+
if err != nil {
705+
t.Errorf("parseMemoryBytes(%q) unexpected error: %v", tc.in, err)
706+
continue
707+
}
708+
if got != tc.want {
709+
t.Errorf("parseMemoryBytes(%q) = %d, want %d", tc.in, got, tc.want)
710+
}
711+
}
712+
}
713+
714+
// TestApplyResourceLimitsSetsHostConfig verifies a protected service's
715+
// docker.resources block lands on the container HostConfig: the hard memory
716+
// ceiling is parsed to bytes, the negative OOM score is applied, and memory-swap
717+
// is deliberately left unset (0) so Docker derives it from Memory instead of
718+
// pinning swap.
719+
func TestApplyResourceLimitsSetsHostConfig(t *testing.T) {
720+
hc := &container.HostConfig{}
721+
res := catalog.ResourceLimits{MemLimit: "2g", OomScoreAdj: oomPtr(-100)}
722+
if err := applyResourceLimits(hc, res); err != nil {
723+
t.Fatalf("applyResourceLimits returned error: %v", err)
724+
}
725+
if want := int64(2 * 1024 * 1024 * 1024); hc.Memory != want {
726+
t.Fatalf("Memory = %d, want %d", hc.Memory, want)
727+
}
728+
if hc.OomScoreAdj != -100 {
729+
t.Fatalf("OomScoreAdj = %d, want -100", hc.OomScoreAdj)
730+
}
731+
if hc.MemorySwap != 0 {
732+
t.Fatalf("MemorySwap = %d, want 0 (must be left unset)", hc.MemorySwap)
733+
}
734+
if hc.MemoryReservation != 0 {
735+
t.Fatalf("MemoryReservation = %d, want 0 (not specified)", hc.MemoryReservation)
736+
}
737+
}
738+
739+
// TestApplyResourceLimitsSetsReservation covers the optional soft limit: when
740+
// mem_reservation is present it is parsed and set alongside the hard mem_limit.
741+
func TestApplyResourceLimitsSetsReservation(t *testing.T) {
742+
hc := &container.HostConfig{}
743+
res := catalog.ResourceLimits{MemLimit: "768m", MemReservation: "256m", OomScoreAdj: oomPtr(200)}
744+
if err := applyResourceLimits(hc, res); err != nil {
745+
t.Fatalf("applyResourceLimits returned error: %v", err)
746+
}
747+
if want := int64(768 * 1024 * 1024); hc.Memory != want {
748+
t.Fatalf("Memory = %d, want %d", hc.Memory, want)
749+
}
750+
if want := int64(256 * 1024 * 1024); hc.MemoryReservation != want {
751+
t.Fatalf("MemoryReservation = %d, want %d", hc.MemoryReservation, want)
752+
}
753+
if hc.OomScoreAdj != 200 {
754+
t.Fatalf("OomScoreAdj = %d, want 200", hc.OomScoreAdj)
755+
}
756+
}
757+
758+
// TestApplyResourceLimitsAbsentLeavesDefaults verifies an empty docker.resources
759+
// block touches nothing: memory stays unlimited (0) and OomScoreAdj stays at the
760+
// daemon default (0). A nil OomScoreAdj must be left alone rather than written as
761+
// an explicit 0.
762+
func TestApplyResourceLimitsAbsentLeavesDefaults(t *testing.T) {
763+
hc := &container.HostConfig{}
764+
if err := applyResourceLimits(hc, catalog.ResourceLimits{}); err != nil {
765+
t.Fatalf("applyResourceLimits returned error: %v", err)
766+
}
767+
if hc.Memory != 0 || hc.MemoryReservation != 0 || hc.MemorySwap != 0 || hc.OomScoreAdj != 0 {
768+
t.Fatalf("expected all limits unset, got Memory=%d MemoryReservation=%d MemorySwap=%d OomScoreAdj=%d",
769+
hc.Memory, hc.MemoryReservation, hc.MemorySwap, hc.OomScoreAdj)
770+
}
771+
}
772+
773+
// TestApplyResourceLimitsRejectsBadValue makes a malformed size fail the deploy
774+
// (rather than silently running the earner unbounded) and leaves HostConfig
775+
// unmodified.
776+
func TestApplyResourceLimitsRejectsBadValue(t *testing.T) {
777+
for _, res := range []catalog.ResourceLimits{
778+
{MemLimit: "notasize"},
779+
{MemReservation: "12x"},
780+
} {
781+
hc := &container.HostConfig{}
782+
if err := applyResourceLimits(hc, res); err == nil {
783+
t.Errorf("applyResourceLimits(%+v) = nil error, want error", res)
784+
}
785+
if hc.Memory != 0 || hc.MemoryReservation != 0 {
786+
t.Errorf("HostConfig mutated on error: %+v", hc)
787+
}
788+
}
789+
}

services/_schema.yml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,11 @@
3232
# network_mode: "" (optional, e.g. "host")
3333
# cap_add: [] (optional)
3434
# stop_timeout: 30 (optional, seconds to wait before SIGKILL on stop — default 30)
35+
# resources: (optional, per-container limits applied at creation on HostConfig)
36+
# mem_limit: "768m" (hard memory ceiling; Docker size string like "256m"/"2g". Omit = unlimited)
37+
# mem_reservation: "256m" (optional soft memory limit; Docker size string)
38+
# oom_score_adj: -100 (optional OOM-killer priority, -1000..1000; negative protects the
39+
# container, positive makes the kernel kill it first under memory pressure)
3540
# setup: "" (optional, one-time setup instructions/commands to run before first deploy)
3641

3742
# requirements:

services/bandwidth/bitping.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,9 @@ docker:
2424
command: ""
2525
network_mode: ""
2626
privileged: false
27+
resources:
28+
mem_limit: "128m"
29+
oom_score_adj: 200
2730
notes: >
2831
Initial authentication is interactive via the container's web UI.
2932
Credentials are then persisted in the /root/.bitping volume mount.

services/bandwidth/earnapp.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,9 @@ docker:
3434
command: ""
3535
network_mode: ""
3636
privileged: false
37+
resources:
38+
mem_limit: "256m"
39+
oom_score_adj: 300
3740

3841
requirements:
3942
residential_ip: true

services/bandwidth/earnfm.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ docker:
2626
command: ""
2727
network_mode: ""
2828
privileged: false
29+
resources:
30+
mem_limit: "128m"
31+
oom_score_adj: 200
2932

3033
requirements:
3134
residential_ip: true

services/bandwidth/honeygain.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,9 @@ docker:
3838
command: "-tou-accept -email ${HONEYGAIN_EMAIL} -pass ${HONEYGAIN_PASSWORD} -device ${HONEYGAIN_DEVICE_NAME}"
3939
network_mode: ""
4040
privileged: false
41+
resources:
42+
mem_limit: "256m"
43+
oom_score_adj: 200
4144

4245
requirements:
4346
residential_ip: true

0 commit comments

Comments
 (0)