Skip to content

Commit a6ab774

Browse files
aatchisonclaude
andauthored
feat(alerting): warn when shared01 ArgoCD controller memory nears its limit (#55)
* feat(alerting): warn when shared01 ArgoCD controller memory nears its limit (platform-infrastructure#1343) Adds ArgoCDControllerMemoryNearLimit to the catalog-argocd-paging group: application-controller working set / memory limit > 0.8 for 15m on mzla-eks-shared01, severity=warning (Slack, no page). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> * feat(alerting): add ArgoCD controller OOMKilled warning, reword near-limit rule (platform-infrastructure#1343) The near-limit rule (>0.8 for 15m) only catches sustained growth; a 30-day replay shows 0 firings. ArgoCDControllerOOMKilled fires on a restart whose last termination reason is OOMKilled; the replay matches both OOMs (2026-09-23, 2026-10-06) and nothing else. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
1 parent b6c7292 commit a6ab774

1 file changed

Lines changed: 144 additions & 0 deletions

File tree

‎terraform/alerting-catalog-paging.tf‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1595,6 +1595,150 @@ resource "grafana_rule_group" "catalog_argocd" {
15951595
})
15961596
}
15971597
}
1598+
1599+
# --- application-controller memory near limit: catches sustained growth; short reconcile spikes are left to ArgoCDControllerOOMKilled ---
1600+
rule {
1601+
name = "ArgoCDControllerMemoryNearLimit"
1602+
condition = "C"
1603+
for = "15m"
1604+
no_data_state = "OK"
1605+
exec_err_state = "Error"
1606+
labels = {
1607+
severity = "warning"
1608+
cluster = "mzla-eks-shared01"
1609+
service = "argocd"
1610+
}
1611+
annotations = {
1612+
summary = "ArgoCD application-controller {{ $labels.pod }} memory is above 80% of its limit on shared01"
1613+
description = "ArgoCD application-controller pod {{ $labels.pod }} on shared01 has used more than 80% of its memory limit for 15 minutes straight. Its baseline has grown, so the next reconcile spike is likely to OOMKill it. An OOMKill restarts the controller and pauses reconcile/self-heal for every cluster until its caches rebuild. Warning-only (low-urgency PagerDuty, posted to Slack). Raise controller.resources.limits.memory in argocd/helm-values/argocd.yaml (platform-infrastructure) or reduce controller load."
1614+
runbook_url = "https://github.com/thunderbird/platform-infrastructure/issues/1343"
1615+
}
1616+
1617+
data {
1618+
ref_id = "A"
1619+
datasource_uid = var.prometheus_datasource_uid
1620+
relative_time_range {
1621+
from = 900
1622+
to = 0
1623+
}
1624+
model = jsonencode({
1625+
refId = "A"
1626+
datasource = { type = "prometheus", uid = var.prometheus_datasource_uid }
1627+
expr = "max by (pod) (container_memory_working_set_bytes{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\"}) / max by (pod) (kube_pod_container_resource_limits{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\",resource=\"memory\"})"
1628+
instant = true
1629+
intervalMs = 1000
1630+
maxDataPoints = 43200
1631+
})
1632+
}
1633+
data {
1634+
ref_id = "B"
1635+
datasource_uid = "__expr__"
1636+
relative_time_range {
1637+
from = 900
1638+
to = 0
1639+
}
1640+
model = jsonencode({
1641+
refId = "B"
1642+
type = "reduce"
1643+
expression = "A"
1644+
reducer = "last"
1645+
datasource = { type = "__expr__", uid = "__expr__" }
1646+
})
1647+
}
1648+
data {
1649+
ref_id = "C"
1650+
datasource_uid = "__expr__"
1651+
relative_time_range {
1652+
from = 900
1653+
to = 0
1654+
}
1655+
model = jsonencode({
1656+
refId = "C"
1657+
type = "threshold"
1658+
expression = "B"
1659+
datasource = { type = "__expr__", uid = "__expr__" }
1660+
conditions = [{
1661+
type = "query"
1662+
evaluator = { type = "gt", params = [0.8] }
1663+
operator = { type = "and" }
1664+
query = { params = ["C"] }
1665+
reducer = { type = "last", params = [] }
1666+
}]
1667+
})
1668+
}
1669+
}
1670+
1671+
# --- application-controller OOMKilled: it restarts in ~1s, so the up-based and CrashLoop rules miss it ---
1672+
rule {
1673+
name = "ArgoCDControllerOOMKilled"
1674+
condition = "C"
1675+
for = "0s"
1676+
no_data_state = "OK"
1677+
exec_err_state = "Error"
1678+
labels = {
1679+
severity = "warning"
1680+
cluster = "mzla-eks-shared01"
1681+
service = "argocd"
1682+
}
1683+
annotations = {
1684+
summary = "ArgoCD application-controller {{ $labels.pod }} was OOMKilled on shared01"
1685+
description = "ArgoCD application-controller pod {{ $labels.pod }} on shared01 was OOMKilled and restarted in the last 30 minutes. Reconcile/self-heal paused for every cluster while it rebuilt its caches; repeated OOMs stall it. Warning-only (low-urgency PagerDuty, posted to Slack). Raise controller.resources.limits.memory in argocd/helm-values/argocd.yaml (platform-infrastructure) or reduce controller load."
1686+
runbook_url = "https://github.com/thunderbird/platform-infrastructure/issues/1343"
1687+
}
1688+
1689+
data {
1690+
ref_id = "A"
1691+
datasource_uid = var.prometheus_datasource_uid
1692+
relative_time_range {
1693+
from = 1800
1694+
to = 0
1695+
}
1696+
model = jsonencode({
1697+
refId = "A"
1698+
datasource = { type = "prometheus", uid = var.prometheus_datasource_uid }
1699+
expr = "(max by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\"}[30m])) > 0) and on (pod) (max by (pod) (kube_pod_container_status_last_terminated_reason{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\",reason=\"OOMKilled\"}) == 1)"
1700+
instant = true
1701+
intervalMs = 1000
1702+
maxDataPoints = 43200
1703+
})
1704+
}
1705+
data {
1706+
ref_id = "B"
1707+
datasource_uid = "__expr__"
1708+
relative_time_range {
1709+
from = 1800
1710+
to = 0
1711+
}
1712+
model = jsonencode({
1713+
refId = "B"
1714+
type = "reduce"
1715+
expression = "A"
1716+
reducer = "last"
1717+
datasource = { type = "__expr__", uid = "__expr__" }
1718+
})
1719+
}
1720+
data {
1721+
ref_id = "C"
1722+
datasource_uid = "__expr__"
1723+
relative_time_range {
1724+
from = 1800
1725+
to = 0
1726+
}
1727+
model = jsonencode({
1728+
refId = "C"
1729+
type = "threshold"
1730+
expression = "B"
1731+
datasource = { type = "__expr__", uid = "__expr__" }
1732+
conditions = [{
1733+
type = "query"
1734+
evaluator = { type = "gt", params = [0] }
1735+
operator = { type = "and" }
1736+
query = { params = ["C"] }
1737+
reducer = { type = "last", params = [] }
1738+
}]
1739+
})
1740+
}
1741+
}
15981742
}
15991743

16001744
# ---------------------------------------------------------------------------

0 commit comments

Comments
 (0)