You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
Repository navigation
Commit a6ab774
Browse filesBrowse the repository at this point in the historyBrowse files
feat(alerting): warn when shared01 ArgoCD controller memory nears its limit (#55)
* feat(alerting): warn when shared01 ArgoCD controller memory nears its limit (platform-infrastructure#1343)
Adds ArgoCDControllerMemoryNearLimit to the catalog-argocd-paging group:
application-controller working set / memory limit > 0.8 for 15m on
mzla-eks-shared01, severity=warning (Slack, no page).
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
* feat(alerting): add ArgoCD controller OOMKilled warning, reword near-limit rule (platform-infrastructure#1343)
The near-limit rule (>0.8 for 15m) only catches sustained growth; a 30-day
replay shows 0 firings. ArgoCDControllerOOMKilled fires on a restart whose
last termination reason is OOMKilled; the replay matches both OOMs
(2026-09-23, 2026-10-06) and nothing else.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
---------
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
# --- application-controller memory near limit: catches sustained growth; short reconcile spikes are left to ArgoCDControllerOOMKilled ---
1600
+
rule {
1601
+
name="ArgoCDControllerMemoryNearLimit"
1602
+
condition="C"
1603
+
for="15m"
1604
+
no_data_state="OK"
1605
+
exec_err_state="Error"
1606
+
labels={
1607
+
severity ="warning"
1608
+
cluster ="mzla-eks-shared01"
1609
+
service ="argocd"
1610
+
}
1611
+
annotations={
1612
+
summary ="ArgoCD application-controller {{ $labels.pod }} memory is above 80% of its limit on shared01"
1613
+
description ="ArgoCD application-controller pod {{ $labels.pod }} on shared01 has used more than 80% of its memory limit for 15 minutes straight. Its baseline has grown, so the next reconcile spike is likely to OOMKill it. An OOMKill restarts the controller and pauses reconcile/self-heal for every cluster until its caches rebuild. Warning-only (low-urgency PagerDuty, posted to Slack). Raise controller.resources.limits.memory in argocd/helm-values/argocd.yaml (platform-infrastructure) or reduce controller load."
datasource = { type ="prometheus", uid = var.prometheus_datasource_uid }
1627
+
expr ="max by (pod) (container_memory_working_set_bytes{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\"}) / max by (pod) (kube_pod_container_resource_limits{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\",resource=\"memory\"})"
1628
+
instant =true
1629
+
intervalMs =1000
1630
+
maxDataPoints =43200
1631
+
})
1632
+
}
1633
+
data {
1634
+
ref_id="B"
1635
+
datasource_uid="__expr__"
1636
+
relative_time_range {
1637
+
from=900
1638
+
to=0
1639
+
}
1640
+
model=jsonencode({
1641
+
refId ="B"
1642
+
type ="reduce"
1643
+
expression ="A"
1644
+
reducer ="last"
1645
+
datasource = { type ="__expr__", uid ="__expr__" }
1646
+
})
1647
+
}
1648
+
data {
1649
+
ref_id="C"
1650
+
datasource_uid="__expr__"
1651
+
relative_time_range {
1652
+
from=900
1653
+
to=0
1654
+
}
1655
+
model=jsonencode({
1656
+
refId ="C"
1657
+
type ="threshold"
1658
+
expression ="B"
1659
+
datasource = { type ="__expr__", uid ="__expr__" }
1660
+
conditions = [{
1661
+
type ="query"
1662
+
evaluator = { type ="gt", params = [0.8] }
1663
+
operator = { type ="and" }
1664
+
query = { params = ["C"] }
1665
+
reducer = { type ="last", params = [] }
1666
+
}]
1667
+
})
1668
+
}
1669
+
}
1670
+
1671
+
# --- application-controller OOMKilled: it restarts in ~1s, so the up-based and CrashLoop rules miss it ---
1672
+
rule {
1673
+
name="ArgoCDControllerOOMKilled"
1674
+
condition="C"
1675
+
for="0s"
1676
+
no_data_state="OK"
1677
+
exec_err_state="Error"
1678
+
labels={
1679
+
severity ="warning"
1680
+
cluster ="mzla-eks-shared01"
1681
+
service ="argocd"
1682
+
}
1683
+
annotations={
1684
+
summary ="ArgoCD application-controller {{ $labels.pod }} was OOMKilled on shared01"
1685
+
description ="ArgoCD application-controller pod {{ $labels.pod }} on shared01 was OOMKilled and restarted in the last 30 minutes. Reconcile/self-heal paused for every cluster while it rebuilt its caches; repeated OOMs stall it. Warning-only (low-urgency PagerDuty, posted to Slack). Raise controller.resources.limits.memory in argocd/helm-values/argocd.yaml (platform-infrastructure) or reduce controller load."
datasource = { type ="prometheus", uid = var.prometheus_datasource_uid }
1699
+
expr ="(max by (pod) (increase(kube_pod_container_status_restarts_total{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\"}[30m])) > 0) and on (pod) (max by (pod) (kube_pod_container_status_last_terminated_reason{cluster=\"mzla-eks-shared01\",namespace=\"argocd\",container=\"application-controller\",reason=\"OOMKilled\"}) == 1)"
1700
+
instant =true
1701
+
intervalMs =1000
1702
+
maxDataPoints =43200
1703
+
})
1704
+
}
1705
+
data {
1706
+
ref_id="B"
1707
+
datasource_uid="__expr__"
1708
+
relative_time_range {
1709
+
from=1800
1710
+
to=0
1711
+
}
1712
+
model=jsonencode({
1713
+
refId ="B"
1714
+
type ="reduce"
1715
+
expression ="A"
1716
+
reducer ="last"
1717
+
datasource = { type ="__expr__", uid ="__expr__" }
1718
+
})
1719
+
}
1720
+
data {
1721
+
ref_id="C"
1722
+
datasource_uid="__expr__"
1723
+
relative_time_range {
1724
+
from=1800
1725
+
to=0
1726
+
}
1727
+
model=jsonencode({
1728
+
refId ="C"
1729
+
type ="threshold"
1730
+
expression ="B"
1731
+
datasource = { type ="__expr__", uid ="__expr__" }
0 commit comments