Skip to content

Commit 3cef037

Browse files
zivnevozivnevo
authored
Expose Prometheus-scrapable ports to all pods in the cluster (#434)
* Expose Prometheus-scrapable ports to all pods in the cluster * Testing more interesting cases of Prometheus usage * avoid iterating over annotations for each svc port --------- Signed-off-by: Ziv Nevo <[email protected]>
1 parent b0fdbbe commit 3cef037

File tree

10 files changed

+154
-33
lines changed

10 files changed

+154
-33
lines changed

‎pkg/analyzer/connections.go

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -19,21 +19,32 @@ func discoverConnections(resources []*Resource, links []*Service, logger Logger)
1919
logger.Debugf("services matched to %v: %v", destRes.Resource.Name, deploymentServices)
2020
for _, svc := range deploymentServices {
2121
srcRes := findSource(resources, svc)
22-
if len(srcRes) > 0 {
23-
for _, r := range srcRes {
24-
if !r.equals(destRes) {
25-
logger.Debugf("source: %s target: %s link: %s", r.Resource.Name, destRes.Resource.Name, svc.Resource.Name)
26-
connections = append(connections, &Connections{Source: r, Target: destRes, Link: svc})
27-
}
22+
for _, r := range srcRes {
23+
if !r.equals(destRes) {
24+
logger.Debugf("source: %s target: %s link: %s", r.Resource.Name, destRes.Resource.Name, svc.Resource.Name)
25+
connections = append(connections, &Connections{Source: r, Target: destRes, Link: svc})
2826
}
29-
} else {
27+
}
28+
if len(srcRes) == 0 || svcHasExposedPorts(svc) { // found no sources, but some ports need to be exposed
3029
connections = append(connections, &Connections{Target: destRes, Link: svc}) // indicates a source-less service
3130
}
3231
}
3332
}
3433
return connections
3534
}
3635

36+
func svcHasExposedPorts(svc *Service) bool {
37+
if svc.Resource.ExposeExternally {
38+
return true
39+
}
40+
for _, port := range svc.Resource.Network {
41+
if port.exposeToCluster {
42+
return true
43+
}
44+
}
45+
return false
46+
}
47+
3748
// areSelectorsContained returns true if selectors2 is contained in selectors1
3849
func areSelectorsContained(selectors1 map[string]string, selectors2 []string) bool {
3950
elementMap := make(map[string]string)

‎pkg/analyzer/info_to_resource.go

Lines changed: 43 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
2222
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
2323
"k8s.io/apimachinery/pkg/runtime"
24+
"k8s.io/apimachinery/pkg/util/intstr"
2425
"k8s.io/apimachinery/pkg/util/validation"
2526
"k8s.io/cli-runtime/pkg/resource"
2627
)
@@ -104,16 +105,40 @@ func k8sServiceFromInfo(info *resource.Info) (*Service, error) {
104105
serviceCtx.Resource.Type = svcObj.Spec.Type
105106
serviceCtx.Resource.Selectors = matchLabelSelectorToStrLabels(svcObj.Spec.Selector)
106107
serviceCtx.Resource.ExposeExternally = (svcObj.Spec.Type == v1.ServiceTypeLoadBalancer || svcObj.Spec.Type == v1.ServiceTypeNodePort)
107-
serviceCtx.Resource.ExposeToCluster = false
108108

109+
prometheusPort, prometheusPortValid := exposedPrometheusScrapePort(svcObj.Annotations)
109110
for _, p := range svcObj.Spec.Ports {
110-
n := SvcNetworkAttr{Port: int(p.Port), TargetPort: p.TargetPort, Protocol: p.Protocol}
111+
n := SvcNetworkAttr{Port: int(p.Port), TargetPort: p.TargetPort, Protocol: p.Protocol, name: p.Name}
112+
n.exposeToCluster = prometheusPortValid && n.equals(prometheusPort)
111113
serviceCtx.Resource.Network = append(serviceCtx.Resource.Network, n)
112114
}
113115

114116
return &serviceCtx, nil
115117
}
116118

119+
const defaultPrometheusScrapePort = 9090
120+
121+
func exposedPrometheusScrapePort(annotations map[string]string) (*intstr.IntOrString, bool) {
122+
scrapeOn := false
123+
scrapePort := intstr.FromInt(defaultPrometheusScrapePort)
124+
for k, v := range annotations {
125+
if strings.Contains(k, "prometheus") {
126+
if strings.HasSuffix(k, "/scrape") && v == "true" {
127+
scrapeOn = true
128+
} else if strings.HasSuffix(k, "/port") {
129+
scrapePort = intstr.Parse(v)
130+
}
131+
}
132+
}
133+
134+
return &scrapePort, scrapeOn
135+
}
136+
137+
func (port *SvcNetworkAttr) equals(intStrPort *intstr.IntOrString) bool {
138+
return (port.name != "" && port.name == intStrPort.StrVal) ||
139+
(port.Port > 0 && port.Port == int(intStrPort.IntVal))
140+
}
141+
117142
// ocRouteFromInfo updates servicesToExpose based on an OpenShift Route object
118143
func ocRouteFromInfo(info *resource.Info, toExpose servicesToExpose) error {
119144
routeObj := parseResourceFromInfo[ocroutev1.Route](info)
@@ -123,12 +148,12 @@ func ocRouteFromInfo(info *resource.Info, toExpose servicesToExpose) error {
123148

124149
exposedServicesInNamespace, ok := toExpose[routeObj.Namespace]
125150
if !ok {
126-
toExpose[routeObj.Namespace] = map[string]bool{}
151+
toExpose[routeObj.Namespace] = map[string][]*intstr.IntOrString{}
127152
exposedServicesInNamespace = toExpose[routeObj.Namespace]
128153
}
129-
exposedServicesInNamespace[routeObj.Spec.To.Name] = false
154+
appendToSliceInMap(exposedServicesInNamespace, routeObj.Spec.To.Name, &routeObj.Spec.Port.TargetPort)
130155
for _, backend := range routeObj.Spec.AlternateBackends {
131-
exposedServicesInNamespace[backend.Name] = false
156+
appendToSliceInMap(exposedServicesInNamespace, backend.Name, &routeObj.Spec.Port.TargetPort)
132157
}
133158

134159
return nil
@@ -143,13 +168,14 @@ func k8sIngressFromInfo(info *resource.Info, toExpose servicesToExpose) error {
143168

144169
exposedServicesInNamespace, ok := toExpose[ingressObj.Namespace]
145170
if !ok {
146-
toExpose[ingressObj.Namespace] = map[string]bool{}
171+
toExpose[ingressObj.Namespace] = map[string][]*intstr.IntOrString{}
147172
exposedServicesInNamespace = toExpose[ingressObj.Namespace]
148173
}
149174

150175
defaultBackend := ingressObj.Spec.DefaultBackend
151176
if defaultBackend != nil && defaultBackend.Service != nil {
152-
exposedServicesInNamespace[defaultBackend.Service.Name] = false
177+
portToAppend := portFromServiceBackendPort(&defaultBackend.Service.Port)
178+
appendToSliceInMap(exposedServicesInNamespace, defaultBackend.Service.Name, portToAppend)
153179
}
154180

155181
for ruleIdx := range ingressObj.Spec.Rules {
@@ -158,7 +184,8 @@ func k8sIngressFromInfo(info *resource.Info, toExpose servicesToExpose) error {
158184
for pathIdx := range rule.HTTP.Paths {
159185
svc := rule.HTTP.Paths[pathIdx].Backend.Service
160186
if svc != nil {
161-
exposedServicesInNamespace[svc.Name] = false
187+
portToAppend := portFromServiceBackendPort(&svc.Port)
188+
appendToSliceInMap(exposedServicesInNamespace, svc.Name, portToAppend)
162189
}
163190
}
164191
}
@@ -167,6 +194,14 @@ func k8sIngressFromInfo(info *resource.Info, toExpose servicesToExpose) error {
167194
return nil
168195
}
169196

197+
func portFromServiceBackendPort(sbp *networkv1.ServiceBackendPort) *intstr.IntOrString {
198+
res := intstr.FromInt32(sbp.Number)
199+
if sbp.Number == 0 {
200+
res = intstr.FromString(sbp.Name)
201+
}
202+
return &res
203+
}
204+
170205
func parseDeployResource(podSpec *v1.PodTemplateSpec, obj metaV1.Object, resourceCtx *Resource) {
171206
resourceCtx.Resource.Name = obj.GetName()
172207
resourceCtx.Resource.Namespace = obj.GetNamespace()

‎pkg/analyzer/policies_synthesizer_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ func TestExtractConnectionsCustomWalk2(t *testing.T) {
230230
synthesizer := NewPoliciesSynthesizer(WithWalkFn(filepath.WalkDir))
231231
resources, conns, errs := synthesizer.extractConnectionsFromFolderPaths([]string{dirPath})
232232
require.Len(t, errs, 0)
233-
require.Len(t, conns, 14)
233+
require.Len(t, conns, 15)
234234
require.Len(t, resources, 14)
235235
}
236236

‎pkg/analyzer/resource_accumulator.go

Lines changed: 10 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -216,11 +216,16 @@ func (ra *resourceAccumulator) exposeServices() {
216216
if !ok {
217217
continue
218218
}
219-
if exposeExternally, ok := exposedServicesInNamespace[svc.Resource.Name]; ok {
220-
if exposeExternally {
221-
svc.Resource.ExposeExternally = true
222-
} else {
223-
svc.Resource.ExposeToCluster = true
219+
portsToExpose, ok := exposedServicesInNamespace[svc.Resource.Name]
220+
if !ok {
221+
continue
222+
}
223+
for i := range svc.Resource.Network {
224+
port := &svc.Resource.Network[i]
225+
for _, portToExpose := range portsToExpose {
226+
if port.equals(portToExpose) {
227+
port.exposeToCluster = true
228+
}
224229
}
225230
}
226231
}

‎pkg/analyzer/synth_netpols.go

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -99,9 +99,12 @@ func determineConnectivityPerDeployment(connections []*Connections) []*deploymen
9999
for _, conn := range connections {
100100
srcDeploy := findOrAddDeploymentConn(conn.Source, deploysConnectivity)
101101
dstDeploy := findOrAddDeploymentConn(conn.Target, deploysConnectivity)
102-
targetPorts := toNetpolPorts(conn.Link.Resource.Network)
102+
targetPorts := toNetpolPorts(conn.Link.Resource.Network, srcDeploy == nil && !conn.Link.Resource.ExposeExternally)
103103
if conn.Source != nil && len(conn.Source.Resource.UsedPorts) > 0 {
104-
targetPorts = toNetpolPorts(conn.Source.Resource.UsedPorts)
104+
targetPorts = toNetpolPorts(conn.Source.Resource.UsedPorts, false)
105+
}
106+
if len(targetPorts) == 0 {
107+
continue
105108
}
106109

107110
if srcDeploy != nil {
@@ -112,10 +115,10 @@ func determineConnectivityPerDeployment(connections []*Connections) []*deploymen
112115
switch {
113116
case conn.Link.Resource.ExposeExternally:
114117
dstDeploy.addIngressRule([]network.NetworkPolicyPeer{}, targetPorts) // allowing traffic from all sources
115-
case conn.Link.Resource.ExposeToCluster:
118+
case srcDeploy == nil:
116119
peer := network.NetworkPolicyPeer{NamespaceSelector: &metaV1.LabelSelector{}}
117120
dstDeploy.addIngressRule([]network.NetworkPolicyPeer{peer}, targetPorts) // allowing traffic from all cluster sources
118-
case conn.Source != nil:
121+
default:
119122
netpolPeer := getNetpolPeer(dstDeploy, srcDeploy)
120123
dstDeploy.addIngressRule([]network.NetworkPolicyPeer{netpolPeer}, targetPorts) // allow traffic only from this specific source
121124
}
@@ -161,9 +164,12 @@ func getDeployConnSelector(deployConn *deploymentConnectivity) *metaV1.LabelSele
161164
return &metaV1.LabelSelector{MatchLabels: deployConn.Resource.Resource.Labels}
162165
}
163166

164-
func toNetpolPorts(ports []SvcNetworkAttr) []network.NetworkPolicyPort {
167+
func toNetpolPorts(ports []SvcNetworkAttr, exposedOnly bool) []network.NetworkPolicyPort {
165168
netpolPorts := make([]network.NetworkPolicyPort, 0, len(ports))
166169
for _, port := range ports {
170+
if exposedOnly && !port.exposeToCluster {
171+
continue
172+
}
167173
protocol := port.Protocol
168174
if protocol == "" {
169175
protocol = core.ProtocolTCP

‎pkg/analyzer/types.go

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -49,9 +49,11 @@ func (r1 *Resource) equals(r2 *Resource) bool {
4949

5050
// SvcNetworkAttr is used to store port information
5151
type SvcNetworkAttr struct {
52-
Port int `json:"port,omitempty"`
53-
TargetPort intstr.IntOrString `json:"target_port,omitempty"`
54-
Protocol corev1.Protocol `json:"protocol,omitempty"`
52+
name string
53+
Port int `json:"port,omitempty"`
54+
TargetPort intstr.IntOrString `json:"target_port,omitempty"`
55+
Protocol corev1.Protocol `json:"protocol,omitempty"`
56+
exposeToCluster bool
5557
}
5658

5759
// Service is used to store information about a K8s Service
@@ -64,7 +66,6 @@ type Service struct {
6466
FilePath string `json:"filepath,omitempty"`
6567
Kind string `json:"kind,omitempty"`
6668
Network []SvcNetworkAttr `json:"network,omitempty"`
67-
ExposeToCluster bool `json:"-"`
6869
ExposeExternally bool `json:"-"`
6970
} `json:"resource,omitempty"`
7071
}
@@ -76,6 +77,6 @@ type Connections struct {
7677
Link *Service `json:"link"`
7778
}
7879

79-
// A map from namespaces to a map of service names in each namespaces.
80-
// For each service we also hold whether they should be exposed externally (true) or just globally inside the cluster (false)
81-
type servicesToExpose map[string]map[string]bool
80+
// A map from namespaces to a map of service names in each namespaces, which we want to expose within the cluster.
81+
// For each service we hold the ports that should be exposed
82+
type servicesToExpose map[string]map[string][]*intstr.IntOrString

‎pkg/analyzer/utils.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,7 @@ func appendAndLogNewError(errs []FileProcessingError, newErr *FileProcessingErro
2121
errs = append(errs, *newErr)
2222
return errs
2323
}
24+
25+
func appendToSliceInMap[K comparable, V any](m map[K][]V, key K, newVal V) {
26+
m[key] = append(m[key], newVal)
27+
}

‎tests/sockshop/expected_netpol_output.json

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,10 @@
2323
{
2424
"protocol": "TCP",
2525
"port": 80
26+
},
27+
{
28+
"protocol": "TCP",
29+
"port": 9090
2630
}
2731
],
2832
"from": [
@@ -34,6 +38,19 @@
3438
}
3539
}
3640
]
41+
},
42+
{
43+
"ports": [
44+
{
45+
"protocol": "TCP",
46+
"port": 9090
47+
}
48+
],
49+
"from": [
50+
{
51+
"namespaceSelector": {}
52+
}
53+
]
3754
}
3855
],
3956
"egress": [
@@ -203,6 +220,10 @@
203220
{
204221
"protocol": "TCP",
205222
"port": 80
223+
},
224+
{
225+
"protocol": "TCP",
226+
"port": 9090
206227
}
207228
],
208229
"to": [
@@ -309,6 +330,21 @@
309330
"name": "rabbitmq"
310331
}
311332
},
333+
"ingress": [
334+
{
335+
"ports": [
336+
{
337+
"protocol": "TCP",
338+
"port": "exporter"
339+
}
340+
],
341+
"from": [
342+
{
343+
"namespaceSelector": {}
344+
}
345+
]
346+
}
347+
],
312348
"policyTypes": [
313349
"Ingress",
314350
"Egress"
@@ -369,6 +405,21 @@
369405
"name": "user"
370406
}
371407
},
408+
"ingress": [
409+
{
410+
"ports": [
411+
{
412+
"protocol": "TCP",
413+
"port": 7070
414+
}
415+
],
416+
"from": [
417+
{
418+
"namespaceSelector": {}
419+
}
420+
]
421+
}
422+
],
372423
"egress": [
373424
{
374425
"ports": [

‎tests/sockshop/manifests/02-carts-svc.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,5 +13,8 @@ spec:
1313
# the port that this service should serve on
1414
- port: 80
1515
targetPort: 80
16+
- port: 9090
17+
name: exporter
18+
protocol: TCP
1619
selector:
1720
name: carts

‎tests/sockshop/manifests/26-user-svc.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,14 +5,19 @@ metadata:
55
name: user
66
annotations:
77
prometheus.io/scrape: 'true'
8+
prometheus.io/port: '7070'
89
labels:
910
name: user
1011
namespace: sock-shop
1112
spec:
1213
ports:
1314
# the port that this service should serve on
1415
- port: 80
16+
name: http
1517
targetPort: 80
18+
- port: 7070
19+
name: metrics
20+
targetPort: 7070
1621
selector:
1722
name: user
1823

0 commit comments

Comments
 (0)