feat(Controllers): Track previously matched instances for cleanup

[Baarsgaard]

Managed to get a PoC up and running.
The code still needs a fair bit of cleanup and some helper functions to simplify things a bit.

In theory, this should allow making the instanceSelector and allowCrossNamespaceImport mutable.

setup.yml

---    
apiVersion: grafana.integreatly.org/v1beta1
kind: Grafana
metadata:
  labels:
    dashboards: grafana
  name: grafana1
  namespace: default
spec:
  client:
    preferIngress: true
  config:
    security:
      admin_password: secret
      admin_user: root
  ingress:
    spec:
      ingressClassName: nginx
      rules:
      - host: grafana1.127.0.0.1.nip.io
        http:
          paths:
          - backend:
              service:
                name: grafana1-service
                port:
                  number: 3000
            path: /
            pathType: Prefix
---              
apiVersion: grafana.integreatly.org/v1beta1
kind: Grafana
metadata:
  labels:
    dashboards: grafana
  name: grafana2
  namespace: default
spec:
  client:
    preferIngress: true
  config:
    security:
      admin_password: secret
      admin_user: root
  ingress:
    spec:
      ingressClassName: nginx
      rules:
      - host: grafana2.127.0.0.1.nip.io
        http:
          paths:
          - backend:
              service:
                name: grafana2-service
                port:
                  number: 3000
            path: /
            pathType: Prefix
---
apiVersion: grafana.integreatly.org/v1beta1
kind: GrafanaContactPoint
metadata:
  name: grafanacontactpoint-sample
  namespace: default
spec:
  instanceSelector:
    matchLabels:
      dashboards: grafana
  receivers:
  - settings:
      addresses: [email protected]
    type: email
  - settings:
      addresses: [email protected]
    type: email
  resyncPeriod: 10s

make start-kind
kubectl apply --serverside -f setup.yml

# Start the operator in a separate terminal 
make run

# Wait for both Grafanas to be ready
kubectl get grafanas

# ContactPoint matched both instances 
kubectl get grafanas -o yaml | yq '.items[].status.contactPoints'
# - default/grafanacontactpoint-sample/grafanacontactpoint-sample
# - default/grafanacontactpoint-sample/grafanacontactpoint-sample

kubectl get grafanacontactpoints grafanacontactpoint-sample -o yaml | yq '.metadata.annotations'
# annotations:
#   operator.grafana.com/matched-instances: default/grafana1,default/grafana2

# Remove label from an instance
kubectl label grafana grafana2 dashboards-

# Verify it was removed from Grafana status and ContactPoint annotations
kubectl get grafanas -o yaml | yq '.items[].status.contactPoints'
# - default/grafanacontactpoint-sample/grafanacontactpoint-sample
# []
kubectl get grafanacontactpoints grafanacontactpoint-sample -o yaml | yq '.metadata.annotations'
# annotations:
#   operator.grafana.com/matched-instances: default/grafana1

# Label the instance again and see it's matched again.
kubectl label grafana grafana2 dashboards=grafana