fix(helm chart): networkPolicy supports distinct namespaces for prometheus pod & servicemonitor (#1086)

* fix(helm chart): networkPolicy supports distinct namespaces for prometheus pod & servicemonitor

Network policy did not allow ingress from prometheus if it was deployed in a different namespace
from the servicemonitor. This PR allows the ingress to be configured to match the operator's
requirements.

addresses https://github.com/specklesystems/gitOps/issues/68

* provides additional validation and error output when getting secrets
* Fix for kubernetes network policies using s3 details from ConfigMap
* Remove blocking of 10.0.0.0/8 range as this also prevents access to cloud provider private IPs
* Update values.schema.json

utils/helm/speckle-server/templates/_helpers.tpl

@@ -166,7 +166,8 @@ Creates a Kubernetes network policy egress definition for connecting to S3 compa
   {{- if .Values.s3.networkPolicy.inCluster.enabled -}}
 {{ include "speckle.networkpolicy.egress.internal" (dict "podSelector" .Values.s3.networkPolicy.inCluster.kubernetes.podSelector "namespaceSelector" .Values.s3.networkPolicy.inCluster.kubernetes.namespaceSelector "port" $port) }}
   {{- else if .Values.s3.networkPolicy.externalToCluster.enabled -}}
-    {{- $ip := ( include "speckle.networkPolicy.domainFromUrl" .Values.s3.endpoint ) -}}
+    {{- $s3Values := ( include "server.s3Values" . | fromJson ) -}}
+    {{- $ip := ( include "speckle.networkPolicy.domainFromUrl" $s3Values.endpoint ) -}}
 {{ include "speckle.networkpolicy.egress.external" (dict "ip" $ip "port" $port) }}
   {{- end -}}
 {{- end }}
@@ -302,8 +303,10 @@ Limitations:
         # Kubernetes network policy does not support fqdn, so we have to allow egress anywhere
         cidr: 0.0.0.0/0
         # except to kubernetes pods or services
-        except:
-          - 10.0.0.0/8
+        except: []
+          # unfortunately cannot limit to typical kubernetes pod CIDR,
+          # as some cloud vendor private IPs (e.g. for hosted databases) are also in this range
+          # - 10.0.0.0/8
     {{- end }}
   ports:
     - port: {{ printf "%s" .port }}
@@ -336,9 +339,10 @@ Limitations:
 - toCIDRSet:
       # Kubernetes network policy does not support fqdn, so we have to allow egress anywhere
     - cidr: 0.0.0.0/0
-      # except to kubernetes pods or services
-      except:
-        - 10.0.0.0/8
+      # ideally would like to prevent access to kubernetes pods or services
+      # but some cloud provider private IPs (e.g. for hosted services) are in this range
+      except: []
+        # - 10.0.0.0/8
 {{- end }}
   toPorts:
     - ports:
@@ -480,21 +484,6 @@ Usage:
     {{- end }}
 {{- end -}}
 
-{{/*
-Selector labels for Prometheus
-*/}}
-{{- define "speckle.prometheus.selectorLabels" -}}
-{{ include "speckle.prometheus.selectorLabels.release" . }}
-io.kubernetes.pod.namespace: {{ default .Values.namespace .Values.prometheusMonitoring.namespace }}
-{{- end }}
-
-{{/*
-Selector labels for Prometheus release
-*/}}
-{{- define "speckle.prometheus.selectorLabels.release" -}}
-prometheus: {{ default "kube-prometheus-stack" .Values.prometheusMonitoring.release }}-prometheus
-{{- end }}
-
 {{/*
 Ingress pod selector
 */}}
@@ -509,12 +498,19 @@ Usage:
 {{ include "speckle.getSecret" (dict  "secret_name" "server-vars" "secret_key" "postgres_url" "context" $ )}}
 
 Params:
+  - secret_name - Required, the name of the secret.
   - secret_key - Required, the key within the secret.
   - context - Required, must be global context.  Values of global context must include 'namespace' and 'secretName' keys.
 */}}
 {{- define "speckle.getSecret" -}}
 {{- $secretResource := (lookup "v1" "Secret" .context.Values.namespace .secret_name ) -}}
+{{- if not $secretResource -}}
+    {{- printf "\nERROR: Could not discover a secret \"%s\" in namespace \"%s\".\n       Try `kubectl get secret --namespace %s` to view available secrets." .secret_name .context.Values.namespace .context.Values.namespace | fail -}}
+{{- end -}}
 {{- $secret := ( index $secretResource.data .secret_key ) -}}
+{{- if not $secret -}}
+    {{- printf "\nERROR: Could not find a secret key \"%s\" of secret \"%s\" in namespace \"%s\".\n       Try `kubectl describe secret --namespace %s %s` to view available keys in the secret." .secret_key .secret_name .context.Values.namespace .context.Values.namespace .secret_name | fail -}}
+{{- end -}}
 {{- $secretDecoded := (b64dec $secret) -}}
 {{- printf "%s" $secretDecoded }}
 {{- end }}

utils/helm/speckle-server/templates/fileimport_service/networkpolicy.cilium.yml

@@ -13,8 +13,7 @@ spec:
 {{- if .Values.enable_prometheus_monitoring }}
   ingress:
     - fromEndpoints:
-        - matchLabels:
-{{ include "speckle.prometheus.selectorLabels" $ | indent 12 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.cilium.fromEndpoints | toYaml | indent 10 }}
       toPorts:
         - ports:
             - port: "metrics"

utils/helm/speckle-server/templates/fileimport_service/networkpolicy.kubernetes.yml

@@ -17,11 +17,9 @@ spec:
   ingress:
     - from:
         - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: {{ default .Values.namespace .Values.prometheusMonitoring.namespace }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.namespaceSelector | toYaml | indent 12 }}
           podSelector:
-            matchLabels:
-              {{ include  "speckle.prometheus.selectorLabels.release" $ | indent 14 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.podSelector | toYaml | indent 12 }}
       ports:
         - port: metrics
 {{- else }}

utils/helm/speckle-server/templates/frontend/networkpolicy.kubernetes.yml

@@ -19,9 +19,9 @@ spec:
         - namespaceSelector:
             matchLabels:
               kubernetes.io/metadata.name: {{ .Values.ingress.namespace }}
-        - podSelector:
+          podSelector:
             matchLabels:
-{{ include  "frontend.ingress.selector.pod" $ | indent 14 }}
+{{ include  "speckle.ingress.selector.pod" $ | indent 14 }}
       ports:
         - port: www
   egress: [] # block all egress

utils/helm/speckle-server/templates/monitoring/networkpolicy.cilium.yml

@@ -13,8 +13,7 @@ spec:
 {{- if .Values.enable_prometheus_monitoring }}
   ingress:
     - fromEndpoints:
-        - matchLabels:
-{{ include "speckle.prometheus.selectorLabels" $ | indent 12 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.cilium.fromEndpoints | toYaml | indent 10 }}
       toPorts:
         - ports:
             - port: "metrics"

utils/helm/speckle-server/templates/monitoring/networkpolicy.kubernetes.yml

@@ -17,11 +17,9 @@ spec:
   ingress:
     - from:
         - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: {{ default .Values.namespace .Values.prometheusMonitoring.namespace }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.namespaceSelector | toYaml | indent 12 }}
           podSelector:
-            matchLabels:
-              {{ include  "speckle.prometheus.selectorLabels.release" $ | indent 14 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.podSelector | toYaml | indent 12 }}
       ports:
         - port: metrics
 {{- else }}

utils/helm/speckle-server/templates/preview_service/networkpolicy.cilium.yml

@@ -13,8 +13,7 @@ spec:
 {{- if .Values.enable_prometheus_monitoring }}
   ingress:
     - fromEndpoints:
-        - matchLabels:
-{{ include "speckle.prometheus.selectorLabels" $ | indent 12 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.cilium.fromEndpoints | toYaml | indent 10 }}
       toPorts:
         - ports:
             - port: "metrics"

utils/helm/speckle-server/templates/preview_service/networkpolicy.kubernetes.yml

@@ -17,11 +17,9 @@ spec:
   ingress:
     - from:
         - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: {{ default .Values.namespace .Values.prometheusMonitoring.namespace }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.namespaceSelector | toYaml | indent 12 }}
           podSelector:
-            matchLabels:
-              {{ include  "speckle.prometheus.selectorLabels.release" $ | indent 14 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.podSelector | toYaml | indent 12 }}
       ports:
         - port: metrics
 {{- else }}

utils/helm/speckle-server/templates/server/networkpolicy.cilium.yml

@@ -21,8 +21,7 @@ spec:
               protocol: TCP
 {{- if .Values.enable_prometheus_monitoring }}
     - fromEndpoints:
-        - matchLabels:
-{{ include "speckle.prometheus.selectorLabels" $ | indent 12 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.cilium.fromEndpoints | toYaml | indent 10 }}
       toPorts:
         - ports:
             - port: http

utils/helm/speckle-server/templates/server/networkpolicy.kubernetes.yml

@@ -34,9 +34,6 @@ spec:
     - to:
       - ipBlock:
           cidr: 0.0.0.0/0
-          # except to kubernetes pods or services
-          except:
-            - 10.0.0.0/8
       ports:
         - port: 443
 {{- end }}
@@ -70,14 +67,12 @@ spec:
       ports:
         - port: http
 {{- if .Values.enable_prometheus_monitoring }}
-    # allow ingress from servicemonitor/prometheus
+    # allow ingress from pod prometheus
     - from:
         - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: {{ default .Values.namespace .Values.prometheusMonitoring.namespace }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.namespaceSelector | toYaml | indent 12 }}
           podSelector:
-            matchLabels:
-              {{ include  "speckle.prometheus.selectorLabels.release" $ | indent 14 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.podSelector | toYaml | indent 12 }}
       ports:
         - port: http
 {{- end }}

utils/helm/speckle-server/templates/webhook_service/networkpolicy.cilium.yml

@@ -13,8 +13,7 @@ spec:
 {{- if .Values.enable_prometheus_monitoring }}
   ingress:
     - fromEndpoints:
-        - matchLabels:
-{{ include "speckle.prometheus.selectorLabels" $ | indent 12 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.cilium.fromEndpoints | toYaml | indent 10 }}
       toPorts:
         - ports:
             - port: "metrics"

utils/helm/speckle-server/templates/webhook_service/networkpolicy.kubernetes.yml

@@ -17,11 +17,9 @@ spec:
   ingress:
     - from:
         - namespaceSelector:
-            matchLabels:
-              kubernetes.io/metadata.name: {{ default .Values.namespace .Values.prometheusMonitoring.namespace }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.namespaceSelector | toYaml | indent 12 }}
           podSelector:
-            matchLabels:
-              {{ include  "speckle.prometheus.selectorLabels.release" $ | indent 14 }}
+{{ .Values.prometheusMonitoring.networkPolicy.inCluster.kubernetes.podSelector | toYaml | indent 12 }}
       ports:
         - port: metrics
 {{- else }}

utils/helm/speckle-server/values.schema.json

@@ -82,13 +82,51 @@
       "properties": {
         "namespace": {
           "type": "string",
-          "description": "If provided, deploys Speckle's Prometheus resources in the given namespace",
+          "description": "If provided, deploys Speckle's Prometheus resources (e.g. ServiceMonitor) in the given namespace",
           "default": ""
         },
         "release": {
           "type": "string",
           "description": "If provided, adds the value to a `release` label on all the Prometheus resources deployed by Speckle",
           "default": ""
+        },
+        "networkPolicy": {
+          "type": "object",
+          "properties": {
+            "inCluster": {
+              "type": "object",
+              "properties": {
+                "kubernetes": {
+                  "type": "object",
+                  "properties": {
+                    "podSelector": {
+                      "type": "object",
+                      "description": "(Kubernetes Network Policy only) The pod Selector yaml object used to uniquely select the prometheus pods within the cluster and given namespace",
+                      "default": {}
+                    },
+                    "namespaceSelector": {
+                      "type": "object",
+                      "description": "(Kubernetes Network Policy only) The namespace selector yaml object used to uniquely select the namespace in which the prometheus pods are deployed",
+                      "default": {}
+                    }
+                  }
+                },
+                "cilium": {
+                  "type": "object",
+                  "properties": {
+                    "fromEndpoints": {
+                      "type": "array",
+                      "description": "(Cilium Network Policy only) The endpoint selector yaml object used to uniquely select the in-cluster endpoint in which the prometheus pods are deployed",
+                      "default": [],
+                      "items": {
+                        "type": "object"
+                      }
+                    }
+                  }
+                }
+              }
+            }
+          }
         }
       }
     },

utils/helm/speckle-server/values.yaml

@@ -86,7 +86,7 @@ file_size_limit_mb: 100
 enable_prometheus_monitoring: false
 
 prometheusMonitoring:
-  ## @param prometheusMonitoring.namespace If provided, deploys Speckle's Prometheus resources in the given namespace
+  ## @param prometheusMonitoring.namespace If provided, deploys Speckle's Prometheus resources (e.g. ServiceMonitor) in the given namespace
   ## Prometheus prior to v0.19.0, or any version when deployed with default parameters, expects ServiceMonitors to be deployed within the same namespace.
   ## This parameter allows the Prometheus resources provided by Speckle to be deployed in another namespace.
   ## This allows Prometheus (< v0.19.0 or any version with default configuration) to be deployed in a separate namespace from Speckle.
@@ -100,6 +100,32 @@ prometheusMonitoring:
   ## ref: https://github.com/prometheus-operator/prometheus-operator/blob/main/Documentation/user-guides/getting-started.md#related-resources
   ##
   release: ''
+  networkPolicy:
+    ## @extra prometheusMonitoring.networkPolicy.inCluster Parameters for allowing ingress from the Prometheus pod that will scrape this Speckle release. It is assumed that Prometheus is deployed within the Kubernetes cluster.
+    ##
+    inCluster:
+      kubernetes:
+        ## @param prometheusMonitoring.networkPolicy.inCluster.kubernetes.podSelector (Kubernetes Network Policy only) The pod Selector yaml object used to uniquely select the prometheus pods within the cluster and given namespace
+        ## For Kubernetes Network Policies this is a podSelector object.
+        ## For Cilium Network Policies this is ignored.
+        ## ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/#behavior-of-to-and-from-selectors
+        ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+        ##
+        podSelector: {}
+        ## @param prometheusMonitoring.networkPolicy.inCluster.kubernetes.namespaceSelector  (Kubernetes Network Policy only) The namespace selector yaml object used to uniquely select the namespace in which the prometheus pods are deployed
+        ## This is a Kubernetes namespaceSelector object.
+        ## For Cilium Network Policies this is ignored
+        ## ref: https://kubernetes.io/docs/concepts/services-networking/network-policies/#behavior-of-to-and-from-selectors
+        ## ref: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/
+        ##
+        namespaceSelector: {}
+      cilium:
+        ## @param prometheusMonitoring.networkPolicy.inCluster.cilium.fromEndpoints (Cilium Network Policy only) The endpoint selector yaml object used to uniquely select the in-cluster endpoint in which the prometheus pods are deployed
+        ## For Kubernetes Network Policies this is ignored.
+        ## ref: https://docs.cilium.io/en/v1.9/policy/language/#ingress
+        ## ref: https://github.com/cilium/cilium/blob/master/pkg/policy/api/selector.go
+        ##
+        fromEndpoints: []
 
 ## @section Postgres Database
 ## @descriptionStart