Handle Ordinis outage in TGU service
Classify Ordinis classifier transport failures and empty responses as unavailable, keep stale platform cache when possible, and expose Prometheus metrics/alerting for the outage signal.
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
{{- if .Values.prometheusRule.enabled }}
|
||||
apiVersion: monitoring.coreos.com/v1
|
||||
kind: PrometheusRule
|
||||
metadata:
|
||||
name: {{ include "pcp-tgu-service.fullname" . }}-alerts
|
||||
labels:
|
||||
{{- include "pcp-tgu-service.labels" . | nindent 4 }}
|
||||
spec:
|
||||
groups:
|
||||
- name: {{ include "pcp-tgu-service.name" . }}.ordinis
|
||||
rules:
|
||||
- alert: PcpTguOrdinisUnavailable
|
||||
expr: pcp_tgu_ordinis_unavailable == 1
|
||||
for: {{ index .Values.prometheusRule.ordinisUnavailable "for" | quote }}
|
||||
labels:
|
||||
severity: {{ .Values.prometheusRule.ordinisUnavailable.severity | quote }}
|
||||
service: {{ include "pcp-tgu-service.name" . | quote }}
|
||||
annotations:
|
||||
summary: "Ordinis classifier is unavailable for pcp-tgu-service"
|
||||
description: "The last pcp-tgu-service attempt to fetch platforms from Ordinis failed. Existing stale platform cache is kept when available."
|
||||
{{- end }}
|
||||
@@ -35,5 +35,11 @@ resources: {}
|
||||
podAnnotations: {}
|
||||
podLabels: {}
|
||||
|
||||
prometheusRule:
|
||||
enabled: false
|
||||
ordinisUnavailable:
|
||||
for: 5m
|
||||
severity: warning
|
||||
|
||||
nameOverride: ""
|
||||
fullnameOverride: ""
|
||||
|
||||
Reference in New Issue
Block a user