diff --git a/controllers/argocd_metrics_controller.go b/controllers/argocd_metrics_controller.go index 10a8c45c125..a47615a80bb 100644 --- a/controllers/argocd_metrics_controller.go +++ b/controllers/argocd_metrics_controller.go @@ -44,9 +44,11 @@ import ( ) const ( - readRoleNameFormat = "%s-read" - readRoleBindingNameFormat = "%s-prometheus-k8s-read-binding" - alertRuleName = "gitops-operator-argocd-alerts" + readRoleNameFormat = "%s-read" + readRoleBindingNameFormat = "%s-prometheus-k8s-read-binding" + alertRuleName = "gitops-operator-argocd-alerts" + // Use a separate rule so upgrades install the sync-loop alerts. + syncLoopAlertRuleName = "gitops-operator-argocd-sync-loop-alerts" dashboardNamespace = "openshift-config-managed" dashboardFolder = "dashboards" operatorMetricsServiceName = "openshift-gitops-operator-metrics-service" @@ -173,7 +175,7 @@ func (r *ArgoCDMetricsReconciler) Reconcile(ctx context.Context, request reconci } // Create alert rule - err = r.createPrometheusRuleIfAbsent(request.Namespace, argocd, reqLogger) + err = r.createPrometheusRulesIfAbsent(request.Namespace, argocd, reqLogger) if err != nil { return reconcile.Result{}, err } @@ -238,13 +240,15 @@ func (r *ArgoCDMetricsReconciler) Reconcile(ctx context.Context, request reconci return reconcile.Result{}, err } - // Delete alert rule - err = r.Client.Delete(context.TODO(), &monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Namespace: request.Namespace, Name: alertRuleName}}) - if err != nil { - if !errors.IsNotFound(err) { - reqLogger.Error(err, "Error deleting prometheus in ", - "Namespace", request.Namespace) - return reconcile.Result{}, err + // Delete alert rules + for _, name := range []string{alertRuleName, syncLoopAlertRuleName} { + err = r.Client.Delete(context.TODO(), &monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Namespace: request.Namespace, Name: name}}) + if err != nil { + if !errors.IsNotFound(err) { + reqLogger.Error(err, "Error deleting prometheus rule", + "Namespace", request.Namespace, "Name", name) + return reconcile.Result{}, err + } } } @@ -406,8 +410,19 @@ func (r *ArgoCDMetricsReconciler) reconcileOperatorMetricsServiceMonitor(reqLogg return nil } -func (r *ArgoCDMetricsReconciler) createPrometheusRuleIfAbsent(namespace string, argocd *argoapp.ArgoCD, reqLogger logr.Logger) error { - alertRule := newPrometheusRule(namespace) +func (r *ArgoCDMetricsReconciler) createPrometheusRulesIfAbsent(namespace string, argocd *argoapp.ArgoCD, reqLogger logr.Logger) error { + for _, alertRule := range []*monitoringv1.PrometheusRule{ + newPrometheusRule(namespace), + newSyncLoopPrometheusRule(namespace), + } { + if err := r.createPrometheusRuleIfAbsent(alertRule, argocd, reqLogger); err != nil { + return err + } + } + return nil +} + +func (r *ArgoCDMetricsReconciler) createPrometheusRuleIfAbsent(alertRule *monitoringv1.PrometheusRule, argocd *argoapp.ArgoCD, reqLogger logr.Logger) error { existingAlertRule := &monitoringv1.PrometheusRule{} err := r.Client.Get(context.TODO(), types.NamespacedName{Name: alertRule.Name, Namespace: alertRule.Namespace}, existingAlertRule) if err == nil { @@ -415,29 +430,28 @@ func (r *ArgoCDMetricsReconciler) createPrometheusRuleIfAbsent(namespace string, "Namespace", existingAlertRule.Namespace, "Name", existingAlertRule.Name) return nil } - if errors.IsNotFound(err) { - reqLogger.Info("Creating new alert rule", + if !errors.IsNotFound(err) { + reqLogger.Error(err, "Error querying for existing alert rule", "Namespace", alertRule.Namespace, "Name", alertRule.Name) + return err + } - // Set the ArgoCD instance as the owner and controller - if err := controllerutil.SetControllerReference(argocd, alertRule, r.Scheme); err != nil { - reqLogger.Error(err, "Error setting read role owner ref", - "Namespace", alertRule.Namespace, "Name", alertRule.Name, "ArgoCD Name", argocd.Name) - return err - } + reqLogger.Info("Creating new alert rule", + "Namespace", alertRule.Namespace, "Name", alertRule.Name) - err := r.Client.Create(context.TODO(), alertRule) - if err != nil { - reqLogger.Error(err, "Error creating a new alert rule", - "Namespace", alertRule.Namespace, "Name", alertRule.Name) - return err - } + if err := controllerutil.SetControllerReference(argocd, alertRule, r.Scheme); err != nil { + reqLogger.Error(err, "Error setting alert rule owner ref", + "Namespace", alertRule.Namespace, "Name", alertRule.Name, "ArgoCD Name", argocd.Name) + return err + } - return nil + if err := r.Client.Create(context.TODO(), alertRule); err != nil { + reqLogger.Error(err, "Error creating a new alert rule", + "Namespace", alertRule.Namespace, "Name", alertRule.Name) + return err } - reqLogger.Error(err, "Error querying for existing alert rule", - "Namespace", namespace, "Name", alertRuleName) - return err + + return nil } func (r *ArgoCDMetricsReconciler) reconcileDashboards(reqLogger logr.Logger) error { @@ -635,3 +649,56 @@ func newPrometheusRule(namespace string) *monitoringv1.PrometheusRule { Spec: spec, } } + +// newSyncLoopPrometheusRule defines alerts for sustained application sync rates. +func newSyncLoopPrometheusRule(namespace string) *monitoringv1.PrometheusRule { + return &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: syncLoopAlertRuleName, + Namespace: namespace, + }, + Spec: monitoringv1.PrometheusRuleSpec{ + Groups: []monitoringv1.RuleGroup{ + { + Name: "GitOpsOperatorArgoCDSyncLoop", + Rules: []monitoringv1.Rule{ + newRecordingRule("gitops:argocd_app_sync:rate10m", + fmt.Sprintf(`sum by (name, namespace) (rate(argocd_app_sync_total{namespace="%s"}[10m]))`, namespace)), + newRecordingRule("gitops:argocd_app_sync_failed:rate10m", + fmt.Sprintf(`sum by (name, namespace) (rate(argocd_app_sync_total{namespace="%s",phase=~"Error|Failed"}[10m]))`, namespace)), + newAlertRule("ArgoCDAppSyncLoop", "warning", "20m", + fmt.Sprintf(`gitops:argocd_app_sync:rate10m{namespace="%s"} > 0.01`, namespace), + "Argo CD application is syncing continuously", + "Argo CD application {{ $labels.name }} in namespace {{ $labels.namespace }} has a sustained sync rate above 0.01/s (about one sync every ~100s) for 20m. This often indicates a selfHeal conflict (for example HPA fighting declared replicas). Check application sync history, diff, and conflicting controllers."), + newAlertRule("ArgoCDAppSyncFailureLoop", "warning", "15m", + fmt.Sprintf(`gitops:argocd_app_sync_failed:rate10m{namespace="%s"} > 0.005`, namespace), + "Argo CD application syncs are failing repeatedly", + "Argo CD application {{ $labels.name }} in namespace {{ $labels.namespace }} has a sustained failed sync rate above 0.005/s for 15m. Investigate the application operation status and sync errors."), + }, + }, + }, + }, + } +} + +func newRecordingRule(name, expr string) monitoringv1.Rule { + return monitoringv1.Rule{ + Record: name, + Expr: intstr.FromString(expr), + } +} + +func newAlertRule(name, severity, duration, expr, summary, description string) monitoringv1.Rule { + return monitoringv1.Rule{ + Alert: name, + Annotations: map[string]string{ + "summary": summary, + "description": description, + }, + Expr: intstr.FromString(expr), + For: ptr.To(monitoringv1.Duration(duration)), + Labels: map[string]string{ + "severity": severity, + }, + } +} diff --git a/controllers/argocd_metrics_controller_test.go b/controllers/argocd_metrics_controller_test.go index 6670db10f5f..26bf3577e62 100644 --- a/controllers/argocd_metrics_controller_test.go +++ b/controllers/argocd_metrics_controller_test.go @@ -29,6 +29,7 @@ import ( is "gotest.tools/assert/cmp" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -340,6 +341,107 @@ func TestReconciler_add_prometheus_rule(t *testing.T) { } } +func TestReconciler_add_sync_loop_prometheus_rule(t *testing.T) { + testCases := []struct { + instanceName string + namespace string + }{ + { + instanceName: argoCDInstanceName, + namespace: "openshift-gitops", + }, + { + instanceName: "instance-two", + namespace: "namespace-two", + }, + } + flagPtr := false + for _, tc := range testCases { + r := newMetricsReconciler(t, tc.namespace, tc.instanceName, &flagPtr) + _, err := r.Reconcile(context.TODO(), newRequest(tc.namespace, tc.instanceName)) + assert.NilError(t, err) + + rule := monitoringv1.PrometheusRule{} + err = r.Client.Get(context.TODO(), types.NamespacedName{Name: syncLoopAlertRuleName, Namespace: tc.namespace}, &rule) + assert.NilError(t, err) + + assert.Assert(t, is.Len(rule.OwnerReferences, 1)) + assert.Equal(t, rule.OwnerReferences[0].Kind, argocdKind) + assert.Equal(t, rule.OwnerReferences[0].Name, tc.instanceName) + + assert.Equal(t, rule.Spec.Groups[0].Name, "GitOpsOperatorArgoCDSyncLoop") + assert.Assert(t, is.Len(rule.Spec.Groups[0].Rules, 4)) + + recordRule := rule.Spec.Groups[0].Rules[0] + assert.Equal(t, recordRule.Record, "gitops:argocd_app_sync:rate10m") + assert.Equal(t, recordRule.Expr.StrVal, + fmt.Sprintf(`sum by (name, namespace) (rate(argocd_app_sync_total{namespace="%s"}[10m]))`, tc.namespace)) + + failureRecordRule := rule.Spec.Groups[0].Rules[1] + assert.Equal(t, failureRecordRule.Record, "gitops:argocd_app_sync_failed:rate10m") + assert.Equal(t, failureRecordRule.Expr.StrVal, + fmt.Sprintf(`sum by (name, namespace) (rate(argocd_app_sync_total{namespace="%s",phase=~"Error|Failed"}[10m]))`, tc.namespace)) + + loop := rule.Spec.Groups[0].Rules[2] + assert.Equal(t, loop.Alert, "ArgoCDAppSyncLoop") + assert.Equal(t, string(*loop.For), "20m") + assert.Equal(t, loop.Labels["severity"], "warning") + assert.Equal(t, loop.Expr.StrVal, + fmt.Sprintf(`gitops:argocd_app_sync:rate10m{namespace="%s"} > 0.01`, tc.namespace)) + assert.Assert(t, loop.Annotations["summary"] != "") + assert.Assert(t, loop.Annotations["description"] != "") + + failureLoop := rule.Spec.Groups[0].Rules[3] + assert.Equal(t, failureLoop.Alert, "ArgoCDAppSyncFailureLoop") + assert.Equal(t, string(*failureLoop.For), "15m") + assert.Equal(t, failureLoop.Labels["severity"], "warning") + assert.Equal(t, failureLoop.Expr.StrVal, + fmt.Sprintf(`gitops:argocd_app_sync_failed:rate10m{namespace="%s"} > 0.005`, tc.namespace)) + assert.Assert(t, failureLoop.Annotations["summary"] != "") + assert.Assert(t, failureLoop.Annotations["description"] != "") + } +} + +func TestReconcile_remove_prometheus_rules(t *testing.T) { + testCases := []struct { + instanceName string + namespace string + }{ + { + instanceName: argoCDInstanceName, + namespace: "openshift-gitops", + }, + { + instanceName: "instance-two", + namespace: "namespace-two", + }, + } + + for _, tc := range testCases { + r := newMetricsReconciler(t, tc.namespace, tc.instanceName, new(false)) + request := newRequest(tc.namespace, tc.instanceName) + + _, err := r.Reconcile(context.TODO(), request) + assert.NilError(t, err) + + argocd := &argoapp.ArgoCD{} + err = r.Client.Get(context.TODO(), request.NamespacedName, argocd) + assert.NilError(t, err) + argocd.Spec.Monitoring.DisableMetrics = new(true) + err = r.Client.Update(context.TODO(), argocd) + assert.NilError(t, err) + + _, err = r.Reconcile(context.TODO(), request) + assert.NilError(t, err) + + for _, name := range []string{alertRuleName, syncLoopAlertRuleName} { + rule := &monitoringv1.PrometheusRule{} + err = r.Client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: tc.namespace}, rule) + assert.Assert(t, apierrors.IsNotFound(err)) + } + } +} + func TestReconciler_add_dashboard(t *testing.T) { // Need to create openshift-config-managed namespace for dashboards diff --git a/docs/OpenShift GitOps Usage Guide.md b/docs/OpenShift GitOps Usage Guide.md index 0d8876a9922..a95c3810aba 100644 --- a/docs/OpenShift GitOps Usage Guide.md +++ b/docs/OpenShift GitOps Usage Guide.md @@ -10,6 +10,9 @@ 6. [Running default Gitops workloads on Infrastructure Nodes](#running-default-gitops-workloads-on-infrastructure-nodes) 7. [Using NodeSelector and Tolerations in Default Instance of Openshift GitOps](#using-nodeselector-and-tolerations-in-default-instance-of-openshift-gitops) 8. [Monitoring](#monitoring) + a. [Application alerts](#application-alerts) + b. [Sync-loop alerts](#sync-loop-alerts) + c. [Querying Argo CD metrics](#querying-argo-cd-metrics) 9. [Logging](#logging) 10. [Prevent auto-reboot during Argo CD sync with machine configs](#prevent-auto-reboot-during-argo-cd-sync-with-machine-configs) 11. [Machine configs and Argo CD: Performance challenges](#machine-configs-and-argo-cd-performance-challenges) @@ -810,7 +813,121 @@ On following these steps, users should be able to manage their machinesets using ## Monitoring -OpenShift GitOps automatically detects Argo CD instances on the cluster and wires them up with the cluster monitoring stack with one alert installed out-of-the-box for reporting out-of-sync apps. No additional configuration is required. +OpenShift GitOps automatically detects Argo CD instances on the cluster and wires them up with the cluster monitoring stack. Instances in an `openshift-*` namespace need no extra configuration. Instances in other namespaces need user-workload monitoring: set `enableUserWorkload: true` in the `cluster-monitoring-config` ConfigMap in `openshift-monitoring`. Alerts appear under **Observe → Alerting**. + +### Application alerts + +By default, the Operator creates two `PrometheusRule` objects in the Argo CD instance namespace: + +- `gitops-operator-argocd-alerts` — `ArgoCDSyncAlert` (warning): Application is OutOfSync for 5m (Git ≠ cluster). +- `gitops-operator-argocd-sync-loop-alerts` — `ArgoCDAppSyncLoop` and `ArgoCDAppSyncFailureLoop` (warning). + +`ArgoCDSyncAlert` reports **drift**. Sync-loop alerts report **thrashing**: the Application keeps syncing and does not settle. An app can be OutOfSync for hours waiting on a human, or OutOfSync because it is syncing every minute forever. Only the second case trips the sync-loop alerts. + +`spec.monitoring.disableMetrics: true` on the Argo CD CR removes the rules **and** stops all Argo CD metrics collection (ServiceMonitors, the Prometheus read role, and the namespace monitoring label), not just the alerts. To change or silence an individual alert, edit the `PrometheusRule` in place (see [Scope and tuning](#scope-and-tuning)). + +### Sync-loop alerts + +A separate `PrometheusRule` is used so upgrades install these alerts without changing the existing OutOfSync rule. + +| Alert | Condition | `for` | What it means | +|-------|-----------|-------|----------------| +| `ArgoCDAppSyncLoop` | `gitops:argocd_app_sync:rate10m > 0.01` (~1 sync every ~100s) | 20m | Sustained high sync rate (any outcome) | +| `ArgoCDAppSyncFailureLoop` | `gitops:argocd_app_sync_failed:rate10m > 0.005` | 15m | Sustained Error/Failed sync rate | + +Recording rules (same object): + +```promql +gitops:argocd_app_sync:rate10m + = sum by (name, namespace) (rate(argocd_app_sync_total{namespace=""}[10m])) + +gitops:argocd_app_sync_failed:rate10m + = sum by (name, namespace) (rate(argocd_app_sync_total{namespace="",phase=~"Error|Failed"}[10m])) +``` + +`` is the Argo CD instance namespace (for example `openshift-gitops`). + +#### Interpreting the signal + +| Firing | Typical pattern | Likely cause | +|--------|-----------------|--------------| +| `ArgoCDAppSyncLoop` only | Syncs **succeed**, app stays OutOfSync / Progressing | Controller conflict or admission defaulting (HPA vs Git `replicas`, mutating webhook, LimitRange, another operator) | +| `ArgoCDAppSyncFailureLoop` (with or without `ArgoCDAppSyncLoop`) | Syncs **Error/Failed**, auto-sync retries | Bad manifest, webhook reject, immutable field, missing permissions | +| `ArgoCDSyncAlert` only | Drift, little or no sync rate | Waiting on a manual sync / human action — not a loop | + +Prometheus can tell failing loops from succeeding loops via the `phase` label. It cannot tell “HPA conflict” from “mutating webhook”; those share the same metric signature. + +Impact: extra load on the application-controller, repo-server, and Kubernetes API; the app may never settle; other apps on the same instance can starve under heavy load. + +#### Recommended operator actions + +1. Identify the Application from the alert labels (`name` is the Application; `namespace` is the Argo CD instance namespace). +2. Inspect sync / operation status: + + ```console + $ oc get application -n -o yaml + $ oc get application -n -o jsonpath='{.status.operationState}{"\n"}' + ``` + +3. Check Diff in the Argo CD UI or CLI for fields that keep changing. +4. Look for conflicting controllers on the same resources (HPA, other operators, mutating webhooks, LimitRanges): + + ```console + $ oc get hpa -A + $ oc get mutatingwebhookconfiguration + ``` + +5. Confirm the metric rate in Observe → Metrics: + + ```promql + gitops:argocd_app_sync:rate10m{name="",namespace=""} + gitops:argocd_app_sync_failed:rate10m{name="",namespace=""} + ``` + + Application `status.history` length is a poor loop detector: re-applying the same revision often does not grow history. Use `argocd_app_sync_total`. + +Then mitigate: + +- Temporarily disable `selfHeal` / automated sync while you fix the conflict. +- Fix failing manifests, RBAC, or webhook policies if `ArgoCDAppSyncFailureLoop` is firing. + +Known anti-pattern: Git wants `replicas: 1`, HPA sets `minReplicas: 3`, Application has `automated.selfHeal: true`. Syncs succeed, the app stays OutOfSync, and `ArgoCDAppSyncLoop` fires. Remove `replicas` from Git, or ignore that JSON pointer on the Application CR: + +```yaml +# Argo CD Application CR +spec: + ignoreDifferences: + - group: apps + kind: Deployment + jsonPointers: + - /spec/replicas +``` + +#### False positives + +Short bursts should not fire. The long `for` windows are the filter: + +| Scenario | Why it looks busy | Why it should not fire | +|----------|-------------------|------------------------| +| Rapid legitimate commits | Many syncs in a few minutes | Burst ends inside 20m | +| ApplicationSet creates many apps | Spike in sync activity | Rate is **per Application**; one initial sync is tiny | +| Cluster upgrade / node churn | Temporary re-sync | Usually resolves inside `for` | +| One `kubectl edit` + selfHeal | Single corrective sync | Does not sustain | + +#### Scope and tuning + +- Rules are scoped with `namespace=""`, the same pattern as `ArgoCDSyncAlert`. Application CRs that live **only** in other namespaces (apps-in-any-namespace) are not included. +- The Operator creates the `PrometheusRule` if it is absent and does not overwrite user edits. Delete the rule to recreate Operator defaults: + + ```console + $ oc delete prometheusrule gitops-operator-argocd-sync-loop-alerts -n + ``` + +- To change thresholds, edit `gitops-operator-argocd-sync-loop-alerts` in place; the Operator will not revert your changes. Adding a separate `PrometheusRule` with the same alert name does not override the shipped one; both will fire. The Operator does not expose tunables on the Argo CD CR. + +A failing loop with a high enough total rate can fire both `ArgoCDAppSyncFailureLoop` and `ArgoCDAppSyncLoop`. The Operator does not manage Alertmanager. If you want a single notification, inhibit the general loop warning when the failure alert is already firing for the same Application (`name` and `namespace`). + +### Querying Argo CD metrics Note that the metrics provided are for the Argo CD instance itself, and don’t include metrics provided by the applications. diff --git a/test/openshift/e2e/ginkgo/parallel/1-133_validate_argocd_sync_loop_alert_test.go b/test/openshift/e2e/ginkgo/parallel/1-133_validate_argocd_sync_loop_alert_test.go new file mode 100644 index 00000000000..58e13973dff --- /dev/null +++ b/test/openshift/e2e/ginkgo/parallel/1-133_validate_argocd_sync_loop_alert_test.go @@ -0,0 +1,92 @@ +package parallel + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" + "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture" + argocdFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/argocd" + k8sFixture "github.com/redhat-developer/gitops-operator/test/openshift/e2e/ginkgo/fixture/k8s" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" +) + +var _ = Describe("GitOps Operator Parallel E2E Tests", func() { + + Context("1-133_validate_argocd_sync_loop_alert", func() { + + BeforeEach(func() { + fixture.EnsureParallelCleanSlate() + }) + + It("verifying PrometheusRule gitops-operator-argocd-sync-loop-alerts exists and has expected values", Label("openshift"), func() { + + By("checking OpenShift GitOps ArgoCD instance is available") + + argocd, err := argocdFixture.GetOpenShiftGitOpsNSArgoCD() + Expect(err).ToNot(HaveOccurred()) + Eventually(argocd, "5m", "5s").Should(argocdFixture.BeAvailable()) + + By("verifying PrometheusRule gitops-operator-argocd-sync-loop-alerts exists and has expected values") + alertRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gitops-operator-argocd-sync-loop-alerts", + Namespace: "openshift-gitops", + }, + } + Eventually(alertRule).Should(k8sFixture.ExistByName()) + + Expect(alertRule.Spec.Groups).To(Equal([]monitoringv1.RuleGroup{{ + Name: "GitOpsOperatorArgoCDSyncLoop", + Rules: []monitoringv1.Rule{ + { + Record: "gitops:argocd_app_sync:rate10m", + Expr: intstr.FromString(`sum by (name, namespace) (rate(argocd_app_sync_total{namespace="openshift-gitops"}[10m]))`), + }, + { + Record: "gitops:argocd_app_sync_failed:rate10m", + Expr: intstr.FromString(`sum by (name, namespace) (rate(argocd_app_sync_total{namespace="openshift-gitops",phase=~"Error|Failed"}[10m]))`), + }, + { + Alert: "ArgoCDAppSyncLoop", + Annotations: map[string]string{ + "summary": "Argo CD application is syncing continuously", + "description": "Argo CD application {{ $labels.name }} in namespace {{ $labels.namespace }} has a sustained sync rate above 0.01/s " + + "(about one sync every ~100s) for 20m. This often indicates a selfHeal conflict " + + "(for example HPA fighting declared replicas). Check application sync history, diff, and conflicting controllers.", + }, + Expr: intstr.FromString(`gitops:argocd_app_sync:rate10m{namespace="openshift-gitops"} > 0.01`), + For: ptr.To(monitoringv1.Duration("20m")), + Labels: map[string]string{ + "severity": "warning", + }, + }, + { + Alert: "ArgoCDAppSyncFailureLoop", + Annotations: map[string]string{ + "summary": "Argo CD application syncs are failing repeatedly", + "description": "Argo CD application {{ $labels.name }} in namespace {{ $labels.namespace }} has a sustained failed sync rate above 0.005/s for 15m. Investigate the application operation status and sync errors.", + }, + Expr: intstr.FromString(`gitops:argocd_app_sync_failed:rate10m{namespace="openshift-gitops"} > 0.005`), + For: ptr.To(monitoringv1.Duration("15m")), + Labels: map[string]string{ + "severity": "warning", + }, + }, + }, + }})) + + By("verifying existing OutOfSync alert rule is unchanged") + outOfSyncRule := &monitoringv1.PrometheusRule{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gitops-operator-argocd-alerts", + Namespace: "openshift-gitops", + }, + } + Eventually(outOfSyncRule).Should(k8sFixture.ExistByName()) + Expect(outOfSyncRule.Spec.Groups).ToNot(BeEmpty()) + Expect(outOfSyncRule.Spec.Groups[0].Rules[0].Alert).To(Equal("ArgoCDSyncAlert")) + }) + }) +}) diff --git a/test/openshift/e2e/ginkgo/sequential/1-106_validate_argocd_metrics_controller_test.go b/test/openshift/e2e/ginkgo/sequential/1-106_validate_argocd_metrics_controller_test.go index 347aa3c0933..67339590c0b 100644 --- a/test/openshift/e2e/ginkgo/sequential/1-106_validate_argocd_metrics_controller_test.go +++ b/test/openshift/e2e/ginkgo/sequential/1-106_validate_argocd_metrics_controller_test.go @@ -41,8 +41,9 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() { Eventually(&rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "openshift-gitops-prometheus-k8s-read-binding", Namespace: "openshift-gitops"}}).Should(k8sFixture.ExistByName()) - // PrometheusRule + // PrometheusRules Eventually(&monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Name: "gitops-operator-argocd-alerts", Namespace: "openshift-gitops"}}).Should(k8sFixture.ExistByName()) + Eventually(&monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Name: "gitops-operator-argocd-sync-loop-alerts", Namespace: "openshift-gitops"}}).Should(k8sFixture.ExistByName()) } @@ -79,6 +80,7 @@ var _ = Describe("GitOps Operator Sequential E2E Tests", func() { Eventually(&rbacv1.RoleBinding{ObjectMeta: metav1.ObjectMeta{Name: "openshift-gitops-prometheus-k8s-read-binding", Namespace: "openshift-gitops"}}).Should(k8sFixture.NotExistByName()) Eventually(&monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Name: "gitops-operator-argocd-alerts", Namespace: "openshift-gitops"}}).Should(k8sFixture.NotExistByName()) + Eventually(&monitoringv1.PrometheusRule{ObjectMeta: metav1.ObjectMeta{Name: "gitops-operator-argocd-sync-loop-alerts", Namespace: "openshift-gitops"}}).Should(k8sFixture.NotExistByName()) By("re-enabling metrics") argocdFixture.Update(defaultArgoCD, func(ac *argov1beta1api.ArgoCD) {