[test] Stabilize flaky e2e tests by replacing hard sleeps with polling loops

e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml

@@ -73,21 +73,23 @@ spec:
         - script: 
             content: |
               set -euo pipefail
-              sleep 30
               invalid_ip=$(kubectl get configmap invalid-ip-config -o=jsonpath='{.data.InvalidIP}' -n $NAMESPACE)
               if [[ -z "$invalid_ip" ]]; then
                   echo "Error: No invalid ip found in configmap"
               fi
 
               annotation="service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4"
-              events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json)
-              message=$(echo $events | jq .items[0].message)
 
-              if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then
-                echo "Warning event found"
-              else
-                echo "Warning event not found"
-              fi
+              for i in {1..10}; do
+                events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json)
+                message=$(echo $events | jq .items[0].message)
+
+                if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then
+                  echo "Warning event found"
+                  break
+                fi
+                sleep 10
+              done
 
               service_ip=$(kubectl get svc svc-test -n $NAMESPACE -o jsonpath='{.status.loadBalancer.ingress[0].ip}')
               if [[ "$service_ip" != "" ]]; then

e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml

@@ -163,29 +163,32 @@ spec:
 
               parallel -j 2 patch_annotation ::: $reserved_ip2 "100.10.10.10"
 
-              sleep 20
+              for i in {1..10}; do
+                all_events=$(kubectl get events -n $NAMESPACE)
+                events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
+                num_events=$(echo $events | jq '.items | length')
 
-              all_events=$(kubectl get events -n $NAMESPACE)
+                #k8s scheduler will flatten the work queue of updates for an object into a single reconcile call.
+                #If k8s scheduler squashes both the patches into 1 only 1 event is generated
 
-              events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
-              num_events=$(echo $events | jq '.items | length')
-
-              #k8s scheduler will flatten the work queue of updates for an object into a single reconcile call. 
-              #If k8s scheduler squashes both the patches into 1 only 1 event is generated
+                if [[ num_events -eq 0 ]]; then
+                  sleep 10
+                  continue
+                fi
 
-              if [[ num_events -eq 0 ]]; then
-                echo "Warning event not found"
-              else
                 message=$(echo $events | jq .items[0].message)
                 if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ || "$message" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
                   echo "First warning event found"
+                  break
                 elif [[ num_events -eq 2 ]]; then
                   message2=$(echo $events | jq .items[1].message)
                   if [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ || "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
                     echo "Second warning event found"
+                    break
                   fi
                 fi
-              fi             
+                sleep 10
+              done
             check:
               ($error == null): true
               (contains($stdout, 'No reserved ip found in configmap')): false

e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml

@@ -131,17 +131,16 @@ spec:
                 echo "Unable to update annotation"
               fi
 
-              sleep 20
-
-              events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
-
-              message=$(echo $events | jq .items[0].message)
-
-              if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $unreserved_ip,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
-                echo "Warning event found"
-              else
-                echo "Warning event not found"
-              fi
+              for i in {1..10}; do
+                events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
+                message=$(echo $events | jq .items[0].message)
+
+                if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $unreserved_ip,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
+                  echo "Warning event found"
+                  break
+                fi
+                sleep 10
+              done
             check:
               ($error == null): true
               (contains($stdout, 'Warning event found')): true

e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml

@@ -159,17 +159,16 @@ spec:
                 echo "Unable to update annotation"
               fi
 
-              sleep 20
-
-              events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
-
-              message=$(echo $events | jq .items[0].message)
-
-              if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
-                echo "Warning event found"
-              else
-                echo "Warning event not found"
-              fi
+              for i in {1..10}; do
+                events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
+                message=$(echo $events | jq .items[0].message)
+
+                if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
+                  echo "Warning event found"
+                  break
+                fi
+                sleep 10
+              done
             check:
               ($error == null): true
               (contains($stdout, 'Warning event found')): true

e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml

@@ -174,15 +174,16 @@ spec:
                 echo "IPs do not match"
               fi
 
-              sleep 30
               #Run a curl command to the service ip
               URL="http://$service_ip:80/"
-              HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
-              if [ "$HTTP_RESPONSE" -eq 200 ]; then
-                 echo "Request was successful (HTTP 200)"
-              else
-                 echo "Request failed with response code: $HTTP_RESPONSE"
-              fi
+              for i in {1..10}; do
+                HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true)
+                if [ "$HTTP_RESPONSE" -eq 200 ]; then
+                   echo "Request was successful (HTTP 200)"
+                   break
+                fi
+                sleep 10
+              done
             check:
               ($error == null): true
               (contains($stdout, 'No reserved ip found in configmap')): false

e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml

@@ -2,7 +2,7 @@
 apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
-  name: c
+  name: lb-created-with-reserved-ip-multiple-change-ip
   labels:
     all:
     lke:
@@ -161,8 +161,6 @@ spec:
                 }
               }")
 
-              sleep 30
-
               patch2=$(kubectl patch service svc-test -n $NAMESPACE --patch "{
                 \"metadata\": {
                   \"annotations\": {
@@ -171,30 +169,25 @@ spec:
                 }
               }")
 
-              sleep 30
               if [[ "$patch" != "service/svc-test patched" ]]; then
                 echo "Unable to update annotation"
               fi
               if [[ "$patch2" != "service/svc-test patched" ]]; then
                 echo "Unable to update annotation"
               fi
-
-              events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
 
-              message=$(echo $events | jq .items[0].message)
-              message2=$(echo $events | jq .items[1].message)
+              for i in {1..10}; do
+                events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json)
+                message=$(echo $events | jq .items[0].message)
+                message2=$(echo $events | jq .items[1].message)
 
-              if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
-                echo "Warning event found"
-              else
-                echo "Warning event not found"
-              fi
-
-              if [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
-                echo "Warning event found"
-              else
-                echo "Warning event not found"
-              fi
+                if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]] && \
+                   [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then
+                  echo "Warning event found"
+                  break
+                fi
+                sleep 10
+              done
             check:
               ($error == null): true
               (contains($stdout, 'No reserved ip found in configmap')): false

e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml

@@ -206,15 +206,16 @@ spec:
                 echo "IPs do not match"
               fi
 
-              sleep 30
               #Run a curl command to the service ip
               URL="http://$service_ip:80/"
-              HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL")
-              if [ "$HTTP_RESPONSE" -eq 200 ]; then
-                 echo "Request was successful (HTTP 200)"
-              else
-                 echo "Request failed with response code: $HTTP_RESPONSE"
-              fi
+              for i in {1..10}; do
+                HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true)
+                if [ "$HTTP_RESPONSE" -eq 200 ]; then
+                   echo "Request was successful (HTTP 200)"
+                   break
+                fi
+                sleep 10
+              done
             check:
               ($error == null): true
               (contains($stdout, 'No reserved ip found in configmap')): false

e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml

@@ -170,7 +170,6 @@ spec:
               else
                 echo "IPs do not match"
               fi
-              sleep 20
 
             check:
               ($error == null): true

e2e/test/lb-fw-delete-acl/chainsaw-test.yaml

@@ -65,45 +65,45 @@ spec:
     - name: Delete ACL and check that firewall no longer exists
       try:
         - script:
+            timeout: 5m
             content: |
               set -euo pipefail
 
-              for i in {1..10}; do
-                  nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
-
-                  fw=$(curl -s --request GET \
-                    -H "Authorization: Bearer $LINODE_TOKEN" \
-                    -H "Content-Type: application/json" --fail-early --retry 3 \
-                    -H "accept: application/json" \
-                    "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
-
-                  fwid=$(echo $fw | jq -r '.data[].id')
-
-                  # Patch service to remove ACL annotation
-                  kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]'
-                  sleep 5
-
-                  # Check that firewall is no longer attached to nb
-                  fw=$(curl -s --request GET \
-                    -H "Authorization: Bearer $LINODE_TOKEN" \
-                    -H "Content-Type: application/json" --fail-early --retry 3 \
-                    -H "accept: application/json" \
-                    "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
-
-                  fwCount=$(echo $fw | jq -r '.data | length')
-
-                  # Check if firewall is deleted
-                  fwRespCode=$(curl -s -o /dev/null -w "%{http_code}" \
-                    --request GET \
-                    -H "Authorization: Bearer $LINODE_TOKEN" \
-                    -H "accept: application/json" \
-                    "$LINODE_URL/v4/networking/firewalls/${fwid}" || true)
-
-                  if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then
-                      echo "firewall detatched and deleted"
-                      break
-                  fi
-                  sleep 10
+              nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
+
+              fw=$(curl -s --request GET \
+                -H "Authorization: Bearer $LINODE_TOKEN" \
+                -H "Content-Type: application/json" --fail-early --retry 3 \
+                -H "accept: application/json" \
+                "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
+
+              fwid=$(echo $fw | jq -r '.data[].id')
+
+              # Patch service to remove ACL annotation
+              kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]'
+
+              for i in {1..30}; do
+                # Check that firewall is no longer attached to nb
+                fw=$(curl -s --request GET \
+                  -H "Authorization: Bearer $LINODE_TOKEN" \
+                  -H "Content-Type: application/json" --fail-early --retry 3 \
+                  -H "accept: application/json" \
+                  "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true)
+
+                fwCount=$(echo $fw | jq -r '.data | length')
+
+                # Check if firewall is deleted
+                fwRespCode=$(curl -s -o /dev/null -w "%{http_code}" \
+                  --request GET \
+                  -H "Authorization: Bearer $LINODE_TOKEN" \
+                  -H "accept: application/json" \
+                  "$LINODE_URL/v4/networking/firewalls/${fwid}" || true)
+
+                if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then
+                    echo "firewall detatched and deleted"
+                    break
+                fi
+                sleep 10
               done
             check:
                 ($error == null): true

e2e/test/lb-update-port/chainsaw-test.yaml

@@ -41,12 +41,11 @@ spec:
         - script:
             content: |
               set -euo pipefail
-              sleep 30
               IP=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].ip)
 
               podnames=()
 
-              for i in {1..10}; do
+              for i in {1..20}; do
                   if [[ ${#podnames[@]} -lt 2 ]]; then
                       output=$(curl -s $IP:80 | jq -e .podName || true)
 
@@ -86,12 +85,11 @@ spec:
             content: |
               set -euo pipefail
               #wait for changes to propagate to the LB
-              sleep 60
               IP=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].ip)
 
               podnames=()
 
-              for i in {1..20}; do
+              for i in {1..30}; do
                   if [[ ${#podnames[@]} -lt 2 ]]; then
                       output=$(curl -s $IP:8080 | jq -e .podName || true)
 

e2e/test/lb-with-http-to-https/chainsaw-test.yaml

@@ -41,10 +41,9 @@ spec:
               set -euo pipefail
               kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-port-443='{"tls-secret-name": "tls-secret-1", "protocol": "https"}'
               kubectl patch svc svc-test -n $NAMESPACE --type='json' -p='[{"op": "add", "path": "/spec/ports/-", "value": {"name": "https", "port": 443, "targetPort": 8080, "protocol": "TCP"}}]'
-              sleep 10
             check:
               ($error == null): true
-    - name: Check endpoints
+    - name: Check endpoints have addresses
       try:
       - assert:
           resource:
@@ -54,14 +53,22 @@ spec:
               name: svc-test
             (subsets[0].addresses != null): true
             (subsets[0].ports != null): true
-            (length(subsets[0].ports)): 2
       catch:
         - describe:
             apiVersion: v1
             kind: Pod
         - describe:
             apiVersion: v1
             kind: Service
+    - name: Check endpoints have two ports
+      try:
+      - assert:
+          resource:
+            apiVersion: v1
+            kind: Endpoints
+            metadata:
+              name: svc-test
+            (length(subsets[0].ports)): 2
     - name: Check that loadbalancer ip is assigned
       try:
       - assert: