From afe40f9c9412a5ec2d7b5a5a36710544a80ba01c Mon Sep 17 00:00:00 2001 From: Moshe Vayner Date: Fri, 17 Apr 2026 13:37:05 -0400 Subject: [PATCH 1/5] fix(test): Stabilize flaky e2e tests by replacing hard sleeps with polling loops Signed-off-by: Moshe Vayner --- .../chainsaw-test.yaml | 18 ++++---- .../chainsaw-test.yaml | 25 ++++++----- .../chainsaw-test.yaml | 21 +++++---- .../chainsaw-test.yaml | 21 +++++---- .../chainsaw-test.yaml | 15 ++++--- .../chainsaw-test.yaml | 31 ++++++------- .../chainsaw-test.yaml | 15 ++++--- .../chainsaw-test.yaml | 1 - e2e/test/lb-update-port/chainsaw-test.yaml | 6 +-- .../lb-with-http-to-https/chainsaw-test.yaml | 1 - .../lb-with-node-addition/chainsaw-test.yaml | 4 +- .../chainsaw-test.yaml | 43 +++++++------------ .../chainsaw-test.yaml | 27 ++++++------ .../chainsaw-test.yaml | 27 ++++++------ .../chainsaw-test.yaml | 15 ++++--- 15 files changed, 131 insertions(+), 139 deletions(-) diff --git a/e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml b/e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml index e7f3b68f..be9d932c 100644 --- a/e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml @@ -73,21 +73,23 @@ spec: - script: content: | set -euo pipefail - sleep 30 invalid_ip=$(kubectl get configmap invalid-ip-config -o=jsonpath='{.data.InvalidIP}' -n $NAMESPACE) if [[ -z "$invalid_ip" ]]; then echo "Error: No invalid ip found in configmap" fi annotation="service.beta.kubernetes.io/linode-loadbalancer-reserved-ipv4" - events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json) - message=$(echo $events | jq .items[0].message) - if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then - echo "Warning event found" - else - echo "Warning event not found" - fi + for i in {1..10}; do + events=$(kubectl get events -n $NAMESPACE --field-selector reason=SyncLoadBalancerFailed --sort-by='.lastTimestamp' -o json) + message=$(echo $events | jq .items[0].message) + + if [[ "$message" == *"Error syncing load balancer: failed to ensure load balancer: [400] Invalid IPv4 address"* ]]; then + echo "Warning event found" + break + fi + sleep 10 + done service_ip=$(kubectl get svc svc-test -n $NAMESPACE -o jsonpath='{.status.loadBalancer.ingress[0].ip}') if [[ "$service_ip" != "" ]]; then diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml index 05531a42..5068c3f5 100644 --- a/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml @@ -163,29 +163,32 @@ spec: parallel -j 2 patch_annotation ::: $reserved_ip2 "100.10.10.10" - sleep 20 + for i in {1..10}; do + all_events=$(kubectl get events -n $NAMESPACE) + events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) + num_events=$(echo $events | jq '.items | length') - all_events=$(kubectl get events -n $NAMESPACE) + #k8s scheduler will flatten the work queue of updates for an object into a single reconcile call. + #If k8s scheduler squashes both the patches into 1 only 1 event is generated - events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) - num_events=$(echo $events | jq '.items | length') - - #k8s scheduler will flatten the work queue of updates for an object into a single reconcile call. - #If k8s scheduler squashes both the patches into 1 only 1 event is generated + if [[ num_events -eq 0 ]]; then + sleep 10 + continue + fi - if [[ num_events -eq 0 ]]; then - echo "Warning event not found" - else message=$(echo $events | jq .items[0].message) if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ || "$message" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then echo "First warning event found" + break elif [[ num_events -eq 2 ]]; then message2=$(echo $events | jq .items[1].message) if [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ || "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then echo "Second warning event found" + break fi fi - fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'No reserved ip found in configmap')): false diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml index a6ca2b6b..063120cf 100644 --- a/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml @@ -131,17 +131,16 @@ spec: echo "Unable to update annotation" fi - sleep 20 - - events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) - - message=$(echo $events | jq .items[0].message) - - if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $unreserved_ip,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then - echo "Warning event found" - else - echo "Warning event not found" - fi + for i in {1..10}; do + events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) + message=$(echo $events | jq .items[0].message) + + if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $unreserved_ip,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then + echo "Warning event found" + break + fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'Warning event found')): true diff --git a/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml index 19a490db..3dd8680a 100644 --- a/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-change-ip/chainsaw-test.yaml @@ -159,17 +159,16 @@ spec: echo "Unable to update annotation" fi - sleep 20 - - events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) - - message=$(echo $events | jq .items[0].message) - - if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then - echo "Warning event found" - else - echo "Warning event not found" - fi + for i in {1..10}; do + events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) + message=$(echo $events | jq .items[0].message) + + if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then + echo "Warning event found" + break + fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'Warning event found')): true diff --git a/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml index 2054168f..c259e35c 100644 --- a/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml @@ -174,15 +174,16 @@ spec: echo "IPs do not match" fi - sleep 30 #Run a curl command to the service ip URL="http://$service_ip:80/" - HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL") - if [ "$HTTP_RESPONSE" -eq 200 ]; then - echo "Request was successful (HTTP 200)" - else - echo "Request failed with response code: $HTTP_RESPONSE" - fi + for i in {1..10}; do + HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL") + if [ "$HTTP_RESPONSE" -eq 200 ]; then + echo "Request was successful (HTTP 200)" + break + fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'No reserved ip found in configmap')): false diff --git a/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml index f4a8db13..aee02845 100644 --- a/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-multiple-change-ip/chainsaw-test.yaml @@ -2,7 +2,7 @@ apiVersion: chainsaw.kyverno.io/v1alpha1 kind: Test metadata: - name: c + name: lb-created-with-reserved-ip-multiple-change-ip labels: all: lke: @@ -161,8 +161,6 @@ spec: } }") - sleep 30 - patch2=$(kubectl patch service svc-test -n $NAMESPACE --patch "{ \"metadata\": { \"annotations\": { @@ -171,30 +169,25 @@ spec: } }") - sleep 30 if [[ "$patch" != "service/svc-test patched" ]]; then echo "Unable to update annotation" fi if [[ "$patch2" != "service/svc-test patched" ]]; then echo "Unable to update annotation" fi - - events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) - message=$(echo $events | jq .items[0].message) - message2=$(echo $events | jq .items[1].message) + for i in {1..10}; do + events=$(kubectl get events -n $NAMESPACE --field-selector reason=NodeBalancerIPChangeIgnored --sort-by='.lastTimestamp' -o json) + message=$(echo $events | jq .items[0].message) + message2=$(echo $events | jq .items[1].message) - if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then - echo "Warning event found" - else - echo "Warning event not found" - fi - - if [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then - echo "Warning event found" - else - echo "Warning event not found" - fi + if [[ "$message" =~ ^\"IPv4\ annotation\ changed\ to\ $reserved_ip2,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]] && \ + [[ "$message2" =~ ^\"IPv4\ annotation\ changed\ to\ 100.10.10.10,\ but\ NodeBalancer\ \([0-9]+\)\ IP\ cannot\ be\ updated\ after\ creation.\ It\ will\ remain\ $reserved_ip\"$ ]]; then + echo "Warning event found" + break + fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'No reserved ip found in configmap')): false diff --git a/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml index 91d5d646..1c6bd4cf 100644 --- a/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml @@ -206,15 +206,16 @@ spec: echo "IPs do not match" fi - sleep 30 #Run a curl command to the service ip URL="http://$service_ip:80/" - HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL") - if [ "$HTTP_RESPONSE" -eq 200 ]; then - echo "Request was successful (HTTP 200)" - else - echo "Request failed with response code: $HTTP_RESPONSE" - fi + for i in {1..10}; do + HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL") + if [ "$HTTP_RESPONSE" -eq 200 ]; then + echo "Request was successful (HTTP 200)" + break + fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'No reserved ip found in configmap')): false diff --git a/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml b/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml index ddfa3d5b..fbb94ceb 100644 --- a/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-specified-nb-id-reserved/chainsaw-test.yaml @@ -170,7 +170,6 @@ spec: else echo "IPs do not match" fi - sleep 20 check: ($error == null): true diff --git a/e2e/test/lb-update-port/chainsaw-test.yaml b/e2e/test/lb-update-port/chainsaw-test.yaml index 464a0d0b..6dfc8095 100644 --- a/e2e/test/lb-update-port/chainsaw-test.yaml +++ b/e2e/test/lb-update-port/chainsaw-test.yaml @@ -41,12 +41,11 @@ spec: - script: content: | set -euo pipefail - sleep 30 IP=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].ip) podnames=() - for i in {1..10}; do + for i in {1..20}; do if [[ ${#podnames[@]} -lt 2 ]]; then output=$(curl -s $IP:80 | jq -e .podName || true) @@ -86,12 +85,11 @@ spec: content: | set -euo pipefail #wait for changes to propagate to the LB - sleep 60 IP=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].ip) podnames=() - for i in {1..20}; do + for i in {1..30}; do if [[ ${#podnames[@]} -lt 2 ]]; then output=$(curl -s $IP:8080 | jq -e .podName || true) diff --git a/e2e/test/lb-with-http-to-https/chainsaw-test.yaml b/e2e/test/lb-with-http-to-https/chainsaw-test.yaml index 2b0d9b7a..2f5d39a5 100644 --- a/e2e/test/lb-with-http-to-https/chainsaw-test.yaml +++ b/e2e/test/lb-with-http-to-https/chainsaw-test.yaml @@ -41,7 +41,6 @@ spec: set -euo pipefail kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-port-443='{"tls-secret-name": "tls-secret-1", "protocol": "https"}' kubectl patch svc svc-test -n $NAMESPACE --type='json' -p='[{"op": "add", "path": "/spec/ports/-", "value": {"name": "https", "port": 443, "targetPort": 8080, "protocol": "TCP"}}]' - sleep 10 check: ($error == null): true - name: Check endpoints diff --git a/e2e/test/lb-with-node-addition/chainsaw-test.yaml b/e2e/test/lb-with-node-addition/chainsaw-test.yaml index e4e5fb8b..5bf602ee 100644 --- a/e2e/test/lb-with-node-addition/chainsaw-test.yaml +++ b/e2e/test/lb-with-node-addition/chainsaw-test.yaml @@ -82,12 +82,10 @@ spec: required_replicas=$((current_replicas + 1)) KUBECONFIG=$MGMT_KUBECONFIG kubectl patch machinedeployment ${CLUSTER_NAME}-md-0 --type='merge' -p "{\"spec\":{\"replicas\":$required_replicas}}" - sleep 180 - nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) echo "Nodebalancer ID: $nbid" - for i in {1..20}; do + for i in {1..30}; do response=$(curl -sf \ -H "Authorization: Bearer $LINODE_TOKEN" \ -H "Content-Type: application/json" --fail-early --retry 3 \ diff --git a/e2e/test/lb-with-proxyprotocol-default-annotation/chainsaw-test.yaml b/e2e/test/lb-with-proxyprotocol-default-annotation/chainsaw-test.yaml index ef4cdd9b..9cf3dda6 100644 --- a/e2e/test/lb-with-proxyprotocol-default-annotation/chainsaw-test.yaml +++ b/e2e/test/lb-with-proxyprotocol-default-annotation/chainsaw-test.yaml @@ -42,7 +42,6 @@ spec: content: | set -euo pipefail kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-proxy-protocol=v2 - sleep 10 check: ($error == null): true - name: Check NodeBalancerConfig for port 80 and 8080 have ProxyProtocol v2 @@ -79,7 +78,6 @@ spec: content: | set -euo pipefail kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-default-proxy-protocol=v1 - sleep 10 check: ($error == null): true - name: Check NodeBalancerConfig for port 80 and 8080 have ProxyProtocol v1 @@ -88,34 +86,25 @@ spec: content: | set -euo pipefail - re='^[0-9]+$' - - hostname=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .status.loadBalancer.ingress[0].hostname) - ip=$(echo $hostname | awk -F'.' '{gsub("-", ".", $1); print $1}') - nbid=$(curl -s \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - -H "X-Filter: {\"ipv4\": \"$ip\"}" \ - "$LINODE_URL/v4/nodebalancers" | jq .data[].id) - - if ! [[ $nbid =~ $re ]]; then - echo "Nodebalancer id [$nbid] is incorrect, doesn't meet regex requirements" - exit 1 - fi + nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) - nbconfig=$(curl -s \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - "$LINODE_URL/v4/nodebalancers/$nbid/configs") + for i in {1..10}; do + nbconfig=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + "$LINODE_URL/v4/nodebalancers/$nbid/configs") - port_80_v1=$(echo $nbconfig | jq -r '.data[] | select(.port == 80) | .proxy_protocol == "v1"') - port_8080_v1=$(echo $nbconfig | jq -r '.data[] | select(.port == 8080) | .proxy_protocol == "v1"') + port_80_v1=$(echo $nbconfig | jq -r '.data[] | select(.port == 80) | .proxy_protocol == "v1"') + port_8080_v1=$(echo $nbconfig | jq -r '.data[] | select(.port == 8080) | .proxy_protocol == "v1"') - if [[ $port_80_v1 == "true" && $port_8080_v1 == "true" ]]; then - echo "Conditions met" - else - echo "Conditions not met" - fi + if [[ $port_80_v1 == "true" && $port_8080_v1 == "true" ]]; then + echo "Conditions met" + break + else + echo "Conditions not met" + fi + sleep 10 + done check: ($error): ~ (contains($stdout, 'Conditions met')): true diff --git a/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml b/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml index 38951cd7..974710e2 100644 --- a/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml +++ b/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml @@ -42,7 +42,6 @@ spec: content: | set -euo pipefail kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-port-8080='{"proxy-protocol": "v2"}' - sleep 10 check: ($error == null): true - name: Check NodeBalancerConfig for port 80 to not have ProxyProtocol and port 8080 to have ProxyProtocol v2 @@ -53,19 +52,23 @@ spec: nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) - nbconfig=$(curl -s \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - "$LINODE_URL/v4/nodebalancers/$nbid/configs") + for i in {1..10}; do + nbconfig=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + "$LINODE_URL/v4/nodebalancers/$nbid/configs") - port_80_none=$(echo $nbconfig | jq -r '.data[] | select(.port == 80) | .proxy_protocol == "none"') - port_8080_v2=$(echo $nbconfig | jq -r '.data[] | select(.port == 8080) | .proxy_protocol == "v2"') + port_80_none=$(echo $nbconfig | jq -r '.data[] | select(.port == 80) | .proxy_protocol == "none"') + port_8080_v2=$(echo $nbconfig | jq -r '.data[] | select(.port == 8080) | .proxy_protocol == "v2"') - if [[ $port_80_none == "true" && $port_8080_v2 == "true" ]]; then - echo "Conditions met" - else - echo "Conditions not met" - fi + if [[ $port_80_none == "true" && $port_8080_v2 == "true" ]]; then + echo "Conditions met" + break + else + echo "Conditions not met" + fi + sleep 10 + done check: ($error): ~ (contains($stdout, 'Conditions met')): true diff --git a/e2e/test/lb-with-proxyprotocol-set/chainsaw-test.yaml b/e2e/test/lb-with-proxyprotocol-set/chainsaw-test.yaml index d7c7e99a..b6fc6f36 100644 --- a/e2e/test/lb-with-proxyprotocol-set/chainsaw-test.yaml +++ b/e2e/test/lb-with-proxyprotocol-set/chainsaw-test.yaml @@ -53,7 +53,6 @@ spec: set -euo pipefail kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-port-80='{"proxy-protocol": "v1"}' kubectl annotate svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-port-8080='{"proxy-protocol": "v2"}' - sleep 10 check: ($error == null): true - name: Check NodeBalancerConfig for port 80 to have ProxyProtocol v1 and port 8080 to have ProxyProtocol v2 @@ -64,19 +63,23 @@ spec: nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) - nbconfig=$(curl -s \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - "$LINODE_URL/v4/nodebalancers/$nbid/configs") + for i in {1..10}; do + nbconfig=$(curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + "$LINODE_URL/v4/nodebalancers/$nbid/configs") - port_80_v1=$(echo $nbconfig | jq -r '.data[] | select(.port == 80) | .proxy_protocol == "v1"') - port_8080_v2=$(echo $nbconfig | jq -r '.data[] | select(.port == 8080) | .proxy_protocol == "v2"') + port_80_v1=$(echo $nbconfig | jq -r '.data[] | select(.port == 80) | .proxy_protocol == "v1"') + port_8080_v2=$(echo $nbconfig | jq -r '.data[] | select(.port == 8080) | .proxy_protocol == "v2"') - if [[ $port_80_v1 == "true" && $port_8080_v2 == "true" ]]; then - echo "Conditions met" - else - echo "Conditions not met" - fi + if [[ $port_80_v1 == "true" && $port_8080_v2 == "true" ]]; then + echo "Conditions met" + break + else + echo "Conditions not met" + fi + sleep 10 + done check: ($error): ~ (contains($stdout, 'Conditions met')): true diff --git a/e2e/test/lb-with-udp-ports-algorithm/chainsaw-test.yaml b/e2e/test/lb-with-udp-ports-algorithm/chainsaw-test.yaml index b7a1f755..3fbea44c 100644 --- a/e2e/test/lb-with-udp-ports-algorithm/chainsaw-test.yaml +++ b/e2e/test/lb-with-udp-ports-algorithm/chainsaw-test.yaml @@ -53,12 +53,17 @@ spec: echo "Nodebalancer config found, updating config algorithm" kubectl annotate --overwrite svc svc-test -n $NAMESPACE service.beta.kubernetes.io/linode-loadbalancer-default-algorithm=ring_hash - sleep 5s - + echo "Verifying that algorithm is set to ring hash" - nbconfig=$(LINODE_TOKEN=$LINODE_TOKEN NBID=$nbid ../scripts/get-nb-config.sh) - algorithm=$(echo $nbconfig | jq -r '.algorithm') - echo "algorithm is $algorithm" + for i in {1..10}; do + nbconfig=$(LINODE_TOKEN=$LINODE_TOKEN NBID=$nbid ../scripts/get-nb-config.sh) + algorithm=$(echo $nbconfig | jq -r '.algorithm') + echo "algorithm is $algorithm" + if [[ "$algorithm" == "ring_hash" ]]; then + break + fi + sleep 10 + done check: ($error == null): true (contains($stdout, 'algorithm is ring_hash')): true From 26dbcddbd684ec840a5dedef166cc5b35408dd2b Mon Sep 17 00:00:00 2001 From: Moshe Vayner Date: Fri, 17 Apr 2026 14:30:49 -0400 Subject: [PATCH 2/5] [test]: Additional fixes for flaky tests Signed-off-by: Moshe Vayner --- e2e/test/lb-fw-delete-acl/chainsaw-test.yaml | 63 +++++++++---------- .../lb-with-http-to-https/chainsaw-test.yaml | 12 +++- .../chainsaw-test.yaml | 10 +++ .../chainsaw-test.yaml | 10 +++ 4 files changed, 61 insertions(+), 34 deletions(-) diff --git a/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml b/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml index cde38aa8..4a6afb4f 100644 --- a/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml +++ b/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml @@ -68,42 +68,41 @@ spec: content: | set -euo pipefail - for i in {1..10}; do - nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) + nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) - fw=$(curl -s --request GET \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - -H "accept: application/json" \ - "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true) - - fwid=$(echo $fw | jq -r '.data[].id') + fw=$(curl -s --request GET \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + -H "accept: application/json" \ + "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true) - # Patch service to remove ACL annotation - kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]' - sleep 5 + fwid=$(echo $fw | jq -r '.data[].id') - # Check that firewall is no longer attached to nb - fw=$(curl -s --request GET \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "Content-Type: application/json" --fail-early --retry 3 \ - -H "accept: application/json" \ - "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true) - - fwCount=$(echo $fw | jq -r '.data | length') + # Patch service to remove ACL annotation + kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]' - # Check if firewall is deleted - fwRespCode=$(curl -s -o /dev/null -w "%{http_code}" \ - --request GET \ - -H "Authorization: Bearer $LINODE_TOKEN" \ - -H "accept: application/json" \ - "$LINODE_URL/v4/networking/firewalls/${fwid}" || true) - - if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then - echo "firewall detatched and deleted" - break - fi - sleep 10 + for i in {1..10}; do + # Check that firewall is no longer attached to nb + fw=$(curl -s --request GET \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "Content-Type: application/json" --fail-early --retry 3 \ + -H "accept: application/json" \ + "$LINODE_URL/v4/nodebalancers/${nbid}/firewalls" || true) + + fwCount=$(echo $fw | jq -r '.data | length') + + # Check if firewall is deleted + fwRespCode=$(curl -s -o /dev/null -w "%{http_code}" \ + --request GET \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "accept: application/json" \ + "$LINODE_URL/v4/networking/firewalls/${fwid}" || true) + + if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then + echo "firewall detatched and deleted" + break + fi + sleep 10 done check: ($error == null): true diff --git a/e2e/test/lb-with-http-to-https/chainsaw-test.yaml b/e2e/test/lb-with-http-to-https/chainsaw-test.yaml index 2f5d39a5..288689f7 100644 --- a/e2e/test/lb-with-http-to-https/chainsaw-test.yaml +++ b/e2e/test/lb-with-http-to-https/chainsaw-test.yaml @@ -43,7 +43,7 @@ spec: kubectl patch svc svc-test -n $NAMESPACE --type='json' -p='[{"op": "add", "path": "/spec/ports/-", "value": {"name": "https", "port": 443, "targetPort": 8080, "protocol": "TCP"}}]' check: ($error == null): true - - name: Check endpoints + - name: Check endpoints have addresses try: - assert: resource: @@ -53,7 +53,6 @@ spec: name: svc-test (subsets[0].addresses != null): true (subsets[0].ports != null): true - (length(subsets[0].ports)): 2 catch: - describe: apiVersion: v1 @@ -61,6 +60,15 @@ spec: - describe: apiVersion: v1 kind: Service + - name: Check endpoints have two ports + try: + - assert: + resource: + apiVersion: v1 + kind: Endpoints + metadata: + name: svc-test + (length(subsets[0].ports)): 2 - name: Check that loadbalancer ip is assigned try: - assert: diff --git a/e2e/test/lb-with-proxyprotocol-override/chainsaw-test.yaml b/e2e/test/lb-with-proxyprotocol-override/chainsaw-test.yaml index 145968d1..86576075 100644 --- a/e2e/test/lb-with-proxyprotocol-override/chainsaw-test.yaml +++ b/e2e/test/lb-with-proxyprotocol-override/chainsaw-test.yaml @@ -36,6 +36,16 @@ spec: name: svc-test (subsets[0].addresses != null): true (subsets[0].ports != null): true + - name: Check that loadbalancer ip is assigned + try: + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: svc-test + status: + (loadBalancer.ingress[0].ip != null): true - name: Annotate service port 80 with v1 and 8080 with v2 try: - script: diff --git a/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml b/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml index 974710e2..b1ee9e8a 100644 --- a/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml +++ b/e2e/test/lb-with-proxyprotocol-port-specific/chainsaw-test.yaml @@ -36,6 +36,16 @@ spec: name: svc-test (subsets[0].addresses != null): true (subsets[0].ports != null): true + - name: Check that loadbalancer ip is assigned + try: + - assert: + resource: + apiVersion: v1 + kind: Service + metadata: + name: svc-test + status: + (loadBalancer.ingress[0].ip != null): true - name: Annotate service port 80 with v1 and 8080 with v2 try: - script: From ca81688c749f46f52589a96772cafb011db46244 Mon Sep 17 00:00:00 2001 From: Moshe Vayner Date: Fri, 17 Apr 2026 15:23:06 -0400 Subject: [PATCH 3/5] [test] Increase timeout and retry attempts for firewall deletion in chainsaw test Signed-off-by: Moshe Vayner --- e2e/test/lb-fw-delete-acl/chainsaw-test.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml b/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml index 4a6afb4f..ba6cdead 100644 --- a/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml +++ b/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml @@ -65,6 +65,7 @@ spec: - name: Delete ACL and check that firewall no longer exists try: - script: + timeout: 5m content: | set -euo pipefail @@ -81,7 +82,7 @@ spec: # Patch service to remove ACL annotation kubectl patch service svc-test -n $NAMESPACE --type=json -p='[{"op": "remove", "path": "/metadata/annotations/service.beta.kubernetes.io~1linode-loadbalancer-firewall-acl"}]' - for i in {1..10}; do + for i in {1..30}; do # Check that firewall is no longer attached to nb fw=$(curl -s --request GET \ -H "Authorization: Bearer $LINODE_TOKEN" \ @@ -98,6 +99,8 @@ spec: -H "accept: application/json" \ "$LINODE_URL/v4/networking/firewalls/${fwid}" || true) + echo "attempt $i: fwCount=$fwCount fwRespCode=$fwRespCode" + if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then echo "firewall detatched and deleted" break From 77ddc9e80daa7ae2394d41fd81590b1c0ece8c70 Mon Sep 17 00:00:00 2001 From: Moshe Vayner Date: Fri, 17 Apr 2026 18:16:54 -0400 Subject: [PATCH 4/5] [test] Re-add the 180 seconds sleep in `lb-with-node-addition` We need this to allow the new node to come up Signed-off-by: Moshe Vayner --- e2e/test/lb-with-node-addition/chainsaw-test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/e2e/test/lb-with-node-addition/chainsaw-test.yaml b/e2e/test/lb-with-node-addition/chainsaw-test.yaml index 5bf602ee..11c16b8c 100644 --- a/e2e/test/lb-with-node-addition/chainsaw-test.yaml +++ b/e2e/test/lb-with-node-addition/chainsaw-test.yaml @@ -82,6 +82,8 @@ spec: required_replicas=$((current_replicas + 1)) KUBECONFIG=$MGMT_KUBECONFIG kubectl patch machinedeployment ${CLUSTER_NAME}-md-0 --type='merge' -p "{\"spec\":{\"replicas\":$required_replicas}}" + sleep 180 + nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh) echo "Nodebalancer ID: $nbid" From e991f37509a8d781e8f80b719d1743e491da857e Mon Sep 17 00:00:00 2001 From: Moshe Vayner Date: Fri, 17 Apr 2026 19:21:07 -0400 Subject: [PATCH 5/5] [test]: Fix curl error handling in reserved IP HTTP response checks curl -w "%{http_code}" already outputs "000" when a connection fails, then exits non-zero. Without error suppression, `set -euo pipefail` exits the script immediately. Add `|| true` to suppress the exit code while preserving curl's own `"000"` output, allowing the retry loop to continue. Signed-off-by: Moshe Vayner --- .../lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml | 2 +- .../lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml | 2 +- e2e/test/lb-fw-delete-acl/chainsaw-test.yaml | 2 -- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml index c259e35c..d57b0ea7 100644 --- a/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-linode-range/chainsaw-test.yaml @@ -177,7 +177,7 @@ spec: #Run a curl command to the service ip URL="http://$service_ip:80/" for i in {1..10}; do - HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL") + HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true) if [ "$HTTP_RESPONSE" -eq 200 ]; then echo "Request was successful (HTTP 200)" break diff --git a/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml b/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml index 1c6bd4cf..b5d18237 100644 --- a/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml +++ b/e2e/test/lb-created-with-reserved-ip-nb-range/chainsaw-test.yaml @@ -209,7 +209,7 @@ spec: #Run a curl command to the service ip URL="http://$service_ip:80/" for i in {1..10}; do - HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL") + HTTP_RESPONSE=$(curl -s -o /dev/null -w "%{http_code}" "$URL" || true) if [ "$HTTP_RESPONSE" -eq 200 ]; then echo "Request was successful (HTTP 200)" break diff --git a/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml b/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml index ba6cdead..538336cb 100644 --- a/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml +++ b/e2e/test/lb-fw-delete-acl/chainsaw-test.yaml @@ -99,8 +99,6 @@ spec: -H "accept: application/json" \ "$LINODE_URL/v4/networking/firewalls/${fwid}" || true) - echo "attempt $i: fwCount=$fwCount fwRespCode=$fwRespCode" - if [[ $fwCount -eq 0 && $fwRespCode -eq "404" ]]; then echo "firewall detatched and deleted" break