[test][improvement] Standardize bash error handling and add CCM log collection for e2e tests

e2e/bgp-test/lb-cilium-bgp/chainsaw-test.yaml

@@ -7,6 +7,12 @@ metadata:
     all:
 spec:
   namespace: "cilium-bgp-test"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "cilium-bgp-test" | tail -100
   steps:
     - name: Check if CCM is deployed
       try:
@@ -61,7 +67,7 @@ spec:
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               delete_nanode() {
                 local NANODE_RESPONSE=$(curl -s -H "Authorization: Bearer $LINODE_TOKEN" \

e2e/subnet-test/chainsaw-test.yaml

@@ -10,12 +10,18 @@ spec:
     - name: fwname
       value: (join('-', ['ccm-fwtest', env('CLUSTER_NAME')]))
   namespace: "subnet-filtering-test"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "subnet-filtering-test" | tail -100
   steps:
     - name: Check if the CCM for each cluster focus on their individual subnets
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               if [ -z "$FIRST_CONFIG" ] || [ -z "$SECOND_CONFIG" ] || [ -z "$LINODE_TOKEN" ]; then
                   echo "Error: FIRST_CONFIG, SECOND_CONFIG, and LINODE_TOKEN environment variables must be set"

e2e/test/fw-use-specified-nb/chainsaw-test.yaml

@@ -11,6 +11,12 @@ spec:
     - name: fwname
       value: (join('-', ['ccm-fwtest', env('CLUSTER_NAME')]))
   namespace: "fw-use-specified-nb"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "fw-use-specified-nb" | tail -100
   steps:
     - name: Create firewall, Create pods and services
       try:
@@ -19,7 +25,7 @@ spec:
               - name: FWLABEL
                 value: ($fwname)
             content: |
-              set -e
+              set -euo pipefail
 
               create_fw=$(curl -s --write-out "%{http_code}\n" --output /dev/null --request POST \
                 -H "Authorization: Bearer $LINODE_TOKEN" \
@@ -79,7 +85,7 @@ spec:
               - name: FWLABEL
                 value: ($fwname)
             content: |
-              set -e
+              set -euo pipefail
               re='^[0-9]+$'
 
               fwid=$(curl -s \

e2e/test/lb-created-with-invalid-ip/chainsaw-test.yaml

@@ -8,12 +8,18 @@ metadata:
     lke:
 spec:
   namespace: "lb-created-with-invalid-ip"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-invalid-ip" | tail -100
   steps:
     - name: create reserved ip and nodebalancer resources
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               invalid_ip="100.1000.1000.1000"
 
@@ -41,7 +47,7 @@ spec:
       cleanup:
         - script:
             content: |
-              set -e 
+              set -euo pipefail
 
               delete_cm=$(kubectl delete configmap invalid-ip-config -n $NAMESPACE)
               if [[ "$delete_cm" == "configmap \"invalid-ip-config\" deleted" ]]; then
@@ -66,7 +72,7 @@ spec:
       try:
         - script: 
             content: |
-              set -e
+              set -euo pipefail
               sleep 30
               invalid_ip=$(kubectl get configmap invalid-ip-config -o=jsonpath='{.data.InvalidIP}' -n $NAMESPACE)
               if [[ -z "$invalid_ip" ]]; then

e2e/test/lb-created-with-new-nb-id/chainsaw-test.yaml

@@ -8,6 +8,12 @@ metadata:
     lke:
 spec:
   namespace: "lb-created-with-new-nb-id"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-new-nb-id" | tail -100
   steps:
     - name: Create dummy service
       try:
@@ -34,7 +40,7 @@ spec:
               - name: nbconf
                 value: (json_parse($stdout))
             content: |
-              set -e
+              set -euo pipefail
 
               re='^[0-9]+$'
               LABEL="ccm-$(head /dev/urandom | tr -dc 'a-z0-9' | head -c 5)"
@@ -80,7 +86,7 @@ spec:
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               expectedId=$(kubectl get svc svc-test -n $NAMESPACE -o json | jq -r .metadata.annotations[])
               nbid=$(KUBECONFIG=$KUBECONFIG NAMESPACE=$NAMESPACE LINODE_TOKEN=$LINODE_TOKEN ../scripts/get-nb-id.sh)
@@ -95,7 +101,7 @@ spec:
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               re='^[0-9]+$'
 

e2e/test/lb-created-with-reserved-ip-and-nb-id-annotations/chainsaw-test.yaml

@@ -8,6 +8,12 @@ metadata:
     lke:
 spec:
   namespace: "lb-created-with-reserved-and-nb-id-annotations"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-reserved-and-nb-id-annotations" | tail -100
   steps:
     - name: Create dummy service
       try:
@@ -31,7 +37,7 @@ spec:
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               re='^((25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})\.){3}(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})$'
 
@@ -88,7 +94,7 @@ spec:
       cleanup:
         - script:
             content: |
-              set -e 
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -132,7 +138,7 @@ spec:
       try:
         - script: 
             content: |
-              set -e
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -166,4 +172,4 @@ spec:
               (contains($stdout, 'No service ip found for service svc-test')): false
               (contains($stdout, 'IPs do not match')): false
               (contains($stdout, 'NB IDs do not match')): false
-
+

e2e/test/lb-created-with-reserved-ip-attached-to-nb/chainsaw-test.yaml

@@ -8,6 +8,12 @@ metadata:
     lke:
 spec:
   namespace: "lb-created-with-reserved-ip-attached-to-nb"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-reserved-ip-attached-to-nb" | tail -100
   steps:
     - name: Create dummy service
       try:
@@ -32,7 +38,7 @@ spec:
       try:
         - script:
             content: |
-              set -e
+              set -euo pipefail
 
               re='^((25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})\.){3}(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})$'
 
@@ -89,7 +95,7 @@ spec:
       cleanup:
         - script:
             content: |
-              set -e 
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -150,7 +156,7 @@ spec:
       try:
         - script: 
             content: |
-              set -e
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap nb-config -o=jsonpath='{.data.Reserved_IP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -179,4 +185,4 @@ spec:
               (contains($stdout, 'No reserved ip found in configmap')): false
               (contains($stdout, 'Service ip found for service svc-test. Expected to be empty')): false
               (contains($stdout, 'IPs do not match')): true
-
+

e2e/test/lb-created-with-reserved-ip-change-ip-concurrently/chainsaw-test.yaml

@@ -8,13 +8,19 @@ metadata:
     lke:
 spec:
   namespace: "lb-created-with-reserved-ip-change-ip-concurrently"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-reserved-ip-change-ip-concurrently" | tail -100
   steps:
     - name: create reserved ip and nodebalancer resources
       try:
         - script:
             content: |
+              set -euo pipefail
               #!/bin/bash
-              set -e
 
               re='^((25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})\.){3}(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})$'
 
@@ -72,7 +78,7 @@ spec:
       cleanup:
         - script:
             content: |
-              set -e 
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -133,8 +139,8 @@ spec:
       try:
         - script:
             content: |
+              set -euo pipefail
               #!/bin/bash
-              set -e
 
               patch_annotation() {
                 VALUE=$1
@@ -188,7 +194,7 @@ spec:
       try:
         - script: 
             content: |
-              set -e
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then

e2e/test/lb-created-with-reserved-ip-change-ip-unreserved/chainsaw-test.yaml

@@ -8,13 +8,19 @@ metadata:
     lke:
 spec:
   namespace: "lb-created-with-reserved-ip-change-ip-unreserved"
+  catch:
+    - script:
+        content: |
+          set -euo pipefail
+          echo "Test failed. Fetching CCM logs..."
+          kubectl logs -n kube-system daemonsets/ccm-linode | grep "lb-created-with-reserved-ip-change-ip-unreserved" | tail -100
   steps:
     - name: create reserved ip and nodebalancer resources
       try:
         - script:
             content: |
+              set -euo pipefail
               #!/bin/bash
-              set -e
 
               re='^((25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})\.){3}(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})$'
 
@@ -57,7 +63,7 @@ spec:
       cleanup:
         - script: 
             content: |
-              set -e 
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -100,8 +106,8 @@ spec:
       try:
         - script:
             content: |
+              set -euo pipefail
               #!/bin/bash
-              set -e
 
               reserved_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -145,7 +151,7 @@ spec:
       try:
         - script: 
             content: |
-              set -e
+              set -euo pipefail
 
               reserved_ip=$(kubectl get configmap reserved-ip-config -o=jsonpath='{.data.ReservedIP}' -n $NAMESPACE)
               if [ -z "$reserved_ip" ]; then
@@ -170,4 +176,4 @@ spec:
               (contains($stdout, 'No reserved ip found in configmap')): false
               (contains($stdout, 'No service ip found for service svc-test')): false
               (contains($stdout, 'IPs do not match')): false
-
+