From a9912ade8b2bd7c7a08d7d3039f586defcff2312 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Wed, 28 Jan 2026 15:18:09 +0100 Subject: [PATCH 01/10] feat: naive performance test and handling results storing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Stores performance tests as artifacts and compares the results with released version performance tests Signed-off-by: Attila Mészáros --- .github/workflows/integration-tests.yml | 7 + .github/workflows/pr.yml | 48 +++++ .../performance/PerformanceTestResult.java | 31 +++ .../performance/PerformanceTestSummary.java | 58 ++++++ .../performance/SimplePerformanceTestIT.java | 197 ++++++++++++++++++ .../SimplePerformanceTestReconciler.java | 85 ++++++++ .../SimplePerformanceTestResource.java | 29 +++ .../SimplePerformanceTestSpec.java | 29 +++ .../SimplePerformanceTestStatus.java | 34 +++ .../finalizer/SSAFinalizerIssueStatus.java | 2 +- 10 files changed, 519 insertions(+), 1 deletion(-) create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java create mode 100644 operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 77c268d6bc..e795350c6c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -55,3 +55,10 @@ jobs: echo "Using profile: ${it_profile}" ./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml + + - name: Upload performance test results + uses: actions/upload-artifact@v4 + with: + name: performance-results-java${{ inputs.java-version }}-k8s${{ inputs.kube-version }}-${{ inputs.http-client }} + path: operator-framework/target/performance_test_result.json + if-no-files-found: ignore diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 7a5964ba35..57e655ca4e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -32,3 +32,51 @@ jobs: build: uses: ./.github/workflows/build.yml + + performance_report: + name: Post Performance Results + runs-on: ubuntu-latest + needs: build + if: always() + permissions: + pull-requests: write + steps: + - uses: actions/checkout@v6 + + - name: Download all performance artifacts + uses: actions/download-artifact@v4 + with: + pattern: performance-results-* + path: performance-results + merge-multiple: true + + - name: Check for performance results + id: check_results + run: | + if [ -d "performance-results" ] && [ "$(ls -A performance-results/*.json 2>/dev/null)" ]; then + echo "has_results=true" >> $GITHUB_OUTPUT + else + echo "has_results=false" >> $GITHUB_OUTPUT + fi + + - name: Convert performance results to markdown + if: steps.check_results.outputs.has_results == 'true' + id: convert + run: | + echo "# Performance Test Results" > comment.md + echo "" >> comment.md + for file in performance-results/*.json; do + if [ -f "$file" ]; then + echo "Processing $file" + python3 .github/scripts/performance-to-markdown.py "$file" >> comment.md + echo "" >> comment.md + fi + done + + - name: Post PR comment + if: steps.check_results.outputs.has_results == 'true' && github.event_name == 'pull_request' + uses: marocchino/sticky-pull-request-comment@v2 + with: + path: comment.md + recreate: true + diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java new file mode 100644 index 0000000000..61b0feacf2 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java @@ -0,0 +1,31 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import java.util.List; + +public class PerformanceTestResult { + + private List summaries; + + public List getSummaries() { + return summaries; + } + + public void setSummaries(List summaries) { + this.summaries = summaries; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java new file mode 100644 index 0000000000..a8137dbad8 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestSummary.java @@ -0,0 +1,58 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +public class PerformanceTestSummary { + + private String name; + + // data about the machine + private int numberOfProcessors; + private long maxMemory; + private long duration; + + public String getName() { + return name; + } + + public void setName(String name) { + this.name = name; + } + + public int getNumberOfProcessors() { + return numberOfProcessors; + } + + public void setNumberOfProcessors(int numberOfProcessors) { + this.numberOfProcessors = numberOfProcessors; + } + + public long getMaxMemory() { + return maxMemory; + } + + public void setMaxMemory(long maxMemory) { + this.maxMemory = maxMemory; + } + + public long getDuration() { + return duration; + } + + public void setDuration(long duration) { + this.duration = duration; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java new file mode 100644 index 0000000000..bc4eab008a --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java @@ -0,0 +1,197 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.fabric8.kubernetes.client.informers.ResourceEventHandler; +import io.fabric8.kubernetes.client.informers.SharedIndexInformer; +import io.javaoperatorsdk.operator.junit.LocallyRunOperatorExtension; +import io.vertx.core.impl.ConcurrentHashSet; + +import com.fasterxml.jackson.databind.ObjectMapper; + +public class SimplePerformanceTestIT { + + private static final Logger log = LoggerFactory.getLogger(SimplePerformanceTestIT.class); + public static final String INITIAL_VALUE = "initialValue"; + public static final String RESOURCE_NAME_PREFIX = "resource"; + public static final String INDEX = "index"; + + @RegisterExtension + LocallyRunOperatorExtension extension = + LocallyRunOperatorExtension.builder() + .withReconciler(new SimplePerformanceTestReconciler()) + .build(); + + final int WARM_UP_RESOURCE_NUMBER = 10; + final int TEST_RESOURCE_NUMBER = 150; + + ExecutorService executor = Executors.newFixedThreadPool(TEST_RESOURCE_NUMBER); + + @Test + void simpleNaivePerformanceTest() { + var processors = Runtime.getRuntime().availableProcessors(); + long maxMemory = Runtime.getRuntime().maxMemory(); + log.info("Running performance test with memory: {} and processors: {}", maxMemory, processors); + + var primaryInformer = + extension + .getKubernetesClient() + .resources(SimplePerformanceTestResource.class) + .inNamespace(extension.getNamespace()) + .inform(); + + var statusChecker = + new StatusChecker(INITIAL_VALUE, 0, WARM_UP_RESOURCE_NUMBER, primaryInformer); + createResources(0, WARM_UP_RESOURCE_NUMBER, INITIAL_VALUE); + statusChecker.waitUntilAllInStatus(); + + long startTime = System.currentTimeMillis(); + statusChecker = + new StatusChecker( + INITIAL_VALUE, WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, primaryInformer); + createResources(WARM_UP_RESOURCE_NUMBER, TEST_RESOURCE_NUMBER, INITIAL_VALUE); + statusChecker.waitUntilAllInStatus(); + var duration = System.currentTimeMillis() - startTime; + + log.info("Create duration: {}", duration); + saveResults(duration); + } + + private void saveResults(long duration) { + try { + var result = new PerformanceTestResult(); + var summary = new PerformanceTestSummary(); + result.setSummaries(List.of(summary)); + summary.setName("Naive performance test"); + summary.setDuration(duration); + summary.setNumberOfProcessors(Runtime.getRuntime().availableProcessors()); + summary.setMaxMemory(Runtime.getRuntime().maxMemory()); + var objectMapper = new ObjectMapper(); + objectMapper.writeValue(new File("target/performance_test_result.json"), result); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + private void createResources(int startIndex, int number, String value) { + try { + List> callables = new ArrayList<>(number); + + for (int i = startIndex; i < startIndex + number; i++) { + var res = new SimplePerformanceTestResource(); + res.setMetadata( + new ObjectMetaBuilder() + .withAnnotations(Map.of(INDEX, "" + i)) + .withName(RESOURCE_NAME_PREFIX + i) + .build()); + res.setSpec(new SimplePerformanceTestSpec()); + res.getSpec().setValue(value); + callables.add( + () -> { + extension.create(res); + return null; + }); + } + var futures = executor.invokeAll(callables); + for (var future : futures) { + future.get(); + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + static class StatusChecker { + private final String expectedStatus; // null indicates deleted + private final Set remaining = new ConcurrentHashSet<>(); + + StatusChecker( + String expectedStatus, + int startIndex, + int number, + SharedIndexInformer primaryInformer) { + this.expectedStatus = expectedStatus; + for (int i = startIndex; i < startIndex + number; i++) { + remaining.add(i); + } + primaryInformer.addEventHandler( + new ResourceEventHandler<>() { + @Override + public void onAdd(SimplePerformanceTestResource obj) { + checkOnStatus(obj); + } + + @Override + public void onUpdate( + SimplePerformanceTestResource oldObj, SimplePerformanceTestResource newObj) { + checkOnStatus(newObj); + } + + @Override + public void onDelete( + SimplePerformanceTestResource obj, boolean deletedFinalStateUnknown) { + if (expectedStatus == null) { + synchronized (remaining) { + remaining.remove(Integer.parseInt(obj.getMetadata().getAnnotations().get(INDEX))); + remaining.notifyAll(); + } + } + } + }); + primaryInformer.getStore().list().forEach(this::checkOnStatus); + } + + private void checkOnStatus(SimplePerformanceTestResource res) { + if (expectedStatus != null + && res.getStatus() != null + && res.getStatus().getValue().equals(expectedStatus)) { + synchronized (remaining) { + remaining.remove(Integer.parseInt(res.getMetadata().getAnnotations().get(INDEX))); + remaining.notifyAll(); + } + } + } + + public void waitUntilAllInStatus() { + synchronized (remaining) { + while (!remaining.isEmpty()) { + try { + remaining.wait(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + } + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java new file mode 100644 index 0000000000..eff83d1514 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestReconciler.java @@ -0,0 +1,85 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import java.util.List; +import java.util.Map; + +import io.fabric8.kubernetes.api.model.ConfigMap; +import io.fabric8.kubernetes.api.model.ConfigMapBuilder; +import io.fabric8.kubernetes.api.model.ObjectMetaBuilder; +import io.javaoperatorsdk.operator.api.config.informer.InformerEventSourceConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.Cleaner; +import io.javaoperatorsdk.operator.api.reconciler.Context; +import io.javaoperatorsdk.operator.api.reconciler.ControllerConfiguration; +import io.javaoperatorsdk.operator.api.reconciler.DeleteControl; +import io.javaoperatorsdk.operator.api.reconciler.EventSourceContext; +import io.javaoperatorsdk.operator.api.reconciler.Reconciler; +import io.javaoperatorsdk.operator.api.reconciler.UpdateControl; +import io.javaoperatorsdk.operator.processing.event.source.EventSource; +import io.javaoperatorsdk.operator.processing.event.source.informer.InformerEventSource; + +@ControllerConfiguration +public class SimplePerformanceTestReconciler + implements Reconciler, Cleaner { + + public static final String KEY = "key"; + + @Override + public UpdateControl reconcile( + SimplePerformanceTestResource resource, Context context) { + var cm = configMap(resource); + + context.getClient().resource(cm).serverSideApply(); + + resource.setStatus(new SimplePerformanceTestStatus()); + resource.getStatus().setValue(resource.getSpec().getValue()); + return UpdateControl.patchStatus(resource); + } + + private ConfigMap configMap(SimplePerformanceTestResource primary) { + var cm = + new ConfigMapBuilder() + .withMetadata( + new ObjectMetaBuilder() + .withName(primary.getMetadata().getName()) + .withNamespace(primary.getMetadata().getNamespace()) + .build()) + .withData(Map.of(KEY, primary.getSpec().getValue())) + .build(); + cm.addOwnerReference(primary); + return cm; + } + + @Override + public DeleteControl cleanup( + SimplePerformanceTestResource resource, Context context) { + return DeleteControl.defaultDelete(); + } + + @Override + public List> prepareEventSources( + EventSourceContext context) { + InformerEventSource es = + new InformerEventSource<>( + InformerEventSourceConfiguration.from( + ConfigMap.class, SimplePerformanceTestResource.class) + .withNamespacesInheritedFromController() + .build(), + context); + return List.of(es); + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java new file mode 100644 index 0000000000..cc1e5dc53d --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestResource.java @@ -0,0 +1,29 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +import io.fabric8.kubernetes.api.model.Namespaced; +import io.fabric8.kubernetes.client.CustomResource; +import io.fabric8.kubernetes.model.annotation.Group; +import io.fabric8.kubernetes.model.annotation.ShortNames; +import io.fabric8.kubernetes.model.annotation.Version; + +@Group("sample.javaoperatorsdk") +@Version("v1") +@ShortNames("spt") +public class SimplePerformanceTestResource + extends CustomResource + implements Namespaced {} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java new file mode 100644 index 0000000000..16c54e0f09 --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestSpec.java @@ -0,0 +1,29 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +public class SimplePerformanceTestSpec { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java new file mode 100644 index 0000000000..01df9a8ffe --- /dev/null +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestStatus.java @@ -0,0 +1,34 @@ +/* + * Copyright Java Operator SDK Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.javaoperatorsdk.operator.baseapi.performance; + +public class SimplePerformanceTestStatus { + + private String value; + + public String getValue() { + return value; + } + + public void setValue(String configMapStatus) { + this.value = configMapStatus; + } + + @Override + public String toString() { + return "SimplePerformanceTestStatus{" + "configMapStatus='" + value + '\'' + '}'; + } +} diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java index efd0377311..b59c393b04 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java @@ -29,6 +29,6 @@ public void setConfigMapStatus(String configMapStatus) { @Override public String toString() { - return "TestCustomResourceStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; + return "SimplePerformanceTestStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; } } From c5c4c8db176ad4664b2d14b397c4dfe00e122bcf Mon Sep 17 00:00:00 2001 From: Chris Laprun Date: Wed, 28 Jan 2026 18:49:38 +0100 Subject: [PATCH 02/10] fix: improper name Signed-off-by: Chris Laprun --- .../baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java index b59c393b04..c19eb3802d 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/ssaissue/finalizer/SSAFinalizerIssueStatus.java @@ -29,6 +29,6 @@ public void setConfigMapStatus(String configMapStatus) { @Override public String toString() { - return "SimplePerformanceTestStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; + return "SSAFinalizerIssueStatus{" + "configMapStatus='" + configMapStatus + '\'' + '}'; } } From 8dbbaf7a62f322896bf02ee7293a7bf69bc96b0a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Fri, 30 Jan 2026 15:55:11 +0100 Subject: [PATCH 03/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/scripts/performance-to-markdown.py | 47 ++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100755 .github/scripts/performance-to-markdown.py diff --git a/.github/scripts/performance-to-markdown.py b/.github/scripts/performance-to-markdown.py new file mode 100755 index 0000000000..c9eed172d8 --- /dev/null +++ b/.github/scripts/performance-to-markdown.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import json +import sys +import os + +def convert_to_markdown(json_file): + """Convert performance test JSON results to markdown format""" + try: + with open(json_file, 'r') as f: + data = json.load(f) + except FileNotFoundError: + return "## Performance Test Results\n\nNo performance test results found." + except json.JSONDecodeError: + return "## Performance Test Results\n\nError parsing performance test results." + + markdown = "## Performance Test Results\n\n" + + if 'summaries' not in data or not data['summaries']: + return markdown + "No test summaries available." + + for summary in data['summaries']: + name = summary.get('name', 'Unknown Test') + duration = summary.get('duration', 0) + processors = summary.get('numberOfProcessors', 0) + max_memory = summary.get('maxMemory', 0) + + # Convert memory from bytes to GB + max_memory_gb = max_memory / (1024 ** 3) if max_memory > 0 else 0 + + markdown += f"### {name}\n\n" + markdown += "| Metric | Value |\n" + markdown += "|--------|-------|\n" + markdown += f"| Duration | {duration} ms |\n" + markdown += f"| Processors | {processors} |\n" + markdown += f"| Max Memory | {max_memory_gb:.2f} GB |\n" + markdown += "\n" + + return markdown + +if __name__ == "__main__": + if len(sys.argv) != 2: + print("Usage: performance-to-markdown.py ") + sys.exit(1) + + json_file = sys.argv[1] + markdown = convert_to_markdown(json_file) + print(markdown) From 98c5427a3a5cea49c6290cc5ff72bd204e52d7cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 09:55:44 +0100 Subject: [PATCH 04/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/workflows/pr.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 57e655ca4e..54d332ee4e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -37,7 +37,6 @@ jobs: name: Post Performance Results runs-on: ubuntu-latest needs: build - if: always() permissions: pull-requests: write steps: From 0d1af4cccf7e0e78ffa46d6095c1ab8c6c50c9f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 11:04:59 +0100 Subject: [PATCH 05/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/workflows/integration-tests.yml | 4 ++-- .github/workflows/pr.yml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e795350c6c..9b95bbd23c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -57,8 +57,8 @@ jobs: ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml - name: Upload performance test results - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6 with: - name: performance-results-java${{ inputs.java-version }}-k8s${{ inputs.kube-version }}-${{ inputs.http-client }} + name: performance-results-run-${{ github.run_id }}-java${{ inputs.java-version }}-k8s${{ inputs.kube-version }}-${{ inputs.http-client }} path: operator-framework/target/performance_test_result.json if-no-files-found: ignore diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 54d332ee4e..c0a6041349 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -43,11 +43,11 @@ jobs: - uses: actions/checkout@v6 - name: Download all performance artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7 with: - pattern: performance-results-* + pattern: performance-results-run-${{ github.run_id }}* path: performance-results - merge-multiple: true + merge-multiple: false - name: Check for performance results id: check_results From 337c3215ccc5d685f8db0200e5f6bbd23b999604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 11:05:41 +0100 Subject: [PATCH 06/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/workflows/pr.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index c0a6041349..0abd758950 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -53,8 +53,10 @@ jobs: id: check_results run: | if [ -d "performance-results" ] && [ "$(ls -A performance-results/*.json 2>/dev/null)" ]; then + echo "Has results" echo "has_results=true" >> $GITHUB_OUTPUT else + echo "Has NO results" echo "has_results=false" >> $GITHUB_OUTPUT fi From 02103d05f1a59e13909785f937c543dc342ae7a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 13:08:08 +0100 Subject: [PATCH 07/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/scripts/performance-to-markdown.py | 93 +++++++++++++------ .github/workflows/integration-tests.yml | 2 +- .github/workflows/pr.yml | 12 +-- .../performance/PerformanceTestResult.java | 17 ++++ .../performance/SimplePerformanceTestIT.java | 20 +++- 5 files changed, 105 insertions(+), 39 deletions(-) diff --git a/.github/scripts/performance-to-markdown.py b/.github/scripts/performance-to-markdown.py index c9eed172d8..3ff2485844 100755 --- a/.github/scripts/performance-to-markdown.py +++ b/.github/scripts/performance-to-markdown.py @@ -2,46 +2,87 @@ import json import sys import os +import glob -def convert_to_markdown(json_file): +def load_all_summaries(json_files): + """Load summaries from all JSON files and group by test name""" + summaries_by_name = {} + + for json_file in json_files: + try: + with open(json_file, 'r') as f: + data = json.load(f) + + if 'summaries' in data and data['summaries']: + for summary in data['summaries']: + name = summary.get('name', 'Unknown Test') + if name not in summaries_by_name: + summaries_by_name[name] = [] + summaries_by_name[name].append(summary) + except (FileNotFoundError, json.JSONDecodeError) as e: + print(f"Warning: Error processing {json_file}: {e}", file=sys.stderr) + continue + + return summaries_by_name + +def convert_to_markdown(json_files): """Convert performance test JSON results to markdown format""" - try: - with open(json_file, 'r') as f: - data = json.load(f) - except FileNotFoundError: + summaries_by_name = load_all_summaries(json_files) + + if not summaries_by_name: return "## Performance Test Results\n\nNo performance test results found." - except json.JSONDecodeError: - return "## Performance Test Results\n\nError parsing performance test results." markdown = "## Performance Test Results\n\n" - if 'summaries' not in data or not data['summaries']: - return markdown + "No test summaries available." + # Create a table for each test name + for name, summaries in sorted(summaries_by_name.items()): + markdown += f"### {name}\n\n" + markdown += "| Duration (ms) | Max Memory (GB) | Processors | Parameters |\n" + markdown += "|---------------|-----------------|------------|------------|\n" - for summary in data['summaries']: - name = summary.get('name', 'Unknown Test') - duration = summary.get('duration', 0) - processors = summary.get('numberOfProcessors', 0) - max_memory = summary.get('maxMemory', 0) + for summary in summaries: + duration = summary.get('duration', 0) + processors = summary.get('numberOfProcessors', 0) + max_memory = summary.get('maxMemory', 0) - # Convert memory from bytes to GB - max_memory_gb = max_memory / (1024 ** 3) if max_memory > 0 else 0 + # Convert memory from bytes to GB + max_memory_gb = max_memory / (1024 ** 3) if max_memory > 0 else 0 + + # Extract dynamic properties (excluding standard fields) + standard_fields = {'name', 'duration', 'numberOfProcessors', 'maxMemory'} + params = [] + for key, value in summary.items(): + if key not in standard_fields: + params.append(f"{key}={value}") + + params_str = ", ".join(params) if params else "-" + + markdown += f"| {duration} | {max_memory_gb:.2f} | {processors} | {params_str} |\n" - markdown += f"### {name}\n\n" - markdown += "| Metric | Value |\n" - markdown += "|--------|-------|\n" - markdown += f"| Duration | {duration} ms |\n" - markdown += f"| Processors | {processors} |\n" - markdown += f"| Max Memory | {max_memory_gb:.2f} GB |\n" markdown += "\n" return markdown if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: performance-to-markdown.py ") + if len(sys.argv) < 2: + print("Usage: performance-to-markdown.py [json_file2 ...]") + print(" or: performance-to-markdown.py ") sys.exit(1) - json_file = sys.argv[1] - markdown = convert_to_markdown(json_file) + # Collect all JSON files from arguments (supporting both direct files and glob patterns) + json_files = [] + for arg in sys.argv[1:]: + if '*' in arg: + json_files.extend(glob.glob(arg, recursive=True)) + else: + json_files.append(arg) + + # Filter to only existing files + json_files = [f for f in json_files if os.path.isfile(f)] + + if not json_files: + print("## Performance Test Results\n\nNo performance test results found.") + sys.exit(0) + + markdown = convert_to_markdown(json_files) print(markdown) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 9b95bbd23c..e7f1cd2e4f 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -44,7 +44,6 @@ jobs: minikube version: 'v1.36.0' kubernetes version: '${{ inputs.kube-version }}' github token: ${{ github.token }} - - name: "${{inputs.it-category}} integration tests (kube: ${{ inputs.kube-version }} / java: ${{ inputs.java-version }} / client: ${{ inputs.http-client }})" run: | if [ -z "${{inputs.it-category}}" ]; then @@ -52,6 +51,7 @@ jobs: else it_profile="integration-tests-${{inputs.it-category}}" fi + echo "{ "kube-version":"${{inputs.kube-version}}", "java-version":"${{ inputs.java-version }}","http-client":"${{inputs.http-client}}"}" >> run-properties.json echo "Using profile: ${it_profile}" ./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0abd758950..af39da37a0 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -52,7 +52,7 @@ jobs: - name: Check for performance results id: check_results run: | - if [ -d "performance-results" ] && [ "$(ls -A performance-results/*.json 2>/dev/null)" ]; then + if [ -d "performance-results" ] && [ "$(ls -A performance-results/**/*.json 2>/dev/null)" ]; then echo "Has results" echo "has_results=true" >> $GITHUB_OUTPUT else @@ -64,15 +64,7 @@ jobs: if: steps.check_results.outputs.has_results == 'true' id: convert run: | - echo "# Performance Test Results" > comment.md - echo "" >> comment.md - for file in performance-results/*.json; do - if [ -f "$file" ]; then - echo "Processing $file" - python3 .github/scripts/performance-to-markdown.py "$file" >> comment.md - echo "" >> comment.md - fi - done + python3 .github/scripts/performance-to-markdown.py performance-results/**/*.json > comment.md - name: Post PR comment if: steps.check_results.outputs.has_results == 'true' && github.event_name == 'pull_request' diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java index 61b0feacf2..901e2136e0 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java @@ -15,10 +15,27 @@ */ package io.javaoperatorsdk.operator.baseapi.performance; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; + +import java.util.HashMap; import java.util.List; +import java.util.Map; public class PerformanceTestResult { + private final Map additionalProperties = new HashMap<>(); + + @JsonAnySetter + public void addProperty(String key, Object value) { + additionalProperties.put(key, value); + } + + @JsonAnyGetter + public Map getProperties() { + return additionalProperties; + } + private List summaries; public List getSummaries() { diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java index bc4eab008a..aedc19be38 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java @@ -18,6 +18,7 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -39,7 +40,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; public class SimplePerformanceTestIT { - + ObjectMapper objectMapper = new ObjectMapper(); private static final Logger log = LoggerFactory.getLogger(SimplePerformanceTestIT.class); public static final String INITIAL_VALUE = "initialValue"; public static final String RESOURCE_NAME_PREFIX = "resource"; @@ -89,19 +90,34 @@ void simpleNaivePerformanceTest() { private void saveResults(long duration) { try { var result = new PerformanceTestResult(); + getRunProperties().forEach((k,v)->result.addProperty(k,v)); var summary = new PerformanceTestSummary(); result.setSummaries(List.of(summary)); summary.setName("Naive performance test"); summary.setDuration(duration); summary.setNumberOfProcessors(Runtime.getRuntime().availableProcessors()); summary.setMaxMemory(Runtime.getRuntime().maxMemory()); - var objectMapper = new ObjectMapper(); + objectMapper.writeValue(new File("target/performance_test_result.json"), result); } catch (IOException e) { throw new RuntimeException(e); } } + private Map getRunProperties() { + try { + File runProperties = new File("../run-properties.json"); + if (runProperties.exists()) { + return objectMapper.readValue(runProperties, HashMap.class); + } else { + log.warn("No run properties file found"); + return Map.of(); + } + } catch (IOException e) { + throw new RuntimeException(e); + } + } + private void createResources(int startIndex, int number, String value) { try { List> callables = new ArrayList<>(number); From 9a194e998f0b0b1b9a8a2cfcc6f5eb9a6c8e6e42 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 13:23:59 +0100 Subject: [PATCH 08/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../performance/PerformanceTestResult.java | 6 ++--- .../performance/SimplePerformanceTestIT.java | 26 +++++++++---------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java index 901e2136e0..9a88a90767 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/PerformanceTestResult.java @@ -15,13 +15,13 @@ */ package io.javaoperatorsdk.operator.baseapi.performance; -import com.fasterxml.jackson.annotation.JsonAnyGetter; -import com.fasterxml.jackson.annotation.JsonAnySetter; - import java.util.HashMap; import java.util.List; import java.util.Map; +import com.fasterxml.jackson.annotation.JsonAnyGetter; +import com.fasterxml.jackson.annotation.JsonAnySetter; + public class PerformanceTestResult { private final Map additionalProperties = new HashMap<>(); diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java index aedc19be38..1b1180bfa9 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java @@ -90,7 +90,7 @@ void simpleNaivePerformanceTest() { private void saveResults(long duration) { try { var result = new PerformanceTestResult(); - getRunProperties().forEach((k,v)->result.addProperty(k,v)); + getRunProperties().forEach((k, v) -> result.addProperty(k, v)); var summary = new PerformanceTestSummary(); result.setSummaries(List.of(summary)); summary.setName("Naive performance test"); @@ -104,19 +104,19 @@ private void saveResults(long duration) { } } - private Map getRunProperties() { - try { - File runProperties = new File("../run-properties.json"); - if (runProperties.exists()) { - return objectMapper.readValue(runProperties, HashMap.class); - } else { - log.warn("No run properties file found"); - return Map.of(); - } - } catch (IOException e) { - throw new RuntimeException(e); - } + private Map getRunProperties() { + try { + File runProperties = new File("../run-properties.json"); + if (runProperties.exists()) { + return objectMapper.readValue(runProperties, HashMap.class); + } else { + log.warn("No run properties file found"); + return Map.of(); + } + } catch (IOException e) { + throw new RuntimeException(e); } + } private void createResources(int startIndex, int number, String value) { try { From 8540905d9c445d1c0e195c0ac182cff18fb62682 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 13:59:06 +0100 Subject: [PATCH 09/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .../operator/baseapi/performance/SimplePerformanceTestIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java index 1b1180bfa9..4a22b52379 100644 --- a/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java +++ b/operator-framework/src/test/java/io/javaoperatorsdk/operator/baseapi/performance/SimplePerformanceTestIT.java @@ -17,6 +17,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.ArrayList; import java.util.HashMap; import java.util.List; @@ -108,6 +109,7 @@ private Map getRunProperties() { try { File runProperties = new File("../run-properties.json"); if (runProperties.exists()) { + log.debug("Run properties: {}", Files.readString(runProperties.toPath())); return objectMapper.readValue(runProperties, HashMap.class); } else { log.warn("No run properties file found"); From 5a64f9033bfc425a0c191a267200ede639444395 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Attila=20M=C3=A9sz=C3=A1ros?= Date: Tue, 3 Feb 2026 14:34:09 +0100 Subject: [PATCH 10/10] wip MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Attila Mészáros --- .github/workflows/integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index e7f1cd2e4f..af9ac76b6c 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -51,7 +51,7 @@ jobs: else it_profile="integration-tests-${{inputs.it-category}}" fi - echo "{ "kube-version":"${{inputs.kube-version}}", "java-version":"${{ inputs.java-version }}","http-client":"${{inputs.http-client}}"}" >> run-properties.json + echo "{ \"kube-version\":\"${{inputs.kube-version}}\", \"java-version\":\"${{ inputs.java-version }}\", \"http-client\":\"${{inputs.http-client}}\" }" >> run-properties.json echo "Using profile: ${it_profile}" ./mvnw ${MAVEN_ARGS} -T1C -B install -DskipTests -Pno-apt --file pom.xml ./mvnw ${MAVEN_ARGS} -T1C -B package -P${it_profile} -Dfabric8-httpclient-impl.name=${{inputs.http-client}} --file pom.xml