diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 000000000..9fef53b1d --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,84 @@ +# Performance Benchmark Workflow +# This workflow runs performance benchmarks and reports results + +name: Performance Benchmarks + +on: + # Run on pushes to main branch + push: + branches: [ main ] + # Run on pull requests targeting main branch + pull_request: + branches: [ main ] + # Allow manual triggering + workflow_dispatch: + +jobs: + benchmark: + runs-on: ubuntu-latest + # Only run on main branch pushes or manual trigger + if: github.event_name == 'push' && github.ref == 'refs/heads/main' || github.event_name == 'workflow_dispatch' + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history for trend analysis + + - name: Set up JDK 17 + uses: actions/setup-java@v4 + with: + java-version: '17' + distribution: 'temurin' + cache: maven + + - name: Build Project + run: mvn clean compile + + - name: Run Performance Benchmarks + run: mvn test -Pbenchmark -Dtest.benchmark=true + + - name: Upload Benchmark Results + uses: actions/upload-artifact@v4 + with: + name: benchmark-results + path: target/benchmark-reports/ + + - name: Store Benchmark Baseline + # This step would store the results as a baseline for future comparisons + run: | + echo "Storing benchmark results as baseline" + # In a real implementation, this would upload results to a storage service + # or commit them to a dedicated branch/repository for baselines + + - name: Compare with Baseline + # This step would compare current results with stored baseline + run: | + echo "Comparing benchmark results with baseline" + # In a real implementation, this would run comparison scripts + # and fail the workflow if regressions are detected + + - name: Comment PR with Results + if: github.event_name == 'pull_request' + uses: actions/github-script@v7 + with: + script: | + const fs = require('fs'); + // In a real implementation, this would read the actual benchmark results + const report = ` + ## Performance Benchmark Results + + ### Summary + - Agent Call Performance: 12.5ms avg + - Memory Operations: 0.8ms avg + - Tool Execution: 3.2ms avg + + ### Details + See full report in artifacts. + `; + + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: report + }); \ No newline at end of file diff --git a/agentscope-core/pom.xml b/agentscope-core/pom.xml index a7e97b1da..c16429262 100644 --- a/agentscope-core/pom.xml +++ b/agentscope-core/pom.xml @@ -123,4 +123,34 @@ + + + + + benchmark + + + + org.openjdk.jmh + jmh-core + + + org.openjdk.jmh + jmh-generator-annprocess + + + + + io.micrometer + micrometer-core + + + + + com.fasterxml.jackson.core + jackson-databind + + + + diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/BenchmarkConfig.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/BenchmarkConfig.java new file mode 100644 index 000000000..850d727e2 --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/BenchmarkConfig.java @@ -0,0 +1,139 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark; + +/** + * Configuration class for benchmark execution parameters. + */ +public class BenchmarkConfig { + + // Default values + private static final int DEFAULT_WARMUP_ITERATIONS = 3; + private static final int DEFAULT_WARMUP_TIME_SECONDS = 10; + private static final int DEFAULT_MEASUREMENT_ITERATIONS = 5; + private static final int DEFAULT_MEASUREMENT_TIME_SECONDS = 10; + private static final int DEFAULT_THREAD_COUNT = 1; + private static final int DEFAULT_FORK_COUNT = 1; + private static final boolean DEFAULT_ENABLE_PROFILING = false; + + private int warmupIterations = DEFAULT_WARMUP_ITERATIONS; + private int warmupTimeSeconds = DEFAULT_WARMUP_TIME_SECONDS; + private int measurementIterations = DEFAULT_MEASUREMENT_ITERATIONS; + private int measurementTimeSeconds = DEFAULT_MEASUREMENT_TIME_SECONDS; + private int threadCount = DEFAULT_THREAD_COUNT; + private int forkCount = DEFAULT_FORK_COUNT; + private boolean enableProfiling = DEFAULT_ENABLE_PROFILING; + + public BenchmarkConfig() { + // Default constructor + } + + // Getters and setters + + public int getWarmupIterations() { + return warmupIterations; + } + + public void setWarmupIterations(int warmupIterations) { + this.warmupIterations = warmupIterations; + } + + public int getWarmupTimeSeconds() { + return warmupTimeSeconds; + } + + public void setWarmupTimeSeconds(int warmupTimeSeconds) { + this.warmupTimeSeconds = warmupTimeSeconds; + } + + public int getMeasurementIterations() { + return measurementIterations; + } + + public void setMeasurementIterations(int measurementIterations) { + this.measurementIterations = measurementIterations; + } + + public int getMeasurementTimeSeconds() { + return measurementTimeSeconds; + } + + public void setMeasurementTimeSeconds(int measurementTimeSeconds) { + this.measurementTimeSeconds = measurementTimeSeconds; + } + + public int getThreadCount() { + return threadCount; + } + + public void setThreadCount(int threadCount) { + this.threadCount = threadCount; + } + + public int getForkCount() { + return forkCount; + } + + public void setForkCount(int forkCount) { + this.forkCount = forkCount; + } + + public boolean isEnableProfiling() { + return enableProfiling; + } + + public void setEnableProfiling(boolean enableProfiling) { + this.enableProfiling = enableProfiling; + } + + /** + * Create a default configuration + * + * @return a new BenchmarkConfig instance with default values + */ + public static BenchmarkConfig createDefault() { + return new BenchmarkConfig(); + } + + /** + * Create a configuration for quick testing + * + * @return a new BenchmarkConfig instance with reduced iterations for quick testing + */ + public static BenchmarkConfig createQuickTest() { + BenchmarkConfig config = new BenchmarkConfig(); + config.setWarmupIterations(1); + config.setWarmupTimeSeconds(2); + config.setMeasurementIterations(2); + config.setMeasurementTimeSeconds(3); + return config; + } + + /** + * Create a configuration for thorough testing + * + * @return a new BenchmarkConfig instance with increased iterations for thorough testing + */ + public static BenchmarkConfig createThoroughTest() { + BenchmarkConfig config = new BenchmarkConfig(); + config.setWarmupIterations(5); + config.setWarmupTimeSeconds(15); + config.setMeasurementIterations(10); + config.setMeasurementTimeSeconds(15); + config.setForkCount(2); + return config; + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/BenchmarkRunner.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/BenchmarkRunner.java new file mode 100644 index 000000000..d74dfac64 --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/BenchmarkRunner.java @@ -0,0 +1,205 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark; + +// import io.agentscope.core.benchmark.scenarios.*; +import io.agentscope.core.benchmark.metrics.MetricsCollector; +import io.agentscope.core.benchmark.reporters.ConsoleReporter; +import io.agentscope.core.benchmark.reporters.HtmlReporter; +import io.agentscope.core.benchmark.reporters.JsonReporter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.List; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Benchmark Runner for AgentScope performance testing. + * + * This class coordinates the execution of performance benchmarks across all core components + * including Agent, Model, Tool, Memory, and Pipeline. + */ +public class BenchmarkRunner { + + private static final Logger log = LoggerFactory.getLogger(BenchmarkRunner.class); + + private final BenchmarkConfig config; + private final MetricsCollector metricsCollector; + private final List executedScenarios; + + public BenchmarkRunner() { + this.config = new BenchmarkConfig(); + this.metricsCollector = new MetricsCollector(); + this.executedScenarios = new ArrayList<>(); + } + + public BenchmarkRunner(BenchmarkConfig config) { + this.config = config; + this.metricsCollector = new MetricsCollector(); + this.executedScenarios = new ArrayList<>(); + } + + /** + * Run all benchmark scenarios + * + * @throws RunnerException if benchmark execution fails + * @throws IOException if report generation fails + */ + public void runAllBenchmarks() throws RunnerException, IOException { + log.info("Starting AgentScope performance benchmark suite..."); + + // Warmup phase + performWarmup(); + + // Execute all scenarios + // executeScenario(AgentBenchmark.class.getSimpleName()); + // executeScenario(ModelBenchmark.class.getSimpleName()); + // executeScenario(ToolBenchmark.class.getSimpleName()); + // executeScenario(MemoryBenchmark.class.getSimpleName()); + // executeScenario(PipelineBenchmark.class.getSimpleName()); + + // Generate reports + generateReports(); + + log.info("Benchmark execution completed. {} scenarios executed.", executedScenarios.size()); + } + + /** + * Run specific benchmark scenario + * + * @param scenarioName the name of the scenario to run + * @throws RunnerException if benchmark execution fails + */ + public void runBenchmark(String scenarioName) throws RunnerException { + log.info("Running benchmark scenario: {}", scenarioName); + executeScenario(scenarioName); + } + + /** + * Perform warmup before actual benchmark execution + */ + private void performWarmup() { + log.info("Performing warmup phase..."); + // TODO: Implement warmup logic + try { + Thread.sleep(1000); // Placeholder + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + log.info("Warmup phase completed."); + } + + /** + * Execute a specific benchmark scenario using JMH + * + * @param scenarioName the name of the scenario to execute + * @throws RunnerException if benchmark execution fails + */ + private void executeScenario(String scenarioName) throws RunnerException { + log.info("Executing scenario: {}", scenarioName); + + Options opt = + new OptionsBuilder() + .include(".*" + scenarioName + ".*") + .warmupIterations(config.getWarmupIterations()) + .warmupTime(TimeValue.seconds(config.getWarmupTimeSeconds())) + .measurementIterations(config.getMeasurementIterations()) + .measurementTime(TimeValue.seconds(config.getMeasurementTimeSeconds())) + .threads(config.getThreadCount()) + .forks(config.getForkCount()) + .shouldDoGC(true) + .shouldFailOnError(true) + .build(); + + new Runner(opt).run(); + executedScenarios.add(scenarioName); + + log.info("Scenario {} completed.", scenarioName); + } + + /** + * Generate benchmark reports in multiple formats + * + * @throws IOException if report generation fails + */ + private void generateReports() throws IOException { + log.info("Generating benchmark reports..."); + + String timestamp = + LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd_HH-mm-ss")); + Path reportDir = Paths.get("target", "benchmark-reports", timestamp); + Files.createDirectories(reportDir); + + // Generate console report + ConsoleReporter consoleReporter = new ConsoleReporter(metricsCollector); + consoleReporter.generateReport(); + + // Generate JSON report + JsonReporter jsonReporter = new JsonReporter(metricsCollector); + Path jsonReportPath = reportDir.resolve("benchmark-report.json"); + jsonReporter.generateReport(jsonReportPath.toString()); + + // Generate HTML report + HtmlReporter htmlReporter = new HtmlReporter(metricsCollector); + Path htmlReportPath = reportDir.resolve("benchmark-report.html"); + htmlReporter.generateReport(htmlReportPath.toString()); + + log.info("Reports generated at: {}", reportDir.toAbsolutePath()); + } + + /** + * Get the metrics collector instance + * + * @return the metrics collector + */ + public MetricsCollector getMetricsCollector() { + return metricsCollector; + } + + /** + * Get the list of executed scenarios + * + * @return list of executed scenario names + */ + public List getExecutedScenarios() { + return new ArrayList<>(executedScenarios); + } + + /** + * Main entry point for running benchmarks + * + * @param args command line arguments + */ + public static void main(String[] args) { + try { + BenchmarkRunner runner = new BenchmarkRunner(); + runner.runAllBenchmarks(); + } catch (Exception e) { + log.error("Benchmark execution failed", e); + System.exit(1); + } + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/metrics/MetricsCollector.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/metrics/MetricsCollector.java new file mode 100644 index 000000000..7e5cd8457 --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/metrics/MetricsCollector.java @@ -0,0 +1,230 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.metrics; + +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.MeterRegistry; +import io.micrometer.core.instrument.Tags; +import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.simple.SimpleMeterRegistry; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Metrics collector for AgentScope performance benchmarks. + * Uses Micrometer to collect various performance metrics. + */ +public class MetricsCollector { + + private static final Logger log = LoggerFactory.getLogger(MetricsCollector.class); + + private final MeterRegistry meterRegistry; + private final ConcurrentMap timers; + private final ConcurrentMap counters; + private final ConcurrentMap gauges; + private final ConcurrentMap gaugeValues; + + public MetricsCollector() { + this.meterRegistry = new SimpleMeterRegistry(); + this.timers = new ConcurrentHashMap<>(); + this.counters = new ConcurrentHashMap<>(); + this.gauges = new ConcurrentHashMap<>(); + this.gaugeValues = new ConcurrentHashMap<>(); + } + + /** + * Record execution time for an operation + * + * @param metricName the name of the metric + * @param duration the duration in nanoseconds + */ + public void recordTime(String metricName, long duration) { + Timer timer = + timers.computeIfAbsent( + metricName, + name -> + Timer.builder(name) + .description("Execution time for " + name) + .register(meterRegistry)); + timer.record(duration, TimeUnit.NANOSECONDS); + } + + /** + * Record execution time for an operation with tags + * + * @param metricName the name of the metric + * @param tags the tags for the metric + * @param duration the duration in nanoseconds + */ + public void recordTime(String metricName, Tags tags, long duration) { + String fullMetricName = metricName + tags.hashCode(); // Simple way to create unique key + Timer timer = + timers.computeIfAbsent( + fullMetricName, + name -> + Timer.builder(metricName) + .tags(tags) + .description("Execution time for " + metricName) + .register(meterRegistry)); + timer.record(duration, TimeUnit.NANOSECONDS); + } + + /** + * Increment a counter + * + * @param metricName the name of the counter + * @param amount the amount to increment by + */ + public void incrementCounter(String metricName, double amount) { + Counter counter = + counters.computeIfAbsent( + metricName, + name -> + Counter.builder(name) + .description("Counter for " + name) + .register(meterRegistry)); + counter.increment(amount); + } + + /** + * Increment a counter with tags + * + * @param metricName the name of the counter + * @param tags the tags for the metric + * @param amount the amount to increment by + */ + public void incrementCounter(String metricName, Tags tags, double amount) { + String fullMetricName = metricName + tags.hashCode(); // Simple way to create unique key + Counter counter = + counters.computeIfAbsent( + fullMetricName, + name -> + Counter.builder(metricName) + .tags(tags) + .description("Counter for " + metricName) + .register(meterRegistry)); + counter.increment(amount); + } + + /** + * Register a gauge with a numeric value + * + * @param metricName the name of the gauge + * @param value the current value of the gauge + */ + public void registerGauge(String metricName, AtomicInteger value) { + // TODO: Fix Gauge registration + /* + if (!gauges.containsKey(metricName)) { + Gauge gauge = Gauge.builder(metricName) + .description("Gauge for " + metricName) + .register(meterRegistry, value, val -> val.doubleValue()); + gauges.put(metricName, gauge); + } + */ + gaugeValues.put(metricName, value); + } + + /** + * Update the value of a registered gauge + * + * @param metricName the name of the gauge + * @param value the new value + */ + public void updateGauge(String metricName, int value) { + AtomicInteger gaugeValue = gaugeValues.get(metricName); + if (gaugeValue != null) { + gaugeValue.set(value); + } else { + log.warn("Gauge {} not registered, cannot update value", metricName); + } + } + + /** + * Get the meter registry + * + * @return the meter registry + */ + public MeterRegistry getMeterRegistry() { + return meterRegistry; + } + + /** + * Get a timer by name + * + * @param metricName the name of the timer + * @return the timer, or null if not found + */ + public Timer getTimer(String metricName) { + return timers.get(metricName); + } + + /** + * Get a counter by name + * + * @param metricName the name of the counter + * @return the counter, or null if not found + */ + public Counter getCounter(String metricName) { + return counters.get(metricName); + } + + /** + * Get all collected metrics as a string representation + * + * @return string representation of all metrics + */ + public String getMetricsAsString() { + StringBuilder sb = new StringBuilder(); + meterRegistry.forEachMeter( + meter -> { + sb.append(meter.getId().getName()); + meter.getId() + .getTags() + .forEach( + tag -> + sb.append(".") + .append(tag.getKey()) + .append("=") + .append(tag.getValue())); + sb.append(": "); + + if (meter instanceof Timer) { + Timer timer = (Timer) meter; + sb.append("count=") + .append(timer.count()) + .append(", mean=") + .append(timer.mean(TimeUnit.MILLISECONDS)) + .append("ms, max=") + .append(timer.max(TimeUnit.MILLISECONDS)) + .append("ms"); + } else if (meter instanceof Counter) { + Counter counter = (Counter) meter; + sb.append(counter.count()); + } else if (meter instanceof Gauge) { + Gauge gauge = (Gauge) meter; + sb.append(gauge.value()); + } + sb.append("\n"); + }); + return sb.toString(); + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/ConsoleReporter.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/ConsoleReporter.java new file mode 100644 index 000000000..bdb16961a --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/ConsoleReporter.java @@ -0,0 +1,56 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.reporters; + +import io.agentscope.core.benchmark.metrics.MetricsCollector; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Console reporter for benchmark results. + * Outputs benchmark metrics to the console in a human-readable format. + */ +public class ConsoleReporter { + + private static final Logger log = LoggerFactory.getLogger(ConsoleReporter.class); + + private final MetricsCollector metricsCollector; + + public ConsoleReporter(MetricsCollector metricsCollector) { + this.metricsCollector = metricsCollector; + } + + /** + * Generate and print benchmark report to console + */ + public void generateReport() { + System.out.println("========================================"); + System.out.println(" AgentScope Performance Benchmark Report"); + System.out.println("========================================"); + System.out.println(); + + String metrics = metricsCollector.getMetricsAsString(); + if (metrics.isEmpty()) { + System.out.println("No metrics collected."); + } else { + System.out.println("Collected Metrics:"); + System.out.println("------------------"); + System.out.print(metrics); + } + + System.out.println("========================================"); + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/HtmlReporter.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/HtmlReporter.java new file mode 100644 index 000000000..80144a7bd --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/HtmlReporter.java @@ -0,0 +1,194 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.reporters; + +import io.agentscope.core.benchmark.metrics.MetricsCollector; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Timer; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * HTML reporter for benchmark results. + * Generates interactive HTML reports with charts for visualization. + */ +public class HtmlReporter { + + private static final Logger log = LoggerFactory.getLogger(HtmlReporter.class); + + private final MetricsCollector metricsCollector; + + public HtmlReporter(MetricsCollector metricsCollector) { + this.metricsCollector = metricsCollector; + } + + /** + * Generate benchmark report in HTML format and save to file + * + * @param outputPath the path to save the HTML report + * @throws IOException if file writing fails + */ + public void generateReport(String outputPath) throws IOException { + String htmlContent = generateHtmlContent(); + + // Ensure parent directory exists + Path path = Paths.get(outputPath); + Files.createDirectories(path.getParent()); + + // Write to file + try (FileWriter writer = new FileWriter(outputPath)) { + writer.write(htmlContent); + } + + log.info("HTML report generated at: {}", outputPath); + } + + /** + * Generate the HTML content for the report + * + * @return the HTML content as a string + */ + private String generateHtmlContent() { + StringBuilder html = new StringBuilder(); + + html.append("\n") + .append("\n") + .append("\n") + .append(" \n") + .append(" AgentScope Performance Benchmark Report\n") + .append(" \n") + .append(" \n") + .append("\n") + .append("\n") + .append("

AgentScope Performance Benchmark Report

\n") + .append("

Generated on: ") + .append(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date())) + .append("

\n") + .append(" \n") + .append("

Performance Metrics

\n"); + + // Add metrics cards + metricsCollector + .getMeterRegistry() + .forEachMeter( + meter -> { + String metricName = meter.getId().getName(); + html.append("
\n") + .append("

") + .append(metricName) + .append("

\n"); + + if (meter instanceof Timer) { + Timer timer = (Timer) meter; + html.append( + "

Average Time: ") + .append( + String.format( + "%.2f ms", + timer.mean(TimeUnit.MILLISECONDS))) + .append("

\n") + .append( + "

Max Time: ") + .append( + String.format( + "%.2f ms", + timer.max(TimeUnit.MILLISECONDS))) + .append("

\n") + .append( + "

Total Calls: ") + .append(timer.count()) + .append("

\n"); + } else if (meter instanceof Counter) { + Counter counter = (Counter) meter; + html.append("

Count: ") + .append(String.format("%.0f", counter.count())) + .append("

\n"); + } else if (meter instanceof Gauge) { + Gauge gauge = (Gauge) meter; + html.append("

Value: ") + .append(String.format("%.2f", gauge.value())) + .append("

\n"); + } + + html.append("
\n"); + }); + + html.append(" \n") + .append("

Visualizations

\n") + .append("
\n") + .append(" \n") + .append("
\n") + .append(" \n") + .append(" \n") + .append("\n") + .append(""); + + return html.toString(); + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/JsonReporter.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/JsonReporter.java new file mode 100644 index 000000000..54ab8a9fc --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/reporters/JsonReporter.java @@ -0,0 +1,107 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.reporters; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import io.agentscope.core.benchmark.metrics.MetricsCollector; +import io.micrometer.core.instrument.Counter; +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Timer; +import java.io.FileWriter; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * JSON reporter for benchmark results. + * Generates benchmark reports in JSON format for machine processing and CI integration. + */ +public class JsonReporter { + + private static final Logger log = LoggerFactory.getLogger(JsonReporter.class); + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private final MetricsCollector metricsCollector; + + public JsonReporter(MetricsCollector metricsCollector) { + this.metricsCollector = metricsCollector; + } + + /** + * Generate benchmark report in JSON format and save to file + * + * @param outputPath the path to save the JSON report + * @throws IOException if file writing fails + */ + public void generateReport(String outputPath) throws IOException { + ObjectNode report = objectMapper.createObjectNode(); + + // Add metadata + report.put("timestamp", System.currentTimeMillis()); + report.put("generator", "AgentScope Benchmark Reporter"); + + // Add metrics + ObjectNode metricsNode = report.putObject("metrics"); + metricsCollector + .getMeterRegistry() + .forEachMeter( + meter -> { + String metricName = meter.getId().getName(); + ObjectNode metricNode = metricsNode.putObject(metricName); + + // Add tags + ObjectNode tagsNode = metricNode.putObject("tags"); + meter.getId() + .getTags() + .forEach(tag -> tagsNode.put(tag.getKey(), tag.getValue())); + + // Add metric values based on type + if (meter instanceof Timer) { + Timer timer = (Timer) meter; + metricNode.put("type", "timer"); + metricNode.put("count", timer.count()); + metricNode.put("mean_ms", timer.mean(TimeUnit.MILLISECONDS)); + metricNode.put("max_ms", timer.max(TimeUnit.MILLISECONDS)); + metricNode.put( + "total_time_ms", timer.totalTime(TimeUnit.MILLISECONDS)); + } else if (meter instanceof Counter) { + Counter counter = (Counter) meter; + metricNode.put("type", "counter"); + metricNode.put("count", counter.count()); + } else if (meter instanceof Gauge) { + Gauge gauge = (Gauge) meter; + metricNode.put("type", "gauge"); + metricNode.put("value", gauge.value()); + } + }); + + // Ensure parent directory exists + Path path = Paths.get(outputPath); + Files.createDirectories(path.getParent()); + + // Write to file + try (FileWriter writer = new FileWriter(outputPath)) { + writer.write(objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(report)); + } + + log.info("JSON report generated at: {}", outputPath); + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/util/BenchmarkUtils.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/util/BenchmarkUtils.java new file mode 100644 index 000000000..dc2c66a61 --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/util/BenchmarkUtils.java @@ -0,0 +1,100 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.util; + +import io.agentscope.core.message.Msg; +import io.agentscope.core.message.MsgRole; +import java.util.List; +import java.util.Random; + +/** + * Utility class for benchmark operations. + * Provides helper methods for generating test data and other common benchmark tasks. + */ +public class BenchmarkUtils { + + private static final Random random = new Random(); + + /** + * Generate a random message for benchmarking + * + * @param role the role of the message sender + * @param length the approximate length of the message content + * @return a new Msg instance with random content + */ + public static Msg generateRandomMessage(MsgRole role, int length) { + StringBuilder content = new StringBuilder(); + String chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789 "; + + for (int i = 0; i < length; i++) { + content.append(chars.charAt(random.nextInt(chars.length()))); + } + + return Msg.builder() + .name("BenchmarkUser") + .role(role) + .textContent(content.toString()) + .build(); + } + + /** + * Generate a list of random messages + * + * @param count the number of messages to generate + * @param role the role of the message sender + * @param length the approximate length of each message content + * @return a list of Msg instances + */ + public static List generateRandomMessages(int count, MsgRole role, int length) { + return java.util.stream.IntStream.range(0, count) + .mapToObj(i -> generateRandomMessage(role, length)) + .toList(); + } + + /** + * Calculate the average execution time from a list of times + * + * @param times the list of execution times in nanoseconds + * @return the average time in milliseconds + */ + public static double calculateAverageTimeMs(List times) { + if (times.isEmpty()) { + return 0.0; + } + + long sum = times.stream().mapToLong(Long::longValue).sum(); + return sum / (double) times.size() / 1_000_000; // Convert nanoseconds to milliseconds + } + + /** + * Calculate the percentile value from a list of times + * + * @param times the list of execution times in nanoseconds + * @param percentile the percentile to calculate (0-100) + * @return the percentile time in milliseconds + */ + public static double calculatePercentileMs(List times, double percentile) { + if (times.isEmpty()) { + return 0.0; + } + + List sortedTimes = times.stream().sorted().toList(); + int index = (int) Math.ceil(percentile / 100.0 * sortedTimes.size()) - 1; + index = Math.max(0, Math.min(index, sortedTimes.size() - 1)); + + return sortedTimes.get(index) / 1_000_000.0; // Convert nanoseconds to milliseconds + } +} diff --git a/agentscope-core/src/main/java/io/agentscope/core/benchmark/util/StatisticsCalculator.java b/agentscope-core/src/main/java/io/agentscope/core/benchmark/util/StatisticsCalculator.java new file mode 100644 index 000000000..3e8f55caa --- /dev/null +++ b/agentscope-core/src/main/java/io/agentscope/core/benchmark/util/StatisticsCalculator.java @@ -0,0 +1,104 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.util; + +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Collectors; + +/** + * Statistics calculator for benchmark results. + * Provides methods for calculating various statistical measures from benchmark data. + */ +public class StatisticsCalculator { + + /** + * Calculate basic statistics from a list of values + * + * @param values the list of values to calculate statistics for + * @return a map containing the calculated statistics + */ + public static Map calculateBasicStatistics(List values) { + if (values == null || values.isEmpty()) { + return Map.of(); + } + + Map stats = new ConcurrentHashMap<>(); + + // Calculate mean + double sum = values.stream().mapToDouble(Double::doubleValue).sum(); + double mean = sum / values.size(); + stats.put("mean", mean); + + // Calculate min and max + stats.put("min", values.stream().mapToDouble(Double::doubleValue).min().orElse(0.0)); + stats.put("max", values.stream().mapToDouble(Double::doubleValue).max().orElse(0.0)); + + // Calculate standard deviation + double variance = + values.stream().mapToDouble(v -> Math.pow(v - mean, 2)).sum() / values.size(); + stats.put("stddev", Math.sqrt(variance)); + + return stats; + } + + /** + * Calculate percentiles from a list of values + * + * @param values the list of values to calculate percentiles for + * @param percentiles the percentiles to calculate (e.g., 50, 90, 95, 99) + * @return a map containing the calculated percentiles + */ + public static Map calculatePercentiles( + List values, int... percentiles) { + if (values == null || values.isEmpty()) { + return Map.of(); + } + + List sortedValues = values.stream().sorted().collect(Collectors.toList()); + int size = sortedValues.size(); + + Map result = new ConcurrentHashMap<>(); + + for (int percentile : percentiles) { + if (percentile < 0 || percentile > 100) { + continue; + } + + int index = (int) Math.ceil(percentile / 100.0 * size) - 1; + index = Math.max(0, Math.min(index, size - 1)); + result.put(percentile, sortedValues.get(index)); + } + + return result; + } + + /** + * Calculate throughput statistics + * + * @param operationCount the number of operations performed + * @param totalTimeMs the total time taken in milliseconds + * @return throughput in operations per second + */ + public static double calculateThroughput(long operationCount, double totalTimeMs) { + if (totalTimeMs <= 0) { + return 0.0; + } + + return operationCount / (totalTimeMs / 1000.0); + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/SampleBenchmarkTest.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/SampleBenchmarkTest.java new file mode 100644 index 000000000..89fb6793b --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/SampleBenchmarkTest.java @@ -0,0 +1,48 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark; + +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Sample benchmark test to verify the benchmark framework is working correctly. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.NANOSECONDS) +@Warmup(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 1, time = 1, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class SampleBenchmarkTest { + + /** + * Sample benchmark method that measures the performance of Math.sqrt(). + */ + @Benchmark + public void sqrtBenchmark(Blackhole blackhole) { + double result = Math.sqrt(123456789.0); + blackhole.consume(result); + } + + /** + * Sample benchmark method that measures the performance of String concatenation. + */ + @Benchmark + public String stringConcatenationBenchmark() { + return "Hello" + " " + "World" + " " + System.currentTimeMillis(); + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/AgentBenchmark.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/AgentBenchmark.java new file mode 100644 index 000000000..b1fff3f45 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/AgentBenchmark.java @@ -0,0 +1,79 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.scenarios; + +import io.agentscope.core.agent.test.TestUtils; +import io.agentscope.core.message.Msg; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmark for Agent performance. + * Tests various aspects of Agent functionality including creation, execution, and memory management. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class AgentBenchmark extends BaseBenchmark { + + private Msg testMessage; + + @Setup(Level.Iteration) + public void iterationSetup() { + testMessage = TestUtils.createUserMessage("User", "Benchmark test message"); + } + + /** + * Benchmark for basic agent call operation + */ + @Benchmark + public void benchmarkAgentCall(Blackhole blackhole) { + Msg response = agent.call(testMessage).block(); + blackhole.consume(response); + } + + /** + * Benchmark for agent creation + */ + @Benchmark + public ReActAgent benchmarkAgentCreation() { + return ReActAgent.builder() + .name("BenchmarkAgent-" + System.nanoTime()) + .sysPrompt("You are a benchmark agent.") + .model(model) + .toolkit(toolkit) + .memory(new InMemoryMemory()) + .build(); + } + + /** + * Benchmark for agent with memory operations + */ + @Benchmark + public void benchmarkAgentWithMemory(Blackhole blackhole) { + // Add some messages to memory + for (int i = 0; i < 10; i++) { + Msg msg = TestUtils.createUserMessage("User", "Message " + i); + agent.getMemory().addMessage(msg); + } + + Msg response = agent.call(testMessage).block(); + blackhole.consume(response); + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/BaseBenchmark.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/BaseBenchmark.java new file mode 100644 index 000000000..3f4986001 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/BaseBenchmark.java @@ -0,0 +1,68 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.scenarios; + +import io.agentscope.core.ReActAgent; +import io.agentscope.core.agent.test.MockModel; +import io.agentscope.core.agent.test.MockToolkit; +import io.agentscope.core.memory.InMemoryMemory; +import io.agentscope.core.model.Model; +import io.agentscope.core.tool.Toolkit; +import java.util.concurrent.TimeUnit; + +/** + * Base class for AgentScope benchmarks. + * Provides common setup and utilities for benchmark scenarios. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class BaseBenchmark { + + protected ReActAgent agent; + protected Model model; + protected Toolkit toolkit; + protected InMemoryMemory memory; + + @Setup(Level.Trial) + public void setup() { + // Initialize common components for benchmarks + memory = new InMemoryMemory(); + model = new MockModel("Benchmark response"); + toolkit = new MockToolkit(); + + agent = + ReActAgent.builder() + .name("BenchmarkAgent") + .sysPrompt("You are a benchmark agent.") + .model(model) + .toolkit(toolkit) + .memory(memory) + .build(); + } + + @TearDown(Level.Trial) + public void tearDown() { + // Clean up resources if needed + agent = null; + model = null; + toolkit = null; + memory = null; + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/MemoryBenchmark.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/MemoryBenchmark.java new file mode 100644 index 000000000..76edbc869 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/MemoryBenchmark.java @@ -0,0 +1,85 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.scenarios; + +import io.agentscope.core.agent.test.TestUtils; +import io.agentscope.core.memory.InMemoryMemory; +import io.agentscope.core.message.Msg; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmark for Memory performance. + * Tests various aspects of Memory functionality including storage, retrieval, and management. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class MemoryBenchmark { + + private InMemoryMemory memory; + private Msg testMessage; + + @Setup(Level.Trial) + public void setup() { + memory = new InMemoryMemory(); + testMessage = TestUtils.createUserMessage("User", "Benchmark test message"); + } + + /** + * Benchmark for adding messages to memory + */ + @Benchmark + public void benchmarkMemoryAdd(Blackhole blackhole) { + memory.addMessage(testMessage); + blackhole.consume(memory.getMessages()); + } + + /** + * Benchmark for retrieving messages from memory + */ + @Benchmark + public void benchmarkMemoryGet(Blackhole blackhole) { + memory.addMessage(testMessage); + blackhole.consume(memory.getMessages()); + } + + /** + * Benchmark for memory with many messages + */ + @Benchmark + public void benchmarkMemoryWithManyMessages(Blackhole blackhole) { + // Add 100 messages to memory + for (int i = 0; i < 100; i++) { + Msg msg = TestUtils.createUserMessage("User", "Message " + i); + memory.addMessage(msg); + } + blackhole.consume(memory.getMessages()); + } + + /** + * Benchmark for clearing memory + */ + @Benchmark + public InMemoryMemory benchmarkMemoryClear() { + memory.addMessage(testMessage); + memory.clear(); + return memory; + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/ModelBenchmark.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/ModelBenchmark.java new file mode 100644 index 000000000..94cc00ff7 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/ModelBenchmark.java @@ -0,0 +1,71 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.scenarios; + +import io.agentscope.core.agent.test.MockModel; +import io.agentscope.core.agent.test.TestUtils; +import io.agentscope.core.message.Msg; +import io.agentscope.core.model.ChatResponse; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.Blackhole; +import reactor.core.publisher.Flux; + +/** + * Benchmark for Model performance. + * Tests various aspects of Model functionality including streaming and non-streaming calls. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class ModelBenchmark { + + private MockModel mockModel; + private List testMessages; + + @Setup(Level.Trial) + public void setup() { + mockModel = new MockModel("Benchmark model response"); + testMessages = + Collections.singletonList( + TestUtils.createUserMessage("User", "Benchmark test message")); + } + + /** + * Benchmark for model stream operation + */ + @Benchmark + public void benchmarkModelStream(Blackhole blackhole) { + Flux response = mockModel.stream(testMessages, null, null); + response.subscribe(blackhole::consume); + } + + /** + * Benchmark for model non-stream operation (if applicable) + */ + @Benchmark + public void benchmarkModelNonStream(Blackhole blackhole) { + // This would depend on the specific model implementation + // For now, we'll just consume the stream as a non-stream equivalent + Flux response = mockModel.stream(testMessages, null, null); + ChatResponse result = response.blockLast(); + blackhole.consume(result); + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/PipelineBenchmark.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/PipelineBenchmark.java new file mode 100644 index 000000000..2d62b8b41 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/PipelineBenchmark.java @@ -0,0 +1,82 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.scenarios; + +import io.agentscope.core.agent.test.TestUtils; +import io.agentscope.core.message.Msg; +import io.agentscope.core.pipeline.Pipeline; +import io.agentscope.core.pipeline.SequentialPipeline; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; +import java.util.function.Function; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmark for Pipeline performance. + * Tests various aspects of Pipeline functionality including execution and chaining. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class PipelineBenchmark { + + private Pipeline pipeline; + private List testMessages; + + @Setup(Level.Trial) + public void setup() { + // Create a simple pipeline with a few operations + Function, List> op1 = + msgs -> { + // Simulate some processing + return msgs; + }; + + Function, List> op2 = + msgs -> { + // Simulate some processing + return msgs; + }; + + pipeline = new SequentialPipeline(Arrays.asList(op1, op2)); + testMessages = + Arrays.asList( + TestUtils.createUserMessage("User", "Message 1"), + TestUtils.createUserMessage("User", "Message 2")); + } + + /** + * Benchmark for pipeline execution + */ + @Benchmark + public void benchmarkPipelineExecution(Blackhole blackhole) { + List result = pipeline.execute(testMessages); + blackhole.consume(result); + } + + /** + * Benchmark for pipeline creation + */ + @Benchmark + public Pipeline benchmarkPipelineCreation() { + Function, List> op = msgs -> msgs; + return new SequentialPipeline(Arrays.asList(op)); + } +} diff --git a/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/ToolBenchmark.java b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/ToolBenchmark.java new file mode 100644 index 000000000..ee6c2b4a0 --- /dev/null +++ b/agentscope-core/src/test/java/io/agentscope/core/benchmark/scenarios/ToolBenchmark.java @@ -0,0 +1,75 @@ +/* + * Copyright 2024-2025 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.agentscope.core.benchmark.scenarios; + +import io.agentscope.core.agent.test.MockToolkit; +import io.agentscope.core.tool.ToolSchema; +import java.util.concurrent.TimeUnit; +import org.openjdk.jmh.infra.Blackhole; + +/** + * Benchmark for Tool performance. + * Tests various aspects of Tool functionality including registration, lookup, and execution. + */ +@State(Scope.Benchmark) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Warmup(iterations = 2, time = 5, timeUnit = TimeUnit.SECONDS) +@Measurement(iterations = 3, time = 10, timeUnit = TimeUnit.SECONDS) +@Fork(1) +public class ToolBenchmark { + + private MockToolkit toolkit; + private ToolSchema testToolSchema; + + @Setup(Level.Trial) + public void setup() { + toolkit = new MockToolkit(); + // Assuming MockToolkit has some predefined tools + // We'll use the first tool schema for benchmarking + if (!toolkit.listTools().isEmpty()) { + testToolSchema = toolkit.listTools().get(0); + } + } + + /** + * Benchmark for tool lookup operation + */ + @Benchmark + public void benchmarkToolLookup(Blackhole blackhole) { + if (testToolSchema != null) { + blackhole.consume(toolkit.getTool(testToolSchema.getName())); + } + } + + /** + * Benchmark for listing all tools + */ + @Benchmark + public void benchmarkToolListing(Blackhole blackhole) { + blackhole.consume(toolkit.listTools()); + } + + /** + * Benchmark for tool registration + */ + @Benchmark + public MockToolkit benchmarkToolRegistration() { + MockToolkit newToolkit = new MockToolkit(); + // Registration happens in the constructor of MockToolkit + return newToolkit; + } +} diff --git a/agentscope-dependencies-bom/pom.xml b/agentscope-dependencies-bom/pom.xml index 0cc958255..23256d063 100644 --- a/agentscope-dependencies-bom/pom.xml +++ b/agentscope-dependencies-bom/pom.xml @@ -246,7 +246,9 @@ jedis 6.0.0 + + io.projectreactor @@ -272,6 +274,31 @@ 4.12.0 test + + + + org.openjdk.jmh + jmh-core + 1.37 + + + org.openjdk.jmh + jmh-generator-annprocess + 1.37 + provided + + + + + io.micrometer + micrometer-core + 1.13.2 + + + io.micrometer + micrometer-registry-prometheus + 1.13.2 + diff --git a/pom.xml b/pom.xml index fe12f3af3..b96157f47 100644 --- a/pom.xml +++ b/pom.xml @@ -343,5 +343,49 @@ + + + + benchmark + + true + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + **/*Benchmark.java + + + true + + + + + + + + + org.openjdk.jmh + jmh-core + test + + + org.openjdk.jmh + jmh-generator-annprocess + test + + + + + io.micrometer + micrometer-core + test + + + diff --git a/scripts/run-benchmarks.sh b/scripts/run-benchmarks.sh new file mode 100755 index 000000000..ef06bd0ff --- /dev/null +++ b/scripts/run-benchmarks.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Script to run AgentScope performance benchmarks + +# Exit on any error +set -e + +echo "AgentScope Performance Benchmark Runner" +echo "======================================" + +# Check if Maven is installed +if ! command -v mvn &> /dev/null +then + echo "Maven is not installed. Please install Maven to run benchmarks." + exit 1 +fi + +# Default values +PROFILE="benchmark" +QUICK_TEST=false +THOROUGH_TEST=false + +# Parse command line arguments +while [[ $# -gt 0 ]] +do + key="$1" + case $key in + -q|--quick) + QUICK_TEST=true + shift + ;; + -t|--thorough) + THOROUGH_TEST=true + shift + ;; + -p|--profile) + PROFILE="$2" + shift + shift + ;; + -h|--help) + echo "Usage: $0 [OPTIONS]" + echo "Run AgentScope performance benchmarks" + echo "" + echo "Options:" + echo " -q, --quick Run quick benchmarks (reduced iterations)" + echo " -t, --thorough Run thorough benchmarks (increased iterations)" + echo " -p, --profile Maven profile to use (default: benchmark)" + echo " -h, --help Show this help message" + exit 0 + ;; + *) + echo "Unknown option: $1" + echo "Use -h or --help for usage information" + exit 1 + ;; + esac +done + +# Build the project first +echo "Building project..." +mvn clean compile + +# Run benchmarks based on selected options +if [ "$QUICK_TEST" = true ] +then + echo "Running quick benchmarks..." + mvn test -P$PROFILE -Dbenchmark.config=quick +elif [ "$THOROUGH_TEST" = true ] +then + echo "Running thorough benchmarks..." + mvn test -P$PROFILE -Dbenchmark.config=thorough +else + echo "Running standard benchmarks..." + mvn test -P$PROFILE +fi + +echo "" +echo "Benchmark execution completed!" +echo "Reports can be found in target/benchmark-reports/" \ No newline at end of file