diff --git a/.github/benchmark-files/generate_summary.php b/.github/benchmark-files/generate_summary.php index 289069a..4d0f0e5 100644 --- a/.github/benchmark-files/generate_summary.php +++ b/.github/benchmark-files/generate_summary.php @@ -84,7 +84,7 @@ sort($phpVersions); // Sort xdebug modes according to the defined order, leaving only those which actually exist in the data -$xdebugModeOrder = ["no", "off", "coverage", "debug", "develop", "gcstats", "profile", "trace"]; +$xdebugModeOrder = ["no", "off", "debug"]; $xdebugModes = array_values(array_filter($xdebugModeOrder, function($mode) use ($xdebugModes) { return in_array($mode, $xdebugModes); })); @@ -282,9 +282,16 @@ function githubApiRequest(string $url): string|false { */ function fetchPreviousBenchmarkResults(): array { // Determine the base branch for comparison - // For pull requests, use GITHUB_BASE_REF + // If COMPARISON_BRANCH is set, use it explicitly + // Otherwise for pull requests, use GITHUB_BASE_REF // For other actions, use GITHUB_REF_NAME (current branch) - $baseBranch = getenv('GITHUB_BASE_REF') ?: getenv('GITHUB_REF_NAME'); + $comparisonBranch = getenv('COMPARISON_BRANCH'); + if ($comparisonBranch !== false && $comparisonBranch !== '') { + $baseBranch = $comparisonBranch; + } else { + $baseBranch = getenv('GITHUB_BASE_REF') ?: getenv('GITHUB_REF_NAME'); + } + if (!$baseBranch) { fwrite(STDERR, "Warning: Could not determine base branch for comparison\n"); return []; diff --git a/.github/benchmark-files/xdebug-benchmark.ini b/.github/benchmark-files/xdebug-benchmark.ini index b6a0b41..ecb5eda 100644 --- a/.github/benchmark-files/xdebug-benchmark.ini +++ b/.github/benchmark-files/xdebug-benchmark.ini @@ -1,4 +1,4 @@ -zend_extension=xdebug +zend_extension=php_debugger.so xdebug.start_with_request=no ; We need this because some of the functions in bench.php do deep recursion and the default nesting level is not enough xdebug.max_nesting_level=2048 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml new file mode 100644 index 0000000..5fb8850 --- /dev/null +++ b/.github/workflows/benchmark.yml @@ -0,0 +1,204 @@ +name: Benchmark Xdebug performance + +on: + # This workflow runs on three triggers: Weekly on schedule, manually, or when pull requests are labeled + schedule: + # Every Sunday at 03:00 UTC + - cron: '0 3 * * 0' + workflow_dispatch: + pull_request: + types: [ labeled ] + +jobs: + run-benchmark: + # When triggered when a pull request is labeled we check the name of the label + if: | + github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || + ( + github.event_name == 'pull_request' && + github.event.label.name == 'performance-check' + ) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + command: ["bench", "symfony", "rector"] +# php: ["8.2", "8.3", "8.4", "8.5"] + php: ["8.2"] + xdebug_mode: ["no", "off", "debug"] + steps: + - uses: actions/checkout@v6 + + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: "${{ matrix.php }}" + coverage: none + # We need to use a specific version of Composer because we are using a specific version of the + # Symfony Demo which is not compatible with the latest versions of Composer + tools: composer:v2.2.21 + + - name: Disable ASLR + # ASLR can cause a lot of noise due to missed sse opportunities for memcpy + # and other operations, so we disable it during benchmarking. + run: echo 0 | sudo tee /proc/sys/kernel/randomize_va_space + + - name: Install valgrind + run: sudo apt-get update && sudo apt-get install -y valgrind + + - name: Compile + run: | + phpize + ./configure --enable-php-debugger + make -j$(nproc) + sudo make install + + - name: create results and opcache folder + run: | + mkdir -p results + mkdir -p /tmp/opcache + + - name: install Symfony demo + if: matrix.command == 'symfony' + # For versions of PHP greater than 8.4 we need to run a different version of the Symfony Demo. + # The older versions suffer from the nulllable parameter deprecation and this generated a lot of + # deprecations in the code and this skewed the tests results a lot. And we cannot use the newer version + # for older versions of PHP as it is not compatible. + run: | + if [ "${{ matrix.php }}" \< "8.4" ]; then + version="v2.0.2" + else + version="v2.7.0" + fi + git clone -q --branch "$version" --depth 1 https://github.com/symfony/demo.git ./symfony-demo + cd symfony-demo + composer install + # We do not want our tests to be skewed by any error, warning, notice or deprecation thrown by the + # code, so we set the error level to 0 on purpose. Unfortunately, Symfony overrides this setup and + # sets their own error reporting level. We fix this by changing all the places where Symfony is + # setting the error reporting level so that it continues to be set to 0 + find . -type f -name "*.php" -exec sed -i.bak -E 's/error_reporting\(.*\);/error_reporting(0);/g' {} + + + - name: install Rector + if: matrix.command == 'rector' + run: | + composer require rector/rector:2.1.2 phpstan/phpstan:2.1.33 --dev + cp .github/benchmark-files/rector.php . + + - name: Copy ini file + run: | + sudo cp ./.github/benchmark-files/benchmark.ini /etc/php/${{ matrix.php }}/cli/conf.d + sudo cp ./.github/benchmark-files/benchmark.ini /etc/php/${{ matrix.php }}/cgi/conf.d + + - name: Copy xdebug ini file + if: matrix.xdebug_mode != 'no' + run: | + sudo cp ./.github/benchmark-files/xdebug-benchmark.ini /etc/php/${{ matrix.php }}/cli/conf.d + sudo cp ./.github/benchmark-files/xdebug-benchmark.ini /etc/php/${{ matrix.php }}/cgi/conf.d + + - name: Set xdebug mode + run: echo "XDEBUG_MODE=${{ matrix.xdebug_mode }}" >> $GITHUB_ENV + + - name: Confirm PHP setup + run: | + php -v + php -i + + - name: benchmark bench.php + if: matrix.command == 'bench' + run: valgrind --tool=callgrind --dump-instr=yes --callgrind-out-file=callgrind.out -- php ./.github/benchmark-files/bench.php + + - name: benchmark Symfony + if: matrix.command == 'symfony' + run: | + cd symfony-demo + # The Symfony demo is run using php-cgi to simulate a web server. Symfony needs that this env + # variable is set, it would usually be set by the web server, we need to set it manually instead + export SCRIPT_FILENAME="$(realpath public/index.php)" + # Symfony also needs this to be set to a non empty value + export APP_SECRET=APP_SECRET + # We run the web page twice so that the files have already been compiled into the opcache in order + # to remove the compilation time from the equation. This also better reflects the normal situation + # in a web server + php-cgi public/index.php + valgrind --tool=callgrind --dump-instr=yes --callgrind-out-file=callgrind.out -- php-cgi public/index.php + mv callgrind.out .. + cd .. + + - name: benchmark rector.php + if: matrix.command == 'rector' + # Rector needs to be run with the --xdebug flag so that it does not try to disable Xdebug and also with + # the --debug flag so that it does not try to run several threads in parallel + run: valgrind --tool=callgrind --dump-instr=yes --callgrind-out-file=callgrind.out -- php vendor/bin/rector --dry-run --xdebug --debug || true + + - name: save matrix values + run: | + # We read the number of executed instructions from the callgrind.out file and save it in an artifact + # to be processed later + awk '/^summary:/ { print $2 }' callgrind.out > results/php-${{ matrix.php }}_cmd-${{ matrix.command }}_xdebug-${{ matrix.xdebug_mode }}.txt + # We also save the matrix variables so that in a later step we can build a list of the executed + # options + echo "${{ matrix.php }},${{ matrix.command }},${{ matrix.xdebug_mode }}" > results/matrix-values-${{ matrix.php }}-${{ matrix.command }}-${{ matrix.xdebug_mode }}.txt + + - name: Upload result and matrix info + uses: actions/upload-artifact@v6 + with: + name: results-${{ matrix.command }}-${{ matrix.php }}-${{ matrix.xdebug_mode }} + path: results/ + + performance: + # After running all benchmarks for all commands, PHP versions and Xdebug modes, we run a new job that builds a + # summary of all execution times + needs: run-benchmark + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + comparison: + - name: "base-branch" + branch: "" # Empty means use default base branch logic + - name: "base" + branch: "base" + steps: + - name: Setup PHP + uses: shivammathur/setup-php@v2 + with: + php-version: "8.5" + coverage: none + + - uses: actions/checkout@v6 + + - name: Download all artifacts + uses: actions/download-artifact@v7 + with: + path: results + + - name: Generate summary table as markdown + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + COMPARISON_BRANCH: ${{ matrix.comparison.branch }} + run: php .github/benchmark-files/generate_summary.php + + - name: print summary + # This generated summary file is copied into a special file which GitHub has defined + # in this special env variable and it will use it to print a summary for the workflow + run: cat summary.md >> $GITHUB_STEP_SUMMARY + + - name: Upload summary.csv as artifact + if: matrix.comparison.name == 'base-branch' + uses: actions/upload-artifact@v6 + with: + name: summary.csv + path: summary.csv + + cleanup: + # Clean up intermediary artifacts after both performance jobs complete + needs: performance + runs-on: ubuntu-latest + steps: + - name: delete intermediary artifacts + # The intermediate files that we generated are not needed and can be deleted, + # helping us keep the summary page clean + uses: geekyeggo/delete-artifact@v6 + with: + name: results-*