Merge branch 'master' into vpellan/otel-threshold-random-value

Author: vpellan

.gitlab/benchmarks.yml

@@ -22,6 +22,17 @@ include:
   when: on_success
   interruptible: false
 
+# Benchmark jobs run on dedicated bare-metal/same-cpu runners; retry the
+# infrastructure-level failures so a flaky runner doesn't fail the pipeline.
+.benchmarks-retry: &benchmarks-retry
+  max: 2
+  when:
+    - unknown_failure
+    - data_integrity_failure
+    - runner_system_failure
+    - scheduler_failure
+    - api_failure
+
 .macrobenchmarks-rules: &macrobenchmarks-rules
   - <<: *on-master-and-parent-pipeline-source-pipeline-on-success-and-not-interruptible
   - <<: *on-master-or-nightly-benchmarks-or-release-on-success-and-not-interruptible
@@ -30,6 +41,7 @@ include:
     interruptible: true
 
 linux-php-laravel-realworld-parallel:
+  retry: *benchmarks-retry
   needs:
     - job: "package extension (bundles): [amd64, x86_64-unknown-linux-gnu]"
       artifacts: true
@@ -40,6 +52,7 @@ linux-php-laravel-realworld-parallel:
     DD_TRACER_ARTIFACT_DIR: "$CI_PROJECT_DIR/packages"
 
 linux-php-symfony-realworld-parallel:
+  retry: *benchmarks-retry
   needs:
     - job: "package extension (bundles): [amd64, x86_64-unknown-linux-gnu]"
       artifacts: true
@@ -50,6 +63,7 @@ linux-php-symfony-realworld-parallel:
     DD_TRACER_ARTIFACT_DIR: "$CI_PROJECT_DIR/packages"
 
 linux-php-wordpress-parallel:
+  retry: *benchmarks-retry
   needs:
     - job: "package extension (bundles): [amd64, x86_64-unknown-linux-gnu]"
       artifacts: true
@@ -149,6 +163,7 @@ workflow:
 
 .microbenchmarks:
   stage: benchmarks
+  retry: *benchmarks-retry
   tags: ["runner:apm-k8s-tweaked-metal"]
   needs: []
   image:
@@ -248,6 +263,7 @@ check-big-regressions:
 
 .macrobenchmarks:
   stage: benchmarks
+  retry: *benchmarks-retry
   needs:
     - job: "package extension (bundles): [amd64, x86_64-unknown-linux-gnu]"
       artifacts: true

.gitlab/build-loader.sh

@@ -22,4 +22,13 @@ phpize
 ./configure
 make clean
 make -j "${MAKE_JOBS}" all ECHO_ARG="-e" CFLAGS="-std=gnu11 -O2 -g -Wall -Wextra -Werror -DPHP_DD_LIBRARY_LOADER_VERSION='\"$(cat ../VERSION)\"'"
+
+# The reaper is copied out of the loader. Reject accidental references to its
+# code/data, including compiler instrumentation, before packaging the library.
+REAPER_RELOCATIONS="$(LC_ALL=C objdump -r -j ddloader_reaper_code .libs/telemetry_reaper.o)"
+if [[ "${REAPER_RELOCATIONS}" == *"RELOCATION RECORDS"* ]]; then
+  printf 'Telemetry reaper must not contain relocations:\n%s\n' "${REAPER_RELOCATIONS}"
+  exit 1
+fi
+
 cp modules/dd_library_loader.so "../dd_library_loader-$(uname -m)-${HOST_OS}.so"

.gitlab/build-tracing-asan.sh

@@ -7,7 +7,7 @@ mkdir -p extensions_$(uname -m)
 
 # Build extension basic .so
 switch-php "debug-zts-asan"
-make RUST_DEBUG_BUILD=1
+.gitlab/run-with-retryable-download.sh make RUST_DEBUG_BUILD=1
 cp -v "tmp/build_extension/modules/ddtrace.so" "extensions_$(uname -m)/ddtrace-${ABI_NO}-debug-zts.so"
 
 # Compress debug info

.gitlab/compile_extension.sh

@@ -20,9 +20,30 @@ if [ "${WITH_ASAN}" -eq "1" ]; then
   export COMPILE_ASAN=1
 fi
 # Compile Rust and PHP in parallel
-SHARED=1 ./compile_rust.sh &
+rust_build_log=$(mktemp)
+trap 'rm -f "$rust_build_log"' EXIT
+SHARED=1 ./compile_rust.sh 2>&1 | tee "$rust_build_log" &
+rust_build_pid=$!
 make -j static &
-wait
+c_build_pid=$!
+
+# A bare wait discards the exit statuses of both background builds.
+rust_build_status=0
+c_build_status=0
+wait "$rust_build_pid" || rust_build_status=$?
+wait "$c_build_pid" || c_build_status=$?
+if [ "$c_build_status" -ne 0 ]; then
+  exit "$c_build_status"
+fi
+if [ "$rust_build_status" -ne 0 ]; then
+  # Retry the job with a clean target directory: libddwaf may have left a
+  # partially extracted archive, which cannot safely be reused locally.
+  if grep -Eq 'Failed to (download archive|write archive entry contents to file):.*reqwest::Error.*(ConnectionReset|ConnectionAborted|TimedOut|IncompleteMessage|UnexpectedEof)' "$rust_build_log"; then
+    echo "Transient libddwaf download failure; exiting 75 for GitLab retry." >&2
+    exit 75
+  fi
+  exit "$rust_build_status"
+fi
 
 # Link extension
 cc -shared -Wl,-whole-archive "${MODULES_DIR}/ddtrace.a" \

.gitlab/enable-windows-test-dumps.ps1

@@ -0,0 +1,25 @@
+$ErrorActionPreference = 'Stop'
+
+$root = Split-Path $PSScriptRoot -Parent
+$dumpFolder = Join-Path $root 'dumps'
+New-Item -ItemType Directory -Path $dumpFolder -Force | Out-Null
+
+$wer = 'HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting'
+$localDumps = Join-Path $wer 'LocalDumps\php.exe'
+New-Item -Path $localDumps -Force | Out-Null
+New-ItemProperty -Path $localDumps -Name DumpFolder -Value $dumpFolder -PropertyType ExpandString -Force | Out-Null
+New-ItemProperty -Path $localDumps -Name DumpType -Value 2 -PropertyType DWord -Force | Out-Null
+New-ItemProperty -Path $localDumps -Name DumpCount -Value 10 -PropertyType DWord -Force | Out-Null
+New-ItemProperty -Path $wer -Name DontShowUI -Value 1 -PropertyType DWord -Force | Out-Null
+
+# proc_open's suppress_errors sets SEM_NOGPFAULTERRORBOX in the child, which
+# disables WER. Keep WER enabled for every test's FILE process.
+$runner = Join-Path $root 'run-tests.php'
+$source = [IO.File]::ReadAllText($runner)
+$pattern = "'suppress_errors'\s*=>\s*true"
+if ([regex]::Matches($source, $pattern).Count -ne 1) {
+    throw "Expected one proc_open suppress_errors option in $runner"
+}
+$replacement = "'suppress_errors' => false"
+$source = [regex]::Replace($source, $pattern, [System.Text.RegularExpressions.MatchEvaluator]{ param($match) $replacement })
+[IO.File]::WriteAllText($runner, $source, (New-Object System.Text.UTF8Encoding($false)))

.gitlab/generate-appsec.php

@@ -96,7 +96,7 @@
   image: registry.ddbuild.io/agent-delivery/dd-pkg:v0.9.3
   tags: [ "arch:arm64" ]
   variables:
-    IMG_REGISTRIES: "dockerhub"
+    IMG_REGISTRIES: "dev"
     IMG_SIGNING: "false"
     PUBLIC_IMAGES_PUBLISH_TIMEOUT: "1800"
   script:
@@ -182,7 +182,8 @@
 <?php echo $ecrLoginSnippet, "\n"; ?>
 <?php dockerhub_login() ?>
   script:
-    - apt update && apt install -y openjdk-17-jre
+    - .gitlab/run-with-retryable-download.sh apt update
+    - .gitlab/run-with-retryable-download.sh apt install -y openjdk-17-jre
     - find "$CI_PROJECT_DIR"/appsec/tests/integration/build || true
     - |
       cd appsec/tests/integration
@@ -467,6 +468,7 @@
       - ARCH: ["amd64", "arm64"]
   rules:
     - when: manual
+      allow_failure: true
   needs: []
   artifacts:
     when: on_failure

.gitlab/generate-common.php

@@ -227,7 +227,7 @@ function windows_git_setup_with_packages() {
       KUBERNETES_SERVICE_MEMORY_LIMIT: 512Mi
 
   request-replayer:
-    name: registry.ddbuild.io/ci/dd-trace-php/request-replayer:3.0
+    name: registry.ddbuild.io/ci/dd-trace-php/request-replayer:4.0
     alias: request-replayer
     command: ["php", "-S", "<?= $service_bind_address ?>:80", "index.php"]
     variables:

.gitlab/generate-package.php

@@ -968,9 +968,16 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
       artifacts: true
     - job: "package extension (bundles): [arm64, aarch64-unknown-linux-gnu]"
       artifacts: true
+    - job: "package extension: [amd64, x86_64-alpine-linux-musl]"
+      artifacts: true
+    - job: "package extension: [arm64, aarch64-alpine-linux-musl]"
+      artifacts: true
     - job: datadog-setup.php
       artifacts: true
   variables:
+    INSTALLER_ARTIFACT_SERVER: installer-artifacts
+    INSTALLER_DOCKER_NETWORK: installer-tests
+    INSTALLER_REPOSITORY_URL: http://installer-artifacts
     KUBERNETES_CPU_REQUEST: 2
     KUBERNETES_MEMORY_REQUEST: 2Gi
     KUBERNETES_MEMORY_LIMIT: 4Gi
@@ -981,8 +988,50 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
     - apt install -y make
     - mkdir build
     - mv packages build
+    - |
+      set -- build/packages/dd-library-php-*-x86_64-linux-gnu.tar.gz
+      if [ "$#" -ne 1 ] || [ ! -f "$1" ]; then
+        echo "Expected exactly one full x86_64 GNU bundle, found: $*"
+        exit 1
+      fi
+      version=${1#build/packages/dd-library-php-}
+      version=${version%-x86_64-linux-gnu.tar.gz}
+      printf '%s\n' "$version" > VERSION
+      for platform in \
+        x86_64-linux-gnu \
+        aarch64-linux-gnu \
+        x86_64-linux-musl \
+        aarch64-linux-musl; do
+        bundle="build/packages/dd-library-php-${version}-${platform}.tar.gz"
+        if [ ! -f "$bundle" ]; then
+          echo "Missing installer bundle: $bundle"
+          exit 1
+        fi
+      done
+    - docker network create "$INSTALLER_DOCKER_NETWORK"
+    - |
+      docker run --detach --rm \
+        --name "$INSTALLER_ARTIFACT_SERVER" \
+        --network "$INSTALLER_DOCKER_NETWORK" \
+        --volume "$CI_PROJECT_DIR/build/packages:/usr/share/nginx/html/releases/download/$(<VERSION):ro" \
+        nginx:1.27-alpine
+    - |
+      for attempt in $(seq 1 10); do
+        if docker exec "$INSTALLER_ARTIFACT_SERVER" wget --quiet --spider \
+          "http://127.0.0.1/releases/download/$(<VERSION)/datadog-setup.php"; then
+          break
+        fi
+        if [ "$attempt" -eq 10 ]; then
+          docker logs "$INSTALLER_ARTIFACT_SERVER"
+          exit 1
+        fi
+        sleep 1
+      done
   script:
     - make -C dockerfiles/verify_packages test_installer
+  after_script:
+    - docker rm --force "$INSTALLER_ARTIFACT_SERVER" || true
+    - docker network rm "$INSTALLER_DOCKER_NETWORK" || true
 
 "test early PHP 8.1":
   stage: verify
@@ -1319,6 +1368,8 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
     DDAGENT_HOSTNAME: 127.0.0.1
     DD_AGENT_HOST: 127.0.0.1
     DATADOG_HAVE_DEV_ENV: 1
+    _DD_DEBUG_SIDECAR_LOG_LEVEL: trace
+    _DD_DEBUG_SIDECAR_LOG_METHOD: "file://${CI_PROJECT_DIR}/artifacts/sidecar.log"
   needs:
     - job: "package extension (installers): [amd64, x86_64-unknown-linux-gnu]"
       artifacts: true
@@ -1327,6 +1378,7 @@ function package_extension_jobs(array $platform, string $kind = "all"): array
     - !reference [.services, httpbin-integration]
   before_script:
 <?php dockerhub_login() ?>
+    - mkdir -p artifacts
     - switch-php debug
   script:
     - sudo dpkg -i packages/*amd64*.deb

.gitlab/generate-profiler.php

@@ -161,7 +161,7 @@
     - php -v
     # Fail loudly if the profiler did not load: otherwise the language tests
     # would run profiler-less and pass, giving a false green.
-    - php -m | grep -qx 'datadog-profiling' || { echo 'ERROR datadog-profiling extension is not loaded'; exit 1; }
+    - php -r 'exit((int) !extension_loaded("datadog-profiling"));' || { echo 'ERROR datadog-profiling extension is not loaded'; exit 1; }
     - cat "${XFAIL_LIST}" profiling/tests/php-language-xfail.list > /tmp/profiler-php-language-xfail.list
     - "if php -r 'exit(PHP_VERSION_ID < 80400 ? 0 : 1);'; then cat profiling/tests/php-language-xfail-pre84.list >> /tmp/profiler-php-language-xfail.list; fi"
     - export XFAIL_LIST=/tmp/profiler-php-language-xfail.list

.gitlab/generate-shared.php

@@ -183,7 +183,7 @@
     - switch-php "<?= $switch_php_version ?>"
     - mkdir -p tmp/build_zai && cd tmp/build_zai
     - CMAKE_PREFIX_PATH=/opt/catch2 Tea_ROOT=../../tmp/tea/<?= $switch_php_version ?> cmake <?= $toolchain ?> -DCMAKE_BUILD_TYPE=Debug -DBUILD_ZAI_TESTING=ON -DPhpConfig_ROOT=$(php-config --prefix) ../../zend_abstract_interface
-    - make -j all
+    - ../../.gitlab/run-with-retryable-download.sh make -j all
     - mkdir -p "${CI_PROJECT_DIR}/artifacts"
     - make test ARGS="--output-junit ${CI_PROJECT_DIR}/artifacts/zai-<?= $major_minor ?>-<?= $switch_php_version ?>-results.xml --output-on-failure"
     - grep -e "=== Total [0-9]+ memory leaks detected ===" Testing/Temporary/LastTest.log && exit 1 || true
@@ -213,7 +213,7 @@
     - echo "extension=curl.so" | sudo tee $(php -i | awk -F"=> " '/Scan this dir for additional .ini files/ {print $2}')/curl.ini
     - mkdir -p tmp/build_zai && cd tmp/build_zai
     - CMAKE_PREFIX_PATH=/opt/catch2 Tea_ROOT=../../tmp/tea/nts cmake -DCMAKE_BUILD_TYPE=Debug -DBUILD_ZAI_TESTING=ON -DRUN_SHARED_EXTS_TESTS=1 -DPhpConfig_ROOT=$(php-config --prefix) ../../zend_abstract_interface
-    - make -j all
+    - ../../.gitlab/run-with-retryable-download.sh make -j all
     - TEA_INI_IGNORE=0 make test
     - grep -e "=== Total [0-9]+ memory leaks detected ===" Testing/Temporary/LastTest.log && exit 1 || true
 <?php
@@ -240,7 +240,7 @@
       artifacts: true
   script:
     - switch-php "<?= $switch_php_version ?>"
-    - make install # build ddtrace.so
+    - .gitlab/run-with-retryable-download.sh make install # build ddtrace.so
     - mkdir -p tmp/build_ext-tea && cd tmp/build_ext-tea
     - CMAKE_PREFIX_PATH=/opt/catch2 Tea_ROOT=../../tmp/tea/<?= $switch_php_version ?> cmake <?= $toolchain ?> -DCMAKE_BUILD_TYPE=Debug -S ../../tests/tea
     - cmake --build . --parallel