#!/bin/bash
# Copyright 2026 The JAX Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
# Run Bazel GPU tests without RBE. This runs two commands: single accelerator
# tests with one GPU a piece, multiaccelerator tests with all GPUS.
# If $JAXCI_BUILD_JAXLIB=false, the job requires that jaxlib, jax-oneapi-plugin,
# and jax-oneapi-pjrt wheels are stored inside the ../dist folder
#
# -e: abort script if one command fails
# -u: error if undefined variable used
# -x: log all commands
# -o history: record shell history
# -o allexport: export all functions and variables to be available to subscripts
set -exu -o history -o allexport

# Source default JAXCI environment variables.
source ci/envs/default.env

# Set up the build environment.
source "ci/utilities/setup_build_environment.sh"

# Run Bazel GPU tests (single accelerator and multiaccelerator tests) directly
# on the VM without RBE.
echo "Running single accelerator tests (without RBE)..."

# ==============================================================================
# Detect OneAPI GPU count
# ==============================================================================
detect_oneapi_gpus() {
  if command -v xpu-smi &> /dev/null; then
    local count=$(xpu-smi discovery 2>/dev/null | grep -c "Device Name")
    echo $count
  else
    echo 0
  fi
}

# ==============================================================================
# Set up test environment variables
# ==============================================================================
export gpu_count=$(detect_oneapi_gpus)
if [[ $gpu_count -eq 0 ]]; then
  echo "ERROR: No OneAPI GPUs detected, exiting"
  exit 1
fi

export max_tests_per_gpu=8
if command -v xpu-smi &> /dev/null; then
  gpu_memory_gb=$(xpu-smi discovery --dump 16 | grep -v "Memory\|^$" | awk '{print int($1/1024+0.5)}' | head -1)
  gpu_memory_gb="${gpu_memory_gb:-16}"
  export max_tests_per_gpu=$((gpu_memory_gb / 2))
  [[ $max_tests_per_gpu -lt 4 ]] && export max_tests_per_gpu=4
  [[ $max_tests_per_gpu -gt 16 ]] && export max_tests_per_gpu=16
fi

export num_test_jobs=$((gpu_count * max_tests_per_gpu))
export num_cpu_cores=$(nproc)
export total_ram_gib=$(awk '/MemTotal/ {printf "%.0f", $2/1048576}' /proc/meminfo)
export host_memory_limit=$((total_ram_gib / 6))

if [[ $num_cpu_cores -lt $num_test_jobs ]]; then
  num_test_jobs=$num_cpu_cores
fi

if [[ $host_memory_limit -lt $num_test_jobs ]]; then
  num_test_jobs=$host_memory_limit
fi

# Log detected configuration
echo "=== OneAPI Configuration ==="
echo "GPU Count: $gpu_count"
echo "Max Tests Per GPU: $max_tests_per_gpu"
echo "Num Test Jobs: $num_test_jobs"
echo "Num CPU Cores: $num_cpu_cores"
echo "Total RAM: $total_ram_gib GiB"
echo "=========================="
# End of test environment variables setup.

if [[ "$JAXCI_HERMETIC_PYTHON_VERSION" == *t || "$JAXCI_HERMETIC_PYTHON_VERSION" == *-ft || "$JAXCI_HERMETIC_PYTHON_VERSION" == *-nogil ]]; then
  JAXCI_HERMETIC_PYTHON_VERSION=${JAXCI_HERMETIC_PYTHON_VERSION%t}
  JAXCI_HERMETIC_PYTHON_VERSION=${JAXCI_HERMETIC_PYTHON_VERSION%-ft}
  JAXCI_HERMETIC_PYTHON_VERSION=${JAXCI_HERMETIC_PYTHON_VERSION%-nogil}
  FREETHREADED_FLAG_VALUE="yes"
else
  FREETHREADED_FLAG_VALUE="no"
fi

OVERRIDE_XLA_REPO=""
if [[ -n "${JAXCI_XLA_GIT_DIR}" ]]; then
  OVERRIDE_XLA_REPO="--override_repository=xla=${JAXCI_XLA_GIT_DIR}"
fi

# Get oneapi version
oneapi_version="2025.1"
TEST_CONFIG="oneapi"

# Don't abort the script if one command fails to ensure we run both test
# commands below.
set +e


# Runs single accelerator tests with one GPU apiece.
# It appears --run_under needs an absolute path.
# The product of the `JAX_ACCELERATOR_COUNT`` and `JAX_TESTS_PER_ACCELERATOR`
# should match the VM's CPU core count (set in `--local_test_jobs`).
bazel test --config=$TEST_CONFIG \
       --repo_env=HERMETIC_PYTHON_VERSION="$JAXCI_HERMETIC_PYTHON_VERSION" \
       --@rules_python//python/config_settings:py_freethreaded="$FREETHREADED_FLAG_VALUE" \
       $OVERRIDE_XLA_REPO \
       --//jax:build_jaxlib=$JAXCI_BUILD_JAXLIB \
       --//jax:build_jax=$JAXCI_BUILD_JAX \
       --test_env=XLA_PYTHON_CLIENT_ALLOCATOR=platform \
       --run_under "$(pwd)/build/parallel_accelerator_execute.sh" \
       --test_output=errors \
       --test_env=JAX_ACCELERATOR_COUNT=$gpu_count \
       --test_env=JAX_TESTS_PER_ACCELERATOR=$max_tests_per_gpu \
       --test_env=JAX_LOCK_DIR=/tmp/jax_accelerator_locks \
       --test_env=XLA_FLAGS \
       --local_test_jobs=$num_test_jobs \
       --test_env=JAX_EXCLUDE_TEST_TARGETS=PmapTest.testSizeOverflow \
       --test_tag_filters=-multiaccelerator,-multiaccelerator-only \
       --test_env=TF_CPP_MIN_LOG_LEVEL=0 \
       --test_env=JAX_SKIP_SLOW_TESTS=true \
       --action_env=JAX_ENABLE_X64="$JAXCI_ENABLE_X64" \
       --color=yes \
       //tests:gpu_tests //tests:backend_independent_tests
       # TODO(Intel-tf): uncomment this when pallas GPU tests are working
       # //tests/pallas:gpu_tests //tests/pallas:backend_independent_tests

# Store the return value of the first bazel command.
first_bazel_cmd_retval=$?

echo "Running multi-accelerator tests (without RBE)..."
# Runs multiaccelerator tests with all GPUs directly on the VM without RBE..
bazel test --config=$TEST_CONFIG \
      --repo_env=HERMETIC_PYTHON_VERSION="$JAXCI_HERMETIC_PYTHON_VERSION" \
      --@rules_python//python/config_settings:py_freethreaded="$FREETHREADED_FLAG_VALUE" \
      $OVERRIDE_XLA_REPO \
      --//jax:build_jaxlib=$JAXCI_BUILD_JAXLIB \
      --//jax:build_jax=$JAXCI_BUILD_JAX \
      --test_env=XLA_PYTHON_CLIENT_ALLOCATOR=platform \
      --test_env=XLA_FLAGS \
      --test_output=errors \
      --local_test_jobs=1 \
      --test_tag_filters=multiaccelerator,multiaccelerator-only \
      --test_env=TF_CPP_MIN_LOG_LEVEL=0 \
      --test_env=JAX_SKIP_SLOW_TESTS=true \
      --action_env=JAX_ENABLE_X64="$JAXCI_ENABLE_X64" \
      --color=yes \
      //tests:gpu_tests //tests/multiprocess:gpu_tests
      # TODO(Intel-tf): uncomment this when pallas GPU tests are working
      # //tests/pallas:gpu_tests

# Store the return value of the second bazel command.
second_bazel_cmd_retval=$?

ci/utilities/collect_bazel_test_xmls.sh test-artifacts

# Exit with failure if either command fails.
if [[ $first_bazel_cmd_retval -ne 0 ]]; then
  exit $first_bazel_cmd_retval
elif [[ $second_bazel_cmd_retval -ne 0 ]]; then
  exit $second_bazel_cmd_retval
else
  exit 0
fi