ci: simplify the python env to idf venv only

2022-07-28 15:32:13 +08:00
parent e7bcecb8b5
commit f10e6145dc
12 changed files with 109 additions and 205 deletions
@@ -57,20 +57,21 @@ variables:
  PYTHON_VER: 3.7.10

  CLANG_TIDY_RUNNER_PROJ: 2107  # idf/clang-tidy-runner
-  IDF_BUILD_APPS_PROJ: 2818  # fuhanxi/idf-build-apps
+  IDF_BUILD_APPS_PROJ: 2818  # espressif/idf-build-apps

  # Docker images
  BOT_DOCKER_IMAGE_TAG: ":latest"

-  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.0:2-3"
-  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.0:3"
-  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.0:2-1"
-  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.0:2-1"
-  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.0:2"
-  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.0:2-20220802"
+  ESP_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-env-v5.1:1"
+  AFL_FUZZER_TEST_IMAGE: "$CI_DOCKER_REGISTRY/afl-fuzzer-test-v5.1:1-1"
+  CLANG_STATIC_ANALYSIS_IMAGE: "${CI_DOCKER_REGISTRY}/clang-static-analysis-v5.1:1-1"
+  ESP_IDF_DOC_ENV_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-doc-env-v5.1:1-1"
+  QEMU_IMAGE: "${CI_DOCKER_REGISTRY}/qemu-v5.1:1-20220802"
+  TARGET_TEST_ENV_IMAGE: "$CI_DOCKER_REGISTRY/target-test-env-v5.1:1"

  SONARQUBE_SCANNER_IMAGE: "${CI_DOCKER_REGISTRY}/sonarqube-scanner:3"
-  LINUX_SHELL_IMAGE: "${CI_DOCKER_REGISTRY}/linux-shells-v5.0:2"
+
+  PRE_COMMIT_IMAGE: "$CI_DOCKER_REGISTRY/esp-idf-pre-commit:1"

  # target test config file, used by assign test job
  CI_TARGET_TEST_CONFIG_FILE: "$CI_PROJECT_DIR/.gitlab/ci/target-test.yml"
@@ -110,130 +111,101 @@ cache:
      - .cache/submodule_archives
    policy: pull

-.setup_tools_unless_target_test: &setup_tools_unless_target_test |
+.common_before_scripts: &common-before_scripts |
+  source tools/ci/utils.sh
+  is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
+
  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
-  export IDF_MIRROR_PREFIX_MAP=
-  fi
-  if [[ "$SETUP_TOOLS" == "1" || "$CI_JOB_STAGE" != "target_test" ]]; then
-  tools/idf_tools.py --non-interactive install ${SETUP_TOOLS_LIST:-} && eval "$(tools/idf_tools.py --non-interactive export)" || exit 1
-    if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
-    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
-    wget $OOCD_DISTRO_URL
-    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
-    tar -x -f $ARCH_NAME
-    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
-    export PATH=$PWD/openocd-esp32/bin:$PATH
-    fi
+    export IDF_MIRROR_PREFIX_MAP=
  fi

-.download_test_python_contraint_file: &download_test_python_contraint_file |
+  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
+    export INCLUDE_NIGHTLY_RUN="1"
+  fi
+
+  # configure cmake related flags
+  source tools/ci/configure_ci_environment.sh
+
+  # add extra python packages
+  export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+
+.setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv |
+  # must use after setup_tools_except_target_test
+  # otherwise the export.sh won't work properly
+
+  # download constraint file for dev
  if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then
    wget -O /tmp/constraint.txt --header="Authorization:Bearer ${ESPCI_TOKEN}" ${GITLAB_HTTP_SERVER}/api/v4/projects/2581/repository/files/${CI_PYTHON_CONSTRAINT_FILE}/raw?ref=${CI_PYTHON_CONSTRAINT_BRANCH}
    mkdir -p ~/.espressif
    mv /tmp/constraint.txt ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
  fi

-.download_test_python_tool: &download_test_python_tool |
+  # Mirror
+  if [[ -n "$IDF_DONT_USE_MIRRORS" ]]; then
+    export IDF_MIRROR_PREFIX_MAP=
+  fi
+
+  # install latest python packages
+  # target test jobs
+  if [[ "${CI_JOB_STAGE}" == "target_test" ]]; then
+    # ttfw jobs
+    if ! echo "${CI_JOB_NAME}" | egrep ".+_pytest_.+"; then
+      run_cmd bash install.sh --enable-ci --enable-ttfw
+    else
+      run_cmd bash install.sh --enable-ci --enable-pytest
+    fi
+  elif [[ "${CI_JOB_STAGE}" == "build_doc" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-docs
+  elif [[ "${CI_JOB_STAGE}" == "build" ]]; then
+    run_cmd bash install.sh --enable-ci --enable-pytest
+  else
+    run_cmd bash install.sh --enable-ci
+  fi
+
+  source ./export.sh
+
+  # Custom OpenOCD
+  if [[ ! -z "$OOCD_DISTRO_URL" && "$CI_JOB_STAGE" == "target_test" ]]; then
+    echo "Using custom OpenOCD from ${OOCD_DISTRO_URL}"
+    wget $OOCD_DISTRO_URL
+    ARCH_NAME=$(basename $OOCD_DISTRO_URL)
+    tar -x -f $ARCH_NAME
+    export OPENOCD_SCRIPTS=$PWD/openocd-esp32/share/openocd/scripts
+    export PATH=$PWD/openocd-esp32/bin:$PATH
+  fi
+
  if [[ -n "$CI_PYTHON_TOOL_REPO" ]]; then
    git clone --quiet --depth=1 -b ${CI_PYTHON_TOOL_BRANCH} https://gitlab-ci-token:${ESPCI_TOKEN}@${GITLAB_HTTPS_HOST}/espressif/${CI_PYTHON_TOOL_REPO}.git
    pip install ./${CI_PYTHON_TOOL_REPO}
    rm -rf ${CI_PYTHON_TOOL_REPO}
  fi

-.set_include_nightly_run: &set_include_nightly_run |
-  if echo "$CI_MERGE_REQUEST_LABELS" | egrep "^([^,\n\r]+,)*include_nightly_run(,[^,\n\r]+)*$"; then
-    export INCLUDE_NIGHTLY_RUN="1"
-  fi
-
 before_script:
-  - source tools/ci/utils.sh
-  - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-  - *set_include_nightly_run
-  - source tools/ci/setup_python.sh
+  - *common-before_scripts
+  - *setup_tools_and_idf_python_venv
  - add_gitlab_ssh_keys
-  - source tools/ci/configure_ci_environment.sh
-  - *setup_tools_unless_target_test
  - fetch_submodules
-  - *download_test_python_contraint_file
-  - $IDF_PATH/tools/idf_tools.py install-python-env
-  # install esptool globally, TODO: remove this, IDFCI-1207
-  - pip install esptool -c ~/.espressif/${CI_PYTHON_CONSTRAINT_FILE}
-  - *download_test_python_tool
-
-# used for check scripts which we want to run unconditionally
-.before_script_no_sync_submodule:
-  before_script:
-    - echo "Not setting up GitLab key, not fetching submodules"
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - source tools/ci/setup_python.sh
-    - source tools/ci/configure_ci_environment.sh
-    - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env

 .before_script_minimal:
  before_script:
-    - echo "Only load utils.sh"
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-
-.before_script_integration_test:
-  before_script:
-    # needn't install idf python env
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - add_gitlab_ssh_keys
+    - *common-before_scripts

 .before_script_macos:
  before_script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
-    - *download_test_python_contraint_file
-    - $IDF_PATH/tools/idf_tools.py install-python-env
+    - *common-before_scripts
    # On macOS, these tools need to be installed
+    - export IDF_TOOLS_PATH="${HOME}/.espressif_runner_${CI_RUNNER_ID}_${CI_CONCURRENT_ID}"
    - $IDF_PATH/tools/idf_tools.py --non-interactive install cmake ninja
    # This adds tools (compilers) and the version-specific Python environment to PATH
-    - *setup_tools_unless_target_test
-    # Install packages required by CI scripts into IDF Python environment
-    - pip install -r $IDF_PATH/tools/requirements/requirements.ci.txt
-    - source tools/ci/configure_ci_environment.sh
-    # Part of tools/ci/setup_python.sh; we don't use pyenv on macOS, so can't run the rest of the script.
-    - export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH"
+    - *setup_tools_and_idf_python_venv
    - fetch_submodules

-.before_script_pytest:
-  before_script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - source tools/ci/setup_python.sh
-    - add_gitlab_ssh_keys
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_tools_unless_target_test
-    - fetch_submodules
-    - *download_test_python_contraint_file
-    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
-    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
-
 .before_script_build_jobs:
  before_script:
-    - source tools/ci/utils.sh
-    - is_based_on_commits $REQUIRED_ANCESTOR_COMMITS
-    - *set_include_nightly_run
-    - source tools/ci/setup_python.sh
+    - *common-before_scripts
+    - *setup_tools_and_idf_python_venv
    - add_gitlab_ssh_keys
-    - source tools/ci/configure_ci_environment.sh
-    - *setup_tools_unless_target_test
    - fetch_submodules
-    - *download_test_python_contraint_file
-    # only need ci and pytest related packages, ttfw requirements would be mocked if not installed
-    - $IDF_PATH/tools/idf_tools.py install-python-env --features pytest,ci
-    - eval "$($IDF_PATH/tools/idf_tools.py export)"  # use idf venv instead
    - export EXTRA_CFLAGS=${PEDANTIC_CFLAGS}
    - export EXTRA_CXXFLAGS=${PEDANTIC_CXXFLAGS}