Merge branch 'feature/drop_py2_support' into 'master'

Drop support for unsupported Python versions

Closes IDF-1676

See merge request espressif/esp-idf!13622

.gitlab/ci/build.yml

@@ -1,6 +1,6 @@
 .build_template:
   stage: build
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - build
   variables:
@@ -105,7 +105,7 @@ build_ssc_esp32c3:
     OUTPUT_PATH: ${CI_PROJECT_DIR}/tools/unit-test-app/output
     BUILD_SYSTEM: "cmake"
     TEST_TYPE: "unit_test"
-    PYTHON_VER: 3
+    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
     - ${IDF_PATH}/tools/ci/find_apps_build_apps.sh
@@ -156,7 +156,7 @@ build_esp_idf_tests_cmake_esp32c3:
     TEST_TYPE: example_test
     LOG_PATH: ${CI_PROJECT_DIR}/log_${TEST_PREFIX}
     BUILD_PATH: ${CI_PROJECT_DIR}/build_${TEST_PREFIX}
-    PYTHON_VER: 3
+    PYTHON_VER: 3.6.13
     LDGEN_CHECK_MAPPING: 1
   script:
     # it's not possible to build 100% out-of-tree and have the "artifacts"

.gitlab/ci/deploy.yml

@@ -1,7 +1,7 @@
 .deploy_job_template:
   extends: .before_script_no_sync_submodule
   stage: deploy
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - deploy
 

.gitlab/ci/docs.yml

@@ -55,6 +55,8 @@ check_docs_lang_sync:
 
 .build_docs_template:
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.10
   tags:
     - build_docs
   dependencies: []
@@ -151,6 +153,8 @@ build_docs_pdf:
   extends:
     - .before_script_no_sync_submodule
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.10
   stage: test_deploy
   tags:
     - deploy

.gitlab/ci/host-test.yml

@@ -1,7 +1,9 @@
 .host_test_template:
   extends: .rules:test:host_test
   stage: host_test
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.13
   tags:
     - host_test
   dependencies: []
@@ -11,7 +13,7 @@ test_nvs_on_host:
   extends: .host_test_template
   script:
     - cd components/nvs_flash/test_nvs_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh make test
+    - make test
 
 test_nvs_coverage:
   extends:
@@ -33,7 +35,7 @@ test_partition_table_on_host:
     - build
   script:
     - cd components/partition_table/test_gen_esp32part_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./gen_esp32part_tests.py
+    - ./gen_esp32part_tests.py
 
 test_wl_on_host:
   extends: .host_test_template
@@ -55,10 +57,10 @@ test_ldgen_on_host:
   extends: .host_test_template
   script:
     - cd tools/ldgen/test
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_fragments.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_generation.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_entity.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_output_commands.py
+    - ./test_fragments.py
+    - ./test_generation.py
+    - ./test_entity.py
+    - ./test_output_commands.py
   variables:
     LC_ALL: C.UTF-8
 
@@ -66,7 +68,7 @@ test_ldgen_on_host:
   extends:
     - .host_test_template
     - .rules:labels:fuzzer_test-weekend_test
-  image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+  image: $AFL_FUZZER_TEST_IMAGE
   artifacts:
     when: always
     paths:
@@ -111,7 +113,7 @@ test_spiffs_on_host:
     - cd components/spiffs/test_spiffs_host/
     - make test
     - cd ../test_spiffsgen
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_spiffsgen.py
+    - ./test_spiffsgen.py
 
 test_multi_heap_on_host:
   extends: .host_test_template
@@ -125,26 +127,26 @@ test_certificate_bundle_on_host:
     - build
   script:
     - cd components/mbedtls/esp_crt_bundle/test_gen_crt_bundle/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_gen_crt_bundle.py
+    - ./test_gen_crt_bundle.py
 
 test_confserver:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/confserver
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confserver.py
+    - ./test_confserver.py
 
 test_gen_kconfig_doc:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/gen_kconfig_doc/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_target_visibility.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_kconfig_out.py
+    - ./test_target_visibility.py
+    - ./test_kconfig_out.py
 
 test_confgen:
   extends: .host_test_template
   script:
     - cd tools/kconfig_new/test/confgen/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_confgen.py
+    - ./test_confgen.py
 
 test_idf_monitor:
   extends: .host_test_template
@@ -168,7 +170,7 @@ test_idf_size:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/test_idf_size
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_idf_py:
   extends: .host_test_template
@@ -176,7 +178,7 @@ test_idf_py:
     LC_ALL: C.UTF-8
   script:
     - cd ${IDF_PATH}/tools/test_idf_py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_py.py
+    - ./test_idf_py.py
 
 test_idf_tools:
   extends: .host_test_template
@@ -189,7 +191,7 @@ test_idf_tools:
     # Remove Xtensa and ULP toolchains from the PATH, tests will expect a clean environment
     - export PATH=$(p=$(echo $PATH | tr ":" "\n" | grep -v "/root/.espressif/tools\|/opt/espressif" | tr "\n" ":"); echo ${p%:})
     - cd ${IDF_PATH}/tools/test_idf_tools
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_idf_tools.py
+    - ./test_idf_tools.py
     # Test for create virtualenv. It must be invoked from Python, not from virtualenv.
     - cd ${IDF_PATH}/tools
     - python3 ./idf_tools.py install-python-env
@@ -203,12 +205,10 @@ test_esp32_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
+    - ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
     - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found for esp32 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py ${IDF_PATH}/components/efuse/esp32/esp_efuse_table.csv
-    - git diff --exit-code -- esp32/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_esp32s2_efuse_table_on_host:
   extends: .host_test_template
@@ -219,12 +219,10 @@ test_esp32s2_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
+    - ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
     - git diff --exit-code -- esp32s2/esp_efuse_table.c || { echo 'Differences found for esp32s2 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32s2" ${IDF_PATH}/components/efuse/esp32s2/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s2/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_esp32s3_efuse_table_on_host:
   extends: .host_test_template
@@ -235,12 +233,10 @@ test_esp32s3_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
+    - ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
     - git diff --exit-code -- esp32s3/esp_efuse_table.c || { echo 'Differences found for esp32s3 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32s3" ${IDF_PATH}/components/efuse/esp32s3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32s3/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_esp32c3_efuse_table_on_host:
   extends: .host_test_template
@@ -251,12 +247,10 @@ test_esp32c3_efuse_table_on_host:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/components/efuse/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
+    - ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
     - git diff --exit-code -- esp32c3/esp_efuse_table.c || { echo 'Differences found for esp32c3 target. Please run make efuse_common_table or idf.py efuse_common_table and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./efuse_table_gen.py -t "esp32c3" ${IDF_PATH}/components/efuse/esp32c3/esp_efuse_table.csv
-    - git diff --exit-code -- esp32c3/esp_efuse_table.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
     - cd ${IDF_PATH}/components/efuse/test_efuse_host
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./efuse_tests.py
+    - ./efuse_tests.py
 
 test_espcoredump:
   extends: .host_test_template
@@ -268,7 +262,7 @@ test_espcoredump:
     expire_in: 1 week
   script:
     - cd components/espcoredump/test/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_espcoredump.sh
+    - ./test_espcoredump.sh
 
 test_logtrace_proc:
   extends: .host_test_template
@@ -280,7 +274,7 @@ test_logtrace_proc:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/logtrace
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_sysviewtrace_proc:
   extends: .host_test_template
@@ -292,7 +286,7 @@ test_sysviewtrace_proc:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/esp_app_trace/test/sysview
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test.sh
+    - ./test.sh
 
 test_mkdfu:
   extends: .host_test_template
@@ -300,17 +294,19 @@ test_mkdfu:
     LC_ALL: C.UTF-8
   script:
     - cd ${IDF_PATH}/tools/test_mkdfu
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkdfu.py
+    - ./test_mkdfu.py
 
 test_mkuf2:
   extends: .host_test_template
   script:
     - cd ${IDF_PATH}/tools/test_mkuf2
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ./test_mkuf2.py
+    - ./test_mkuf2.py
 
 test_docs:
   extends: .host_test_template
   image: $ESP_IDF_DOC_ENV_IMAGE
+  variables:
+    PYTHON_VER: 3.6.10
   artifacts:
     when: on_failure
     paths:
@@ -318,9 +314,9 @@ test_docs:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/docs/test
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 pip install -r ${IDF_PATH}/docs/requirements.txt
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_docs.py
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.6.10 ./test_sphinx_idf_extensions.py
+    - python -m pip install -r ${IDF_PATH}/docs/requirements.txt
+    - ./test_docs.py
+    - ./test_sphinx_idf_extensions.py
 
 test_autocomplete:
   extends: .host_test_template
@@ -331,7 +327,7 @@ test_autocomplete:
       - ${IDF_PATH}/*.out
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_autocomplete.py
+    - ${IDF_PATH}/tools/ci/test_autocomplete.py
 
 test_detect_python:
   extends: .host_test_template

.gitlab/ci/pre_check.yml

@@ -1,6 +1,6 @@
 .pre_check_base_template:
   stage: pre_check
-  image: $CI_DOCKER_REGISTRY/esp32-ci-env$BOT_DOCKER_IMAGE_TAG
+  image: $ESP_ENV_IMAGE
   tags:
     - host_test
   dependencies: []
@@ -56,16 +56,20 @@ check_python_style:
   extends:
     - .pre_check_base_template
     - .rules:patterns:python-files
+  variables:
+    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
       - flake8_output.txt
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
+    - python -m flake8 --config=$IDF_PATH/.flake8 --output-file=flake8_output.txt --tee --benchmark $IDF_PATH
 
 test_check_kconfigs:
   extends: .pre_check_job_template
+  variables:
+    PYTHON_VER: 3.6.13
   artifacts:
     when: on_failure
     paths:
@@ -77,7 +81,7 @@ test_check_kconfigs:
       - tools/*/*/*/Kconfig*.new
     expire_in: 1 week
   script:
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh ${IDF_PATH}/tools/ci/test_check_kconfigs.py
+    - python ${IDF_PATH}/tools/ci/test_check_kconfigs.py
 
 check_wifi_lib_md5:
   extends: .pre_check_base_template
@@ -91,7 +95,7 @@ check_wifi_lib_md5:
 
 check_fuzzer_compilation:
    extends: .pre_check_base_template
-   image: $CI_DOCKER_REGISTRY/afl-fuzzer-test
+   image: $AFL_FUZZER_TEST_IMAGE
    script:
      - cd ${IDF_PATH}/components/lwip/test_afl_host
      - make MODE=dhcp_server
@@ -122,10 +126,8 @@ check_esp_err_to_name:
     expire_in: 1 week
   script:
     - cd ${IDF_PATH}/tools/
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 2.7.15 ./gen_esp_err_to_name.py
+    - ./gen_esp_err_to_name.py
     - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found. Please run gen_esp_err_to_name.py and commit the changes.'; exit 1; }
-    - ${IDF_PATH}/tools/ci/multirun_with_pyenv.sh -p 3.4.8 ./gen_esp_err_to_name.py
-    - git diff --exit-code -- ../components/esp_common/src/esp_err_to_name.c || { echo 'Differences found between running under Python 2 and 3.'; exit 1; }
 
 scan_tests:
   extends:

.gitlab/merge_request_templates/Mixed Template.md

@@ -17,7 +17,6 @@ _This entire section can be deleted if all items are checked._
 * [ ] Release note entry if this is a new public feature, or a fix for an issue introduced in the previous release.
 * [ ] The commit log is clean and ready to merge.
 * [ ] All relevant CI jobs have been run, i.e. jobs which cover the code changed by the MR.
-* [ ] Pipeline with Python 3 has been successfully run, i.e. `@bot ...; with Python3`, if the MR adds or changes Python code.
 
 ---