From 10177bfb89d4a2b765c9950e52151c7ba94f256d Mon Sep 17 00:00:00 2001 From: Andrei Kashchikhin Date: Thu, 30 May 2024 10:46:12 +0100 Subject: [PATCH] [CI] [GHA] Introduce Dockerfiles for test jobs (#24742) ### Tickets: - *141574* --- .github/actions/setup_python/action.yml | 20 +++- .github/dockerfiles/docker_tag | 2 +- .../ubuntu_20_04_x64_nvidia/Dockerfile | 89 ++++++++++++++++++ .../ov_test/ubuntu_20_04_arm64/Dockerfile | 1 + .../ov_test/ubuntu_20_04_x64/Dockerfile | 52 +++++++++++ .../ov_test/ubuntu_22_04_x64/Dockerfile | 52 +++++++++++ .../workflows/job_cpu_functional_tests.yml | 8 -- .github/workflows/job_cxx_unit_tests.yml | 8 -- .github/workflows/job_onnx_models_tests.yml | 19 +--- .github/workflows/job_python_unit_tests.yml | 7 -- .github/workflows/job_samples_tests.yml | 11 +-- .../workflows/job_tensorflow_layer_tests.yml | 8 -- .../workflows/job_tensorflow_models_tests.yml | 19 ---- .github/workflows/job_tokenizers.yml | 1 - .github/workflows/linux.yml | 92 +++++-------------- .../linux_conditional_compilation.yml | 5 +- 16 files changed, 240 insertions(+), 154 deletions(-) create mode 100644 .github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile create mode 100644 .github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile create mode 100644 .github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile diff --git a/.github/actions/setup_python/action.yml b/.github/actions/setup_python/action.yml index 6ead25c2f2e..0d9138bc643 100644 --- a/.github/actions/setup_python/action.yml +++ b/.github/actions/setup_python/action.yml @@ -23,7 +23,19 @@ runs: using: 'composite' steps: - - if: ${{ runner.os == 'Linux' && inputs.self-hosted-runner == 'true' }} + - name: Check if Python is already installed (Linux) + if: ${{ runner.os == 'Linux' }} + shell: bash + id: check_python + run: | + PYTHON_INSTALLED=$(python${{ inputs.version }} -V) || true + if [[ $PYTHON_INSTALLED ]]; then + echo "installed=true" >> $GITHUB_OUTPUT + else + echo "installed=false" >> $GITHUB_OUTPUT + fi + + - if: ${{ runner.os == 'Linux' && inputs.self-hosted-runner == 'true' && steps.check_python.outputs.installed == 'false' }} name: Install 'actions/setup-python@v4' dependencies shell: bash run: apt-get update && apt-get install -y ca-certificates software-properties-common gpg-agent tzdata @@ -31,18 +43,18 @@ runs: DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input TZ: "Europe/London" # to prevent tzdata from waiting user input - - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' }} + - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' && steps.check_python.outputs.installed == 'false' }} name: Setup sudo and python3 shell: bash run: apt-get update && apt-get install -y sudo python3 # Needed for the deadsnakes action - - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' }} + - if: ${{ runner.os == 'Linux' && runner.arch == 'ARM64' && steps.check_python.outputs.installed == 'false' }} name: Setup Python ${{ inputs.version }} uses: akashchi/deadsnakes-action@92417281055a5878a0450f240a5b95883eb2d7e2 with: python-version: ${{ inputs.version }} - - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64') }} + - if: ${{ runner.os == 'macOS' || runner.os == 'Windows' || (runner.os == 'Linux' && runner.arch != 'ARM64' && steps.check_python.outputs.installed == 'false' ) }} name: Setup Python ${{ inputs.version }} uses: actions/setup-python@v5 with: diff --git a/.github/dockerfiles/docker_tag b/.github/dockerfiles/docker_tag index 452490c748b..1143a81836c 100644 --- a/.github/dockerfiles/docker_tag +++ b/.github/dockerfiles/docker_tag @@ -1 +1 @@ -pr-24689 \ No newline at end of file +pr-24742 \ No newline at end of file diff --git a/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile new file mode 100644 index 00000000000..c192227085e --- /dev/null +++ b/.github/dockerfiles/ov_build/ubuntu_20_04_x64_nvidia/Dockerfile @@ -0,0 +1,89 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/nvidia/cuda:11.8.0-runtime-ubuntu20.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + wget \ + git \ + ca-certificates \ + gpg-agent \ + tzdata \ + # Pythons + python3.8-dev \ + python3.8-venv \ + python3.8-distutils \ + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + # For Java API + default-jdk \ + # Compiler \ + gcc-10 \ + g++-10 \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Set gcc-10 as a default compiler +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-10 30 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-10 30 + +# Install sscache +ARG SCCACHE_VERSION="v0.7.5" +ENV SCCACHE_HOME="/opt/sccache" \ + SCCACHE_PATH="/opt/sccache/sccache" + +RUN mkdir ${SCCACHE_HOME} && cd ${SCCACHE_HOME} && \ + SCCACHE_ARCHIVE="sccache-${SCCACHE_VERSION}-x86_64-unknown-linux-musl.tar.gz" && \ + curl -SLO https://github.com/mozilla/sccache/releases/download/${SCCACHE_VERSION}/${SCCACHE_ARCHIVE} && \ + tar -xzf ${SCCACHE_ARCHIVE} --strip-components=1 && rm ${SCCACHE_ARCHIVE} + +# Install CUDA +RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin && \ + mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 && \ + apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub && \ + add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" +RUN apt update && apt install -y \ + libcudnn8=8.9.4.*-1+cuda11.8 \ + libcudnn8-dev=8.9.4.*-1+cuda11.8 \ + libcudnn8-samples=8.9.4.*-1+cuda11.8 \ + cuda-runtime-11-8 \ + cuda-11-8 \ + libcutensor1=1.6.1.5-1 \ + libcutensor-dev=1.6.1.5-1 \ + cuda-drivers=520.61.05-1 && \ + rm -rf /var/lib/apt/lists/* + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py + +# Use Python 3.11 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile index b872db9dc05..68d80858dac 100644 --- a/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_arm64/Dockerfile @@ -48,3 +48,4 @@ RUN python3.11 -m venv venv ENV PATH="/venv/bin:$PATH" ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} +ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages diff --git a/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile new file mode 100644 index 00000000000..0a151be1e68 --- /dev/null +++ b/.github/dockerfiles/ov_test/ubuntu_20_04_x64/Dockerfile @@ -0,0 +1,52 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + git \ + ca-certificates \ + gpg-agent \ + tzdata \ + # Python + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + libhdf5-dev \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3.8 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py + +# Use Python 3.11 as default instead of Python 3.8 +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} +ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages diff --git a/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile new file mode 100644 index 00000000000..1566c2305d0 --- /dev/null +++ b/.github/dockerfiles/ov_test/ubuntu_22_04_x64/Dockerfile @@ -0,0 +1,52 @@ +FROM openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04 + +USER root + +# APT configuration +RUN echo 'Acquire::Retries "10";' > /etc/apt/apt.conf && \ + echo 'APT::Get::Assume-Yes "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::Fix-Broken "true";' >> /etc/apt/apt.conf && \ + echo 'APT::Get::no-install-recommends "true";' >> /etc/apt/apt.conf + +ENV DEBIAN_FRONTEND="noninteractive" \ + TZ="Europe/London" + +RUN apt-get update && \ + apt-get install software-properties-common && \ + add-apt-repository --yes --no-update ppa:git-core/ppa && \ + add-apt-repository --yes --no-update ppa:deadsnakes/ppa && \ + apt-get update && \ + apt-get install \ + curl \ + git \ + ca-certificates \ + gpg-agent \ + tzdata \ + # Python + python3.11-dev \ + python3.11-venv \ + python3.11-distutils \ + libhdf5-dev \ + && \ + rm -rf /var/lib/apt/lists/* + +# Install build dependencies +ADD install_build_dependencies.sh /install_build_dependencies.sh +RUN chmod +x /install_build_dependencies.sh && \ + /install_build_dependencies.sh && \ + rm -rf /var/lib/apt/lists/* + +# Setup pip +ENV PIP_VERSION="24.0" +RUN curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \ + python3 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + python3.11 get-pip.py --no-cache-dir pip==${PIP_VERSION} && \ + rm -f get-pip.py + +# Use Python 3.11 as default +# Using venv here 'cause other methods to switch the default Python on Ubuntu 20 break both system and wheels build +RUN python3.11 -m venv venv +ENV PATH="/venv/bin:$SCCACHE_HOME:$PATH" + +ENV PIP_CACHE_DIR=/mount/caches/pip/linux/${PIP_VERSION} +ENV PIP_INSTALL_PATH=/venv/lib/python3.11/site-packages diff --git a/.github/workflows/job_cpu_functional_tests.yml b/.github/workflows/job_cpu_functional_tests.yml index f8a82031af6..bfa0514950d 100644 --- a/.github/workflows/job_cpu_functional_tests.yml +++ b/.github/workflows/job_cpu_functional_tests.yml @@ -30,10 +30,6 @@ jobs: PARALLEL_TEST_SCRIPT: ${{ github.workspace }}/install/tests/functional_test_utils/layer_tests_summary/run_parallel.py PARALLEL_TEST_CACHE: ${{ github.workspace }}/install/tests/test_cache.lst steps: - - name: Set apt retries - if: runner.os == 'Linux' - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -64,10 +60,6 @@ jobs: tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR popd - - name: Install OpenVINO dependencies (Linux) - if: runner.os == 'Linux' - run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y - - name: Fetch setup_python action uses: actions/checkout@v4 with: diff --git a/.github/workflows/job_cxx_unit_tests.yml b/.github/workflows/job_cxx_unit_tests.yml index 2b2dd9e7572..5ed1d17a27c 100644 --- a/.github/workflows/job_cxx_unit_tests.yml +++ b/.github/workflows/job_cxx_unit_tests.yml @@ -32,10 +32,6 @@ jobs: INSTALL_DIR: ${{ github.workspace }}/install INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests steps: - - name: Set apt retries - if: runner.os == 'Linux' - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -63,10 +59,6 @@ jobs: tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR popd - - name: Install OpenVINO dependencies (Linux) - if: runner.os == 'Linux' - run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -c=gpu -y - # # Tests # diff --git a/.github/workflows/job_onnx_models_tests.yml b/.github/workflows/job_onnx_models_tests.yml index 8da73268e00..f919e2c0478 100644 --- a/.github/workflows/job_onnx_models_tests.yml +++ b/.github/workflows/job_onnx_models_tests.yml @@ -35,10 +35,6 @@ jobs: ONNX_MODEL_ZOO_SHA: "5faef4c33eba0395177850e1e31c4a6a9e634c82" if: ${{ github.event_name != 'merge_group' }} steps: - - name: Set apt retries - if: runner.os == 'Linux' - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -70,27 +66,14 @@ jobs: tar -xzf openvino_tests.tar.gz -C ${INSTALL_DIR} popd - - name: Fetch setup_python action and model_zoo_preprocess script + - name: Fetch model_zoo_preprocess script uses: actions/checkout@v4 with: sparse-checkout: | - .github/actions/setup_python/action.yml src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh sparse-checkout-cone-mode: false path: 'openvino' - - name: Install dependencies - run: | - # install git (required to build pip deps from the sources) - apt-get update && apt-get install --assume-yes --no-install-recommends git ca-certificates git-lfs - - - name: Setup Python 3.11 - uses: ./openvino/.github/actions/setup_python - with: - version: '3.11' - should-setup-pip-paths: 'false' - self-hosted-runner: ${{ contains(inputs.runner, 'aks') }} - - name: Update Models run: bash ${OPENVINO_REPO}/src/frontends/onnx/tests/tests_python/model_zoo_preprocess.sh -d ${MODELS_SHARE_PATH} -o -s "${{ env.ONNX_MODEL_ZOO_SHA }}" diff --git a/.github/workflows/job_python_unit_tests.yml b/.github/workflows/job_python_unit_tests.yml index 4cd988ea66c..1c73272550f 100644 --- a/.github/workflows/job_python_unit_tests.yml +++ b/.github/workflows/job_python_unit_tests.yml @@ -37,9 +37,6 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests steps: - - name: Set apt retries - if: runner.os == 'Linux' - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - name: Download OpenVINO package uses: actions/download-artifact@v4 @@ -70,10 +67,6 @@ jobs: tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR popd - - name: Install OpenVINO dependencies (Linux) - if: runner.os == 'Linux' - run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y -c=gpu - - name: Fetch setup_python action uses: actions/checkout@v4 with: diff --git a/.github/workflows/job_samples_tests.yml b/.github/workflows/job_samples_tests.yml index 534ce33409c..3ca9719e75e 100644 --- a/.github/workflows/job_samples_tests.yml +++ b/.github/workflows/job_samples_tests.yml @@ -31,10 +31,6 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests BUILD_DIR: ${{ github.workspace }}/build steps: - - name: Set apt retries - if: runner.os == 'Linux' - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -63,10 +59,6 @@ jobs: tar -xzf openvino_tests.tar.gz -C $INSTALL_DIR popd - - name: Install OpenVINO dependencies (Linux) - if: runner.os == 'Linux' - run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y - - name: Install OpenVINO dependencies (mac) if: runner.os == 'macOS' run: brew install coreutils @@ -84,7 +76,6 @@ jobs: with: version: '3.11' should-setup-pip-paths: 'false' - self-hosted-runner: ${{ runner.os == 'Linux' }} - name: Build cpp samples - GCC run: $INSTALL_DIR/samples/cpp/build_samples.sh -i $INSTALL_DIR -b $BUILD_DIR/cpp_samples @@ -94,7 +85,7 @@ jobs: - name: Build cpp samples - Clang if: runner.os == 'Linux' run: | - apt-get install -y clang + apt-get update && apt-get install -y clang $INSTALL_DIR/samples/cpp/build_samples.sh -i $INSTALL_DIR -b $BUILD_DIR/cpp_samples_clang env: CMAKE_COMPILE_WARNING_AS_ERROR: 'ON' diff --git a/.github/workflows/job_tensorflow_layer_tests.yml b/.github/workflows/job_tensorflow_layer_tests.yml index bb2efa0eec6..01ed55d91d5 100644 --- a/.github/workflows/job_tensorflow_layer_tests.yml +++ b/.github/workflows/job_tensorflow_layer_tests.yml @@ -41,10 +41,6 @@ jobs: INSTALL_TEST_DIR: ${{ github.workspace }}/install/tests LAYER_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/layer_tests steps: - - name: Set apt retries - if: runner.os == 'Linux' - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -92,10 +88,6 @@ jobs: Expand-Archive openvino_tests.zip -DestinationPath ${{ env.INSTALL_DIR }} popd - - name: Install OpenVINO dependencies (Linux) - if: runner.os == 'Linux' - run: $INSTALL_DIR/install_dependencies/install_openvino_dependencies.sh -c=core -c=dev -y -c=gpu - - name: Fetch setup_python action uses: actions/checkout@v4 with: diff --git a/.github/workflows/job_tensorflow_models_tests.yml b/.github/workflows/job_tensorflow_models_tests.yml index 2dd36814d9c..4635fabefbd 100644 --- a/.github/workflows/job_tensorflow_models_tests.yml +++ b/.github/workflows/job_tensorflow_models_tests.yml @@ -34,19 +34,6 @@ jobs: MODEL_HUB_TESTS_INSTALL_DIR: ${{ github.workspace }}/install/tests/model_hub_tests NUMBER_OF_REPLICAS: 2 steps: - - name: Check sudo - if: ${{ runner.os == 'Linux' }} - run: if [ "$(id -u)" -eq 0 ]; then apt update && apt --assume-yes install sudo; fi - - - name: Set apt retries - if: runner.os == 'Linux' - run: | - if [ "$(id -u)" -eq 0 ]; then - echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - else - sudo sh -c "echo 'Acquire::Retries \"10\";' >> /etc/apt/apt.conf.d/80-retries" - fi - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -95,12 +82,6 @@ jobs: sparse-checkout-cone-mode: false path: 'openvino' - - name: Install dependencies - if: ${{ runner.os == 'Linux' }} - run: | - # install git (required to build pip deps from the sources) - sudo apt-get install --assume-yes --no-install-recommends g++ git ca-certificates wget - - name: Setup Python 3.11 uses: ./openvino/.github/actions/setup_python with: diff --git a/.github/workflows/job_tokenizers.yml b/.github/workflows/job_tokenizers.yml index 5198ee5db99..721e4772b03 100644 --- a/.github/workflows/job_tokenizers.yml +++ b/.github/workflows/job_tokenizers.yml @@ -56,7 +56,6 @@ jobs: install_build_dependencies.sh - name: Setup Python ${{ env.PYTHON_VERSION }} - if: ${{ runner.os != 'Linux' }} # We do not need to install Python on Linux as we use Docker with it installed uses: ./.github/actions/setup_python with: version: ${{ env.PYTHON_VERSION }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 008c68854ca..0a7a1b0375d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -69,6 +69,8 @@ jobs: with: images: | ov_build/ubuntu_20_04_x64 + ov_build/ubuntu_20_04_x64_nvidia + ov_test/ubuntu_20_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -308,22 +310,22 @@ jobs: image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' Samples: - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] if: fromJSON(needs.smart_ci.outputs.affected_components).samples uses: ./.github/workflows/job_samples_tests.yml with: runner: 'aks-linux-4-cores-16gb' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} affected-components: ${{ needs.smart_ci.outputs.affected_components }} JS_API: name: OpenVINO JS API - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] if: fromJSON(needs.smart_ci.outputs.affected_components).JS_API uses: ./.github/workflows/job_openvino_js.yml with: runner: 'aks-linux-4-cores-16gb' - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04"}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}"}' Conformance: needs: [ Build, Smart_CI ] @@ -470,79 +472,79 @@ jobs: name: ONNX Models Tests if: fromJSON(needs.smart_ci.outputs.affected_components).Python_API.test || fromJSON(needs.smart_ci.outputs.affected_components).ONNX_FE.test - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] uses: ./.github/workflows/job_onnx_models_tests.yml with: runner: 'aks-linux-16-cores-32gb' - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' CXX_Unit_Tests: name: C++ unit tests - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] uses: ./.github/workflows/job_cxx_unit_tests.yml with: runner: 'aks-linux-4-cores-16gb' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} affected-components: ${{ needs.smart_ci.outputs.affected_components }} Python_Unit_Tests: name: Python unit tests - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] uses: ./.github/workflows/job_python_unit_tests.yml with: runner: 'aks-linux-4-cores-16gb' - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} TensorFlow_Layer_Tests: name: TensorFlow Layer Tests - needs: [ Build, Smart_CI, Openvino_tokenizers ] + needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_tensorflow_layer_tests.yml with: runner: 'aks-linux-4-cores-16gb' shell: bash - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' affected-components: ${{ needs.smart_ci.outputs.affected_components }} CPU_Functional_Tests: name: CPU functional tests if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] uses: ./.github/workflows/job_cpu_functional_tests.yml with: runner: 'aks-linux-8-cores-32gb' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04' + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_20_04_x64 }} TensorFlow_Models_Tests_Precommit: name: TensorFlow Models tests if: fromJSON(needs.smart_ci.outputs.affected_components).TF_FE.test || fromJSON(needs.smart_ci.outputs.affected_components).TFL_FE.test - needs: [ Build, Smart_CI, Openvino_tokenizers ] + needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-16gb' model_scope: 'precommit' - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' TensorFlow_Models_Tests_Nightly_TF_HUB: name: TensorFlow TF Hub Models tests if: ${{ github.event_name == 'schedule' }} - needs: [ Build, Smart_CI, Openvino_tokenizers ] + needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-32gb' model_scope: 'nightly_tf_hub' - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' TensorFlow_Models_Tests_Nightly_HF: name: TensorFlow Hugging Face Models tests if: ${{ github.event_name == 'schedule' }} - needs: [ Build, Smart_CI, Openvino_tokenizers ] + needs: [ Docker, Build, Smart_CI, Openvino_tokenizers ] uses: ./.github/workflows/job_tensorflow_models_tests.yml with: runner: 'aks-linux-8-cores-32gb' model_scope: 'nightly_hf' - container: '{"image": "openvinogithubactions.azurecr.io/dockerhub/ubuntu:20.04", "volumes": ["/mount:/mount"]}' + container: '{"image": "${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64 }}", "volumes": ["/mount:/mount"]}' # TODO: Switch back to self-hosted runners # container: @@ -560,14 +562,14 @@ jobs: NVIDIA_Plugin: name: NVIDIA plugin - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] timeout-minutes: 15 defaults: run: shell: bash runs-on: aks-linux-16-cores-32gb container: - image: openvinogithubactions.azurecr.io/dockerhub/nvidia/cuda:11.8.0-runtime-ubuntu20.04 + image: ${{ fromJSON(needs.docker.outputs.images).ov_build.ubuntu_20_04_x64_nvidia }} volumes: - /mount:/mount options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING @@ -591,20 +593,6 @@ jobs: if: fromJSON(needs.smart_ci.outputs.affected_components).NVIDIA steps: - - name: Set apt retries - run: echo 'Acquire::Retries "10";' > /etc/apt/apt.conf.d/80-retries - - - name: Fetch install_build_dependencies.sh - uses: actions/checkout@v4 - with: - sparse-checkout: | - install_build_dependencies.sh - sparse-checkout-cone-mode: false - path: ${{ env.OPENVINO_REPO }} - - - name: Install Prerequisites - run: apt update && apt install -y git ca-certificates - - name: Download OpenVINO package uses: actions/download-artifact@v4 with: @@ -634,38 +622,6 @@ jobs: path: ${{ env.OPENVINO_CONTRIB_REPO }} ref: 'master' - # - # Dependencies - # - - - name: Install build dependencies - run: | - ${OPENVINO_REPO}/install_build_dependencies.sh - apt -y --no-install-recommends install software-properties-common curl - - - name: Install sccache - uses: mozilla-actions/sccache-action@v0.0.4 - with: - version: "v0.7.5" - - - name: Install CUDA - run: | - wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin - mv cuda-ubuntu2004.pin /etc/apt/preferences.d/cuda-repository-pin-600 - - apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub - add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /" - apt update - apt install -y \ - libcudnn8=8.9.4.*-1+cuda11.8 \ - libcudnn8-dev=8.9.4.*-1+cuda11.8 \ - libcudnn8-samples=8.9.4.*-1+cuda11.8 \ - cuda-runtime-11-8 \ - cuda-11-8 \ - libcutensor1=1.6.1.5-1 \ - libcutensor-dev=1.6.1.5-1 \ - cuda-drivers=520.61.05-1 - # # Build # diff --git a/.github/workflows/linux_conditional_compilation.yml b/.github/workflows/linux_conditional_compilation.yml index bebf06fdb1d..97e8c533b43 100644 --- a/.github/workflows/linux_conditional_compilation.yml +++ b/.github/workflows/linux_conditional_compilation.yml @@ -66,6 +66,7 @@ jobs: with: images: | ov_build/ubuntu_22_04_x64_cc + ov_test/ubuntu_22_04_x64 registry: 'openvinogithubactions.azurecr.io' dockerfiles_root_dir: '.github/dockerfiles' changed_components: ${{ needs.smart_ci.outputs.changed_components }} @@ -333,11 +334,11 @@ jobs: CPU_Functional_Tests: name: CPU functional tests if: fromJSON(needs.smart_ci.outputs.affected_components).CPU.test - needs: [ Build, Smart_CI ] + needs: [ Docker, Build, Smart_CI ] uses: ./.github/workflows/job_cpu_functional_tests.yml with: runner: 'aks-linux-8-cores-32gb' - image: 'openvinogithubactions.azurecr.io/dockerhub/ubuntu:22.04' + image: ${{ fromJSON(needs.docker.outputs.images).ov_test.ubuntu_22_04_x64 }} Overall_Status: name: ci/gha_overall_status_linux_cc