natas_hw
/
openvino
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[GHA] Replace sscache with compressed ccache (#24039) 

### Details:
- use custom compressed action to restore/save cache, it should solve
network related issued with sccache and speedup the build from cache
(from 10 to 6 minutes)
 - 
### Tickets:
 - *ticket-id*
This commit is contained in:
Mikhail Ryzhov 2024-04-18 20:49:34 +02:00 committed by GitHub
parent aa2beefff8
commit dc8db45db4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 60 additions and 98 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
.github/workflows/windows.yml vendored
View File

@ -51,13 +51,11 @@ jobs:
env:
CMAKE_BUILD_TYPE: 'Release'
CMAKE_GENERATOR: 'Ninja Multi-Config'
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
SCCACHE_IGNORE_SERVER_IO_ERROR: 1
SCCACHE_SERVER_PORT: 35555
SCCACHE_IDLE_TIMEOUT: 0
SCCACHE_ERROR_LOG: "${{ github.workspace }}\\openvino\\sccache_log.txt"
SCCACHE_LOG: warn
CMAKE_CXX_COMPILER_LAUNCHER: ccache
CMAKE_C_COMPILER_LAUNCHER: ccache
CCACHE_REMOTE_DIR: "C:\\mount\\caches\\ccache\\windows2022_x86_64\\${{ github.base_ref || github.ref_name }}"
CCACHE_DIR: ${{ github.workspace }}\\ccache
CCACHE_MAXSIZE: 3G
OPENVINO_REPO: "${{ github.workspace }}\\openvino"
OPENVINO_CONTRIB_REPO: "${{ github.workspace }}\\openvino_contrib"
INSTALL_DIR: "${{ github.workspace }}\\openvino_install"
@ -65,7 +63,6 @@ jobs:
INSTALL_TEST_DIR: "${{ github.workspace }}\\tests_install"
BUILD_DIR: "${{ github.workspace }}\\openvino_build"
# TODO: specify version of compiler here
SCCACHE_AZURE_KEY_PREFIX: windows2022_x86_64_Release
if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }}
steps:
@ -124,10 +121,12 @@ jobs:
# For getting rid of SSL issues during model downloading for unit tests
python3 -m pip install certifi
- name: Install sccache
uses: mozilla-actions/sccache-action@v0.0.4
with:
version: "v0.7.5"
- name: Download and install ccache
run: |
Invoke-WebRequest -Uri 'https://github.com/ccache/ccache/releases/download/v4.9.1/ccache-4.9.1-windows-x86_64.zip' -OutFile 'ccache.zip'
Expand-Archive -Path 'ccache.zip' -DestinationPath 'C:\temp\ccache'
Move-Item -Path 'C:\temp\ccache\*' -Destination 'C:\ccache'
Add-Content -Path $env:GITHUB_PATH -Value "C:\ccache"
- name: Install build dependencies
run: |
@ -140,6 +139,17 @@ jobs:
# Build
#
- name: Setup ccache
uses: ./openvino/.github/actions/cache
with:
save-always: ${{ github.event_name == 'push' && 'true' || 'false' }}
cleanup-always: ${{ github.event_name == 'push' && 'true' || 'false' }}
cache-path: ${{ env.CCACHE_REMOTE_DIR }}
path: ${{ env.CCACHE_DIR }}
key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-ccache
- name: Configure Developer Command Prompt for Microsoft Visual C++
uses: ilammy/msvc-dev-cmd@v1
@ -162,27 +172,14 @@ jobs:
-S ${{ env.OPENVINO_REPO }} `
-B ${{ env.BUILD_DIR }}
- name: Clean sccache stats
run: '& "$Env:SCCACHE_PATH" --zero-stats'
# to get more information on the issue
# described in the next step
- name: Show which network ports are used
run: netstat -ban
# the case is the following:
# sccache: error: An attempt was made to access a socket in a way forbidden by its access permissions. (os error 10013)
# This looks like the attempt to use
# a port below 1024 or a port
# which is occupied by another app
- name: Stop sccache server just in case
run: '& "$Env:SCCACHE_PATH" --stop-server'
- name: Clean ccache stats
run: '& ccache --zero-stats'
- name: Cmake build - OpenVINO
run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --verbose
- name: Show sccache stats
run: '& "$Env:SCCACHE_PATH" --show-stats'
- name: Show ccache stats
run: '& ccache --show-stats'
- name: Cmake install - OpenVINO
run: |
@ -231,13 +228,6 @@ jobs:
#
# Upload build artifacts and logs
#
- name: Upload build logs
uses: actions/upload-artifact@v4
if: always()
with:
name: build_logs
path: ${{ env.SCCACHE_ERROR_LOG }}
if-no-files-found: 'ignore'
- name: Upload openvino package
uses: actions/upload-artifact@v4

96
.github/workflows/windows_conditional_compilation.yml vendored
View File

@ -46,7 +46,7 @@ jobs:
Build:
needs: Smart_CI
timeout-minutes: 180
timeout-minutes: 300 # 5 hours without cache
defaults:
run:
shell: pwsh
@ -54,13 +54,11 @@ jobs:
env:
CMAKE_BUILD_TYPE: 'Release'
CMAKE_GENERATOR: 'Ninja Multi-Config'
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
SCCACHE_IGNORE_SERVER_IO_ERROR: 1
SCCACHE_SERVER_PORT: 35555
SCCACHE_IDLE_TIMEOUT: 0
SCCACHE_ERROR_LOG: "${{ github.workspace }}\\openvino\\sccache_log.txt"
SCCACHE_LOG: warn
CMAKE_CXX_COMPILER_LAUNCHER: ccache
CMAKE_C_COMPILER_LAUNCHER: ccache
CCACHE_REMOTE_DIR: "C:\\mount\\caches\\ccache\\windows2022_x86_64_itt\\${{ github.base_ref || github.ref_name }}"
CCACHE_DIR: ${{ github.workspace }}\\ccache
CCACHE_MAXSIZE: 3G
OPENVINO_REPO: "${{ github.workspace }}\\openvino"
INSTALL_DIR: "${{ github.workspace }}\\openvino_install"
INSTALL_TEST_DIR: "${{ github.workspace }}\\tests_install"
@ -68,7 +66,6 @@ jobs:
MODELS_PATH: "${{ github.workspace }}\\testdata"
SELECTIVE_BUILD_STAT_DIR: "${{ github.workspace }}\\selective_build_stat"
# TODO: specify version of compiler here
SCCACHE_AZURE_KEY_PREFIX: windows2022_x86_64_itt_Release
if: ${{ !needs.smart_ci.outputs.skip_workflow && github.event_name != 'merge_group' }}
steps:
@ -104,10 +101,12 @@ jobs:
should-setup-pip-paths: 'false'
self-hosted-runner: 'false'
- name: Install sccache
uses: mozilla-actions/sccache-action@v0.0.4
with:
version: "v0.7.5"
- name: Download and install ccache
run: |
Invoke-WebRequest -Uri 'https://github.com/ccache/ccache/releases/download/v4.9.1/ccache-4.9.1-windows-x86_64.zip' -OutFile 'ccache.zip'
Expand-Archive -Path 'ccache.zip' -DestinationPath 'C:\temp\ccache'
Move-Item -Path 'C:\temp\ccache\*' -Destination 'C:\ccache'
Add-Content -Path $env:GITHUB_PATH -Value "C:\ccache"
- name: Install build dependencies
run: |
@ -138,6 +137,17 @@ jobs:
# Build
#
- name: Setup ccache
uses: ./openvino/.github/actions/cache
with:
save-always: ${{ github.event_name == 'push' && 'true' || 'false' }}
cleanup-always: ${{ github.event_name == 'push' && 'true' || 'false' }}
cache-path: ${{ env.CCACHE_REMOTE_DIR }}
path: ${{ env.CCACHE_DIR }}
key: ${{ runner.os }}-${{ runner.arch }}-ccache-${{ github.sha }}
restore-keys: |
${{ runner.os }}-${{ runner.arch }}-ccache
- name: Configure Developer Command Prompt for Microsoft Visual C++
uses: ilammy/msvc-dev-cmd@v1
@ -159,29 +169,16 @@ jobs:
-S ${{ env.OPENVINO_REPO }} `
-B ${{ env.BUILD_DIR }}
- name: Clean sccache stats
run: '& "$Env:SCCACHE_PATH" --zero-stats'
# to get more information on the issue
# described in the next step
- name: Show which network ports are used
run: netstat -ban
# the case is the following:
# sccache: error: An attempt was made to access a socket in a way forbidden by its access permissions. (os error 10013)
# This looks like the attempt to use
# a port below 1024 or a port
# which is occupied by another app
- name: Stop sccache server just in case
run: '& "$Env:SCCACHE_PATH" --stop-server'
- name: Clean ccache stats
run: '& ccache --zero-stats'
- name: Cmake build - CC COLLECT
run: |
cmake --build ${{ env.BUILD_DIR }} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }} && `
cmake --build ${{ env.BUILD_DIR }} --parallel 8 --config ${{ env.CMAKE_BUILD_TYPE }} --target sea_itt_lib
- name: Show sccache stats
run: '& "$Env:SCCACHE_PATH" --show-stats'
- name: Show ccache stats
run: '& ccache --show-stats'
- name: Cmake install - OpenVINO
run: cmake -DCMAKE_INSTALL_PREFIX=${{ env.INSTALL_DIR }} -P ${{ env.BUILD_DIR }}/cmake_install.cmake
@ -242,13 +239,6 @@ jobs:
#
# Upload build artifacts and logs
#
- name: Upload build logs
uses: actions/upload-artifact@v4
if: always()
with:
name: build_logs
path: ${{ env.SCCACHE_ERROR_LOG }}
if-no-files-found: 'ignore'
- name: Upload selective build statistics package
if: ${{ always() }}
@ -275,16 +265,10 @@ jobs:
runs-on: aks-win-8-cores-64gb
env:
CMAKE_BUILD_TYPE: 'Release'
CMAKE_CXX_COMPILER_LAUNCHER: sccache
CMAKE_C_COMPILER_LAUNCHER: sccache
SCCACHE_IGNORE_SERVER_IO_ERROR: 1
SCCACHE_SERVER_PORT: 35555
SCCACHE_IDLE_TIMEOUT: 0
OPENVINO_REPO: "${{ github.workspace }}\\openvino"
BUILD_DIR: "${{ github.workspace }}\\openvino_build"
MODELS_PATH: "${{ github.workspace }}\\testdata"
SELECTIVE_BUILD_STAT_DIR: "${{ github.workspace }}\\selective_build_stat"
SCCACHE_AZURE_KEY_PREFIX: windows2022_x86_64_cc_Release
if: "!needs.smart_ci.outputs.skip_workflow"
steps:
@ -318,11 +302,6 @@ jobs:
should-setup-pip-paths: 'false'
self-hosted-runner: 'false'
- name: Install sccache
uses: mozilla-actions/sccache-action@v0.0.4
with:
version: "v0.5.4"
- name: CMake configure - CC ON
run: |
cmake `
@ -340,15 +319,9 @@ jobs:
-S ${{ env.OPENVINO_REPO }} `
-B ${{ env.BUILD_DIR }}
- name: Clean sccache stats
run: '& "$Env:SCCACHE_PATH" --zero-stats'
- name: Cmake build - CC ON
run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} --target benchmark_app
- name: Show sccache stats
run: '& "$Env:SCCACHE_PATH" --show-stats'
- name: List bin files
shell: cmd
run: dir ${{ env.OPENVINO_REPO }}\bin\ /s
@ -402,14 +375,13 @@ jobs:
- name: Install python dependencies for run_parallel.py
run: python3 -m pip install -r ${{ env.INSTALL_TEST_DIR }}/layer_tests_summary/requirements.txt
# Windows pipeline is in nightly mode, uncomment once there is a consistent cache creation
# - name: Restore tests execution time
# uses: actions/cache/restore@v3
# with:
# path: ${{ env.PARALLEL_TEST_CACHE }}
# key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}
# restore-keys: |
# ${{ runner.os }}-tests-functional-cpu-stamp
- name: Restore tests execution time
uses: actions/cache/restore@v4
with:
path: ${{ env.PARALLEL_TEST_CACHE }}
key: ${{ runner.os }}-tests-functional-cpu-stamp-${{ github.sha }}
restore-keys: |
${{ runner.os }}-tests-functional-cpu-stamp
- name: Intel CPU plugin func tests (parallel)
shell: cmd